From c34356b24d5241471b4fb7b4ebbc14bbb924d369 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 9 Jul 2026 10:53:10 -0400 Subject: [PATCH 1/4] Upgrade Pinecone SDK v2 -> v8 (fixes #5) The `createIndex` call omitted `metric`, so the API rejected it with `422 missing field 'metric'`. The v8 SDK client-side-defaults `metric` to `cosine` for dense indexes; also pass it explicitly to match the docs. Handle the v8 breaking changes: - `upsert()` now takes `{ records }` instead of a bare array - `ServerlessSpecCloudEnum` was removed; `cloud` is now typed `string` - `PineconeRecord.values` is now optional Co-Authored-By: Claude Opus 4.8 (1M context) --- package-lock.json | 148 ++---------------------------------- package.json | 2 +- src/index.ts | 4 +- src/recommend.ts | 8 +- src/utils/chunkedUpsert.ts | 2 +- tests/chunkedUpsert.test.ts | 10 +-- 6 files changed, 19 insertions(+), 155 deletions(-) diff --git a/package-lock.json b/package-lock.json index e15c06d..4133173 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "@huggingface/transformers": "^4.2.0", - "@pinecone-database/pinecone": "^2.0.0", + "@pinecone-database/pinecone": "^8.0.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", "console-table-printer": "^2.11.1", @@ -1365,18 +1365,12 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-2.2.2.tgz", - "integrity": "sha512-gbe/4SowHc64pHIm0kBdgY9hVdzsQnnnpcWviwYMB33gOmsL8brvE8fUSpl1dLDvdyXzKcQkzdBsjCDlqgpdMA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-8.0.0.tgz", + "integrity": "sha512-dItFqLdis2Pd5lC67aKn8HhvXajzlOz4+0RyK1CRcZMdSwG8YxUCBtH4yXPC8/6uLxfr2dvMWnRFuKgtPwwajQ==", "license": "Apache-2.0", - "dependencies": { - "@sinclair/typebox": "^0.29.0", - "ajv": "^8.12.0", - "cross-fetch": "^3.1.5", - "encoding": "^0.1.13" - }, "engines": { - "node": ">=14.0.0" + "node": ">=20.0.0" } }, "node_modules/@protobufjs/aspromise": { @@ -1793,12 +1787,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@sinclair/typebox": { - "version": "0.29.6", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", - "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==", - "license": "MIT" - }, "node_modules/@tsconfig/recommended": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/@tsconfig/recommended/-/recommended-1.0.13.tgz", @@ -2218,22 +2206,6 @@ "node": ">=12.0" } }, - "node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2713,15 +2685,6 @@ "simple-wcswidth": "^1.1.2" } }, - "node_modules/cross-fetch": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", - "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", - "license": "MIT", - "dependencies": { - "node-fetch": "^2.7.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2934,15 +2897,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, - "node_modules/encoding": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "license": "MIT", - "dependencies": { - "iconv-lite": "^0.6.2" - } - }, "node_modules/es-abstract": { "version": "1.24.2", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.24.2.tgz", @@ -3630,6 +3584,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, "license": "MIT" }, "node_modules/fast-diff": { @@ -3683,22 +3638,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-uri": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", - "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -4174,18 +4113,6 @@ "node": ">= 0.4" } }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -4745,12 +4672,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -5055,26 +4976,6 @@ "semver": "bin/semver.js" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5642,15 +5543,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve": { "version": "2.0.0-next.7", "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.7.tgz", @@ -5854,12 +5746,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -6461,12 +6347,6 @@ "node": ">=8.0" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "node_modules/tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -7652,22 +7532,6 @@ "url": "https://github.com/sponsors/eemeli" } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 755267d..1f2fc55 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "license": "MIT", "dependencies": { "@huggingface/transformers": "^4.2.0", - "@pinecone-database/pinecone": "^2.0.0", + "@pinecone-database/pinecone": "^8.0.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", "console-table-printer": "^2.11.1", diff --git a/src/index.ts b/src/index.ts index 8dc18ce..cf91003 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,7 +4,6 @@ import * as dotenv from "dotenv"; import { Pinecone, type PineconeRecord, - type ServerlessSpecCloudEnum, } from "@pinecone-database/pinecone"; import { getEnv, validateEnvironmentVariables } from "utils/util.ts"; import cliProgress from "cli-progress"; @@ -26,7 +25,7 @@ const progressBar = new cliProgress.SingleBar( // Index setup const indexName = getEnv("PINECONE_INDEX"); -const indexCloud = getEnv("PINECONE_CLOUD") as ServerlessSpecCloudEnum; +const indexCloud = getEnv("PINECONE_CLOUD"); const indexRegion = getEnv("PINECONE_REGION"); const pinecone = new Pinecone(); @@ -91,6 +90,7 @@ try { await pinecone.createIndex({ name: indexName, dimension: 384, + metric: "cosine", spec: { serverless: { region: indexRegion, cloud: indexCloud } }, waitUntilReady: true, }); diff --git a/src/recommend.ts b/src/recommend.ts index c936960..f03f27f 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -39,7 +39,7 @@ const { query, section } = getQueryingCommandLineArguments(); // We create a simulated user with an interest given a query and a specific section const queryEmbedding = await embedder.embed(query); const queryResult = await index.query({ - vector: queryEmbedding.values, + vector: queryEmbedding.values ?? [], includeMetadata: true, includeValues: true, filter: { @@ -49,9 +49,9 @@ const queryResult = await index.query({ }); // We extract the vectors of the results -const userVectors = queryResult?.matches?.map( - (result: ScoredPineconeRecord) => result.values -); +const userVectors = queryResult?.matches + ?.map((result: ScoredPineconeRecord) => result.values) + .filter((values): values is number[] => values !== undefined); // A couple of functions to calculate mean vector const mean = (arr: number[]): number => diff --git a/src/utils/chunkedUpsert.ts b/src/utils/chunkedUpsert.ts index fc5c3db..b86e4d7 100644 --- a/src/utils/chunkedUpsert.ts +++ b/src/utils/chunkedUpsert.ts @@ -15,7 +15,7 @@ export const chunkedUpsert = async ( await Promise.allSettled( chunks.map(async (chunk) => { try { - await index.namespace(namespace).upsert(chunk); + await index.namespace(namespace).upsert({ records: chunk }); } catch (e) { console.log("Error upserting chunk", e); } diff --git a/tests/chunkedUpsert.test.ts b/tests/chunkedUpsert.test.ts index 20d56c0..f5e8035 100644 --- a/tests/chunkedUpsert.test.ts +++ b/tests/chunkedUpsert.test.ts @@ -24,9 +24,9 @@ describe("chunkedUpsert", () => { expect(result).toBe(true); // 25 vectors, chunk size 10 => 3 upsert calls (10, 10, 5) expect(upsert).toHaveBeenCalledTimes(3); - expect(upsert.mock.calls[0][0]).toHaveLength(10); - expect(upsert.mock.calls[1][0]).toHaveLength(10); - expect(upsert.mock.calls[2][0]).toHaveLength(5); + expect(upsert.mock.calls[0][0].records).toHaveLength(10); + expect(upsert.mock.calls[1][0].records).toHaveLength(10); + expect(upsert.mock.calls[2][0].records).toHaveLength(5); expect(namespace).toHaveBeenCalledWith("ns"); }); @@ -37,8 +37,8 @@ describe("chunkedUpsert", () => { await chunkedUpsert(index, vectors, "ns"); expect(upsert).toHaveBeenCalledTimes(2); - expect(upsert.mock.calls[0][0]).toHaveLength(10); - expect(upsert.mock.calls[1][0]).toHaveLength(1); + expect(upsert.mock.calls[0][0].records).toHaveLength(10); + expect(upsert.mock.calls[1][0].records).toHaveLength(1); }); it("does not upsert anything for an empty vector list", async () => { From 8a3f68fc15ce8c3ccb26142c49a469609cbd54b9 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 9 Jul 2026 11:27:59 -0400 Subject: [PATCH 2/4] Refactor source structure and clean up recommend/index scripts Split the util.ts grab-bag into focused modules (env, cli, chunk) and add a shared DEFAULT_NAMESPACE constant so both scripts agree on the namespace. Clean up recommend.ts: wrap the flow in main(), type the query matches instead of `any`, guard against empty query results (replacing the unchecked non-null assertion), fix the table column/data mismatch, and extract the duplicated table printing into printArticleTable. Wrap index.ts in main() and normalize all imports to relative .ts paths. Simplify chunkedUpsert by dropping the dead outer try/catch and the meaningless boolean return. Type-check tests too. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/constants.ts | 4 + src/embeddings.ts | 2 +- src/index.ts | 46 +++--- src/recommend.ts | 213 +++++++++++++++------------- src/utils/chunk.ts | 6 + src/utils/chunkedUpsert.ts | 33 ++--- src/utils/cli.ts | 24 ++++ src/utils/env.ts | 21 +++ src/utils/util.ts | 54 ------- tests/chunk.test.ts | 23 +++ tests/chunkedUpsert.test.ts | 14 +- tests/{util.test.ts => env.test.ts} | 23 +-- tsconfig.json | 4 +- 13 files changed, 242 insertions(+), 225 deletions(-) create mode 100644 src/constants.ts create mode 100644 src/utils/chunk.ts create mode 100644 src/utils/cli.ts create mode 100644 src/utils/env.ts delete mode 100644 src/utils/util.ts create mode 100644 tests/chunk.test.ts rename tests/{util.test.ts => env.test.ts} (56%) diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000..2a43174 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,4 @@ +// The Pinecone namespace that both scripts use for the article vectors. Keeping +// it in one place ensures `npm run index` writes to the same namespace that +// `npm run recommend` reads from. +export const DEFAULT_NAMESPACE = "default"; diff --git a/src/embeddings.ts b/src/embeddings.ts index 85e9c4d..d137cf8 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -9,7 +9,7 @@ import type { RecordMetadata, } from "@pinecone-database/pinecone"; import type { Document } from "./utils/document.ts"; -import { sliceIntoChunks } from "./utils/util.js"; +import { sliceIntoChunks } from "./utils/chunk.ts"; type DocumentOrString = Document | string; diff --git a/src/index.ts b/src/index.ts index cf91003..776de95 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,19 +1,22 @@ -/* eslint-disable import/no-extraneous-dependencies */ -/* eslint-disable dot-notation */ +// Loads the article dataset into Pinecone. +// +// Reads the news CSV, drops incomplete rows, creates the index if needed, then +// embeds each article and upserts the vectors in batches. For this example we +// only ingest the first 100k rows of the dataset. +// +// Run with: npm run index import * as dotenv from "dotenv"; -import { - Pinecone, - type PineconeRecord, -} from "@pinecone-database/pinecone"; -import { getEnv, validateEnvironmentVariables } from "utils/util.ts"; +import { Pinecone, type PineconeRecord } from "@pinecone-database/pinecone"; import cliProgress from "cli-progress"; -import { embedder } from "embeddings.ts"; -import loadCSVFile from "utils/csvLoader.ts"; -import { dropEmptyRows } from "utils/csv.ts"; -import splitFile from "utils/fileSplitter.ts"; -import type { ArticleRecord } from "types.ts"; +import { getEnv, validateEnvironmentVariables } from "./utils/env.ts"; +import loadCSVFile from "./utils/csvLoader.ts"; +import { dropEmptyRows } from "./utils/csv.ts"; +import splitFile from "./utils/fileSplitter.ts"; import { Document } from "./utils/document.ts"; import { chunkedUpsert } from "./utils/chunkedUpsert.ts"; +import { embedder } from "./embeddings.ts"; +import { DEFAULT_NAMESPACE } from "./constants.ts"; +import type { ArticleRecord } from "./types.ts"; dotenv.config(); validateEnvironmentVariables(); @@ -23,12 +26,13 @@ const progressBar = new cliProgress.SingleBar( cliProgress.Presets.shades_classic ); -// Index setup const indexName = getEnv("PINECONE_INDEX"); const indexCloud = getEnv("PINECONE_CLOUD"); const indexRegion = getEnv("PINECONE_REGION"); const pinecone = new Pinecone(); +// Yields the records in `chunkSize` batches, wrapping each one in a `Document` +// that carries the article text (`pageContent`) plus the chosen metadata fields. async function* processInChunks( records: T[], chunkSize: number, @@ -50,6 +54,7 @@ async function* processInChunks( } } +// Embeds every article and upserts the resulting vectors into the index. async function embedAndUpsert(records: ArticleRecord[], chunkSize: number) { const chunkGenerator = processInChunks< ArticleRecord, @@ -68,23 +73,23 @@ async function embedAndUpsert(records: ArticleRecord[], chunkSize: number) { documents, chunkSize, async (embeddings: PineconeRecord[]) => { - await chunkedUpsert(index, embeddings, "default"); + await chunkedUpsert(index, embeddings, DEFAULT_NAMESPACE); progressBar.increment(embeddings.length); } ); } } -try { +async function main() { const fileParts = await splitFile("./data/all-the-news-2-1.csv", 100000); const firstFile = fileParts[0]; - // For this example, we will use the first file part to create the index + // For this example, we only use the first file part to create the index. const data = await loadCSVFile(firstFile); const clean = dropEmptyRows(data); console.table(clean.slice(0, 5)); - // Create the index if it doesn't already exist + // Create the index if it doesn't already exist. const indexList = await pinecone.listIndexes(); if (!indexList.indexes?.some((index) => index.name === indexName)) { await pinecone.createIndex({ @@ -103,6 +108,9 @@ try { console.log( `Inserted ${progressBar.getTotal()} documents into index ${indexName}` ); -} catch (error) { - console.error(error); } + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/src/recommend.ts b/src/recommend.ts index f03f27f..df9d571 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -1,118 +1,131 @@ -/* eslint-disable import/no-extraneous-dependencies */ -import { - getEnv, - getQueryingCommandLineArguments, - validateEnvironmentVariables, -} from "utils/util.ts"; -import { embedder } from "embeddings.ts"; -import { Table } from "console-table-printer"; +// Queries the article index for a simulated user. +// +// Given a `--query` and `--section`, we fetch the articles the user has +// "read" (the query results), average their vectors into a single taste +// vector, then query again with that mean to produce recommendations. +// +// Run with: npm run recommend -- --query="tennis" --section="Sports" import { Pinecone } from "@pinecone-database/pinecone"; import type { ScoredPineconeRecord } from "@pinecone-database/pinecone"; -import type { ArticleRecord } from "types.ts"; +import { Table } from "console-table-printer"; +import { getEnv, validateEnvironmentVariables } from "./utils/env.ts"; +import { getQueryingCommandLineArguments } from "./utils/cli.ts"; +import { embedder } from "./embeddings.ts"; +import { DEFAULT_NAMESPACE } from "./constants.ts"; +import type { ArticleRecord } from "./types.ts"; + +// Averages a set of equal-length vectors component by component. The resulting +// mean vector represents the user's interests across the articles they've read. +const meanVector = (vectors: number[][]): number[] => { + const mean = (values: number[]): number => + values.reduce((a, b) => a + b, 0) / values.length; + const { length } = vectors[0]; -validateEnvironmentVariables(); + return Array.from({ length }).map((_, i) => mean(vectors.map((v) => v[i]))); +}; -const indexName = getEnv("PINECONE_INDEX"); -const pinecone = new Pinecone(); +// Prints a table of article matches under the given heading. +const printArticleTable = ( + heading: string, + matches: ScoredPineconeRecord[] +) => { + const table = new Table({ + columns: [ + { name: "title", alignment: "left" }, + { name: "article", alignment: "left" }, + { name: "section", alignment: "left" }, + { name: "publication", alignment: "left" }, + ], + }); -// Ensure the index exists -try { - const description = await pinecone.describeIndex(indexName); - if (!description.status?.ready) { - throw new Error( - `Index not ready, description was ${JSON.stringify(description)}` - ); + for (const match of matches) { + const { metadata } = match; + if (metadata) { + const { title, article, section, publication } = metadata; + table.addRow({ + title, + article: `${article.slice(0, 70)}...`, + section, + publication, + }); + } } -} catch (e) { - console.log( - 'An error occurred. Run "npm run index" to load data into the index before querying.' - ); - throw e; -} - -const index = pinecone.index(indexName).namespace("default"); - -await embedder.init("Xenova/all-MiniLM-L6-v2"); - -const { query, section } = getQueryingCommandLineArguments(); -// We create a simulated user with an interest given a query and a specific section -const queryEmbedding = await embedder.embed(query); -const queryResult = await index.query({ - vector: queryEmbedding.values ?? [], - includeMetadata: true, - includeValues: true, - filter: { - section: { $eq: section }, - }, - topK: 10, -}); - -// We extract the vectors of the results -const userVectors = queryResult?.matches - ?.map((result: ScoredPineconeRecord) => result.values) - .filter((values): values is number[] => values !== undefined); - -// A couple of functions to calculate mean vector -const mean = (arr: number[]): number => - arr.reduce((a, b) => a + b, 0) / arr.length; -const meanVector = (vectors: number[][]): number[] => { - const { length } = vectors[0]; - - return Array.from({ length }).map((_, i) => - mean(vectors.map((vec) => vec[i])) - ); + console.log(heading); + table.printTable(); }; -// We calculate the mean vector of the results -// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -const meanVec = meanVector(userVectors!); +async function main() { + validateEnvironmentVariables(); + + const indexName = getEnv("PINECONE_INDEX"); + const pinecone = new Pinecone(); + + // Ensure the index exists and is ready before we query it. + try { + const description = await pinecone.describeIndex(indexName); + if (!description.status?.ready) { + throw new Error( + `Index not ready, description was ${JSON.stringify(description)}` + ); + } + } catch (e) { + console.log( + 'An error occurred. Run "npm run index" to load data into the index before querying.' + ); + throw e; + } -// We query the index with the mean vector to get recommendations for the user -const recommendations = await index.query({ - vector: meanVec, - includeMetadata: true, - includeValues: true, - topK: 10, -}); + const index = pinecone + .index(indexName) + .namespace(DEFAULT_NAMESPACE); -const userPreferences = new Table({ - columns: [ - { name: "title", alignment: "left" }, - { name: "author", alignment: "left" }, - { name: "section", alignment: "left" }, - ], -}); + await embedder.init("Xenova/all-MiniLM-L6-v2"); -const userRecommendations = new Table({ - columns: [ - { name: "title", alignment: "left" }, - { name: "author", alignment: "left" }, - { name: "section", alignment: "left" }, - ], -}); + const { query, section } = getQueryingCommandLineArguments(); -queryResult?.matches?.slice(0, 10).forEach((result: any) => { - const { title, article, publication, section } = result.metadata; - userPreferences.addRow({ - title, - article: `${article.slice(0, 70)}...`, - publication, - section, + // Simulate the articles the user has read: the closest matches to their query + // within the section they're interested in. + const queryEmbedding = await embedder.embed(query); + const queryResult = await index.query({ + vector: queryEmbedding.values ?? [], + includeMetadata: true, + includeValues: true, + filter: { section: { $eq: section } }, + topK: 10, }); -}); -console.log("========== User Preferences =========="); -userPreferences.printTable(); + const readArticles = queryResult.matches ?? []; + if (readArticles.length === 0) { + console.log( + `No articles found for section "${section}" matching "${query}". ` + + "Try a different --query or --section." + ); + return; + } -recommendations?.matches?.slice(0, 10).forEach((result: any) => { - const { title, article, publication, section } = result.metadata; - userRecommendations.addRow({ - title, - article: `${article.slice(0, 70)}...`, - publication, - section, + // Average the read articles' vectors into a single "taste" vector. + const userVectors = readArticles + .map((match) => match.values) + .filter((values): values is number[] => values !== undefined); + const meanVec = meanVector(userVectors); + + // Query again with the taste vector to produce recommendations. + const recommendations = await index.query({ + vector: meanVec, + includeMetadata: true, + includeValues: true, + topK: 10, }); + + printArticleTable("========== User Preferences ==========", readArticles); + printArticleTable( + "=========== Recommendations ==========", + recommendations.matches ?? [] + ); +} + +main().catch((e) => { + console.error(e); + process.exit(1); }); -console.log("=========== Recommendations =========="); -userRecommendations.printTable(); diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts new file mode 100644 index 0000000..7778ca4 --- /dev/null +++ b/src/utils/chunk.ts @@ -0,0 +1,6 @@ +// Splits an array into consecutive chunks of at most `chunkSize` items. Used +// to batch documents for embedding and vectors for upserting. +export const sliceIntoChunks = (arr: T[], chunkSize: number): T[][] => + Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => + arr.slice(i * chunkSize, (i + 1) * chunkSize) + ); diff --git a/src/utils/chunkedUpsert.ts b/src/utils/chunkedUpsert.ts index b86e4d7..3d51b34 100644 --- a/src/utils/chunkedUpsert.ts +++ b/src/utils/chunkedUpsert.ts @@ -1,29 +1,24 @@ import type { Index, PineconeRecord } from "@pinecone-database/pinecone"; -import { sliceIntoChunks } from "./util.ts"; +import { sliceIntoChunks } from "./chunk.ts"; +// Upserts vectors into an index namespace in batches of `chunkSize`. Batches run +// concurrently; a failed batch is logged and skipped so a single bad chunk does +// not abort the whole ingestion run. export const chunkedUpsert = async ( index: Index, - vectors: Array, + vectors: PineconeRecord[], namespace: string, chunkSize = 10 ) => { - // Split the vectors into chunks const chunks = sliceIntoChunks(vectors, chunkSize); - try { - // Upsert each chunk of vectors into the index - await Promise.allSettled( - chunks.map(async (chunk) => { - try { - await index.namespace(namespace).upsert({ records: chunk }); - } catch (e) { - console.log("Error upserting chunk", e); - } - }) - ); - - return true; - } catch (e) { - throw new Error(`Error upserting vectors into index: ${e}`); - } + await Promise.all( + chunks.map(async (chunk) => { + try { + await index.namespace(namespace).upsert({ records: chunk }); + } catch (e) { + console.error("Error upserting chunk", e); + } + }) + ); }; diff --git a/src/utils/cli.ts b/src/utils/cli.ts new file mode 100644 index 0000000..1e68a38 --- /dev/null +++ b/src/utils/cli.ts @@ -0,0 +1,24 @@ +import yargs from "yargs"; +import { hideBin } from "yargs/helpers"; + +// Parses the `--query` and `--section` arguments used by `npm run recommend`. +// Both are required; yargs prints usage and exits if either is missing. +export const getQueryingCommandLineArguments = () => { + const argv = yargs(hideBin(process.argv)) + .option("query", { + alias: "q", + type: "string", + description: "The query to search for", + demandOption: true, + }) + .option("section", { + alias: "s", + type: "string", + description: "The section of the article", + demandOption: true, + }) + .parseSync(); + + const { query, section } = argv; + return { query, section }; +}; diff --git a/src/utils/env.ts b/src/utils/env.ts new file mode 100644 index 0000000..05f44a2 --- /dev/null +++ b/src/utils/env.ts @@ -0,0 +1,21 @@ +// Helpers for reading and validating the environment variables this example +// needs. Values are loaded from a `.env` file via `dotenv` (see the entry +// scripts) before these run. + +// Reads a required environment variable, throwing a clear error if it is unset +// or empty rather than letting an `undefined` propagate deeper into the code. +export const getEnv = (key: string): string => { + const value = process.env[key]; + if (!value) { + throw new Error(`${key} environment variable not set`); + } + return value; +}; + +// Fails fast at startup if any required Pinecone configuration is missing. +export const validateEnvironmentVariables = () => { + getEnv("PINECONE_API_KEY"); + getEnv("PINECONE_INDEX"); + getEnv("PINECONE_CLOUD"); + getEnv("PINECONE_REGION"); +}; diff --git a/src/utils/util.ts b/src/utils/util.ts deleted file mode 100644 index e57a872..0000000 --- a/src/utils/util.ts +++ /dev/null @@ -1,54 +0,0 @@ -import yargs from "yargs"; -import { hideBin } from "yargs/helpers"; - -const sliceIntoChunks = (arr: T[], chunkSize: number) => - Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => - arr.slice(i * chunkSize, (i + 1) * chunkSize) - ); - -const getQueryingCommandLineArguments = () => { - const argv = yargs(hideBin(process.argv)) - .option("query", { - alias: "q", - type: "string", - description: "The query to search for", - demandOption: true, - }) - .option("section", { - alias: "s", - type: "string", - description: "The section of the article", - demandOption: true, - }) - - .parseSync(); - - const { query, section } = argv; - if (!query) { - console.error("Please provide a query"); - process.exit(1); - } - - return { query, section }; -}; - -export const getEnv = (key: string): string => { - const value = process.env[key]; - if (!value) { - throw new Error(`${key} environment variable not set`); - } - return value; -}; - -const validateEnvironmentVariables = () => { - getEnv("PINECONE_API_KEY"); - getEnv("PINECONE_INDEX"); - getEnv("PINECONE_CLOUD"); - getEnv("PINECONE_REGION"); -}; - -export { - getQueryingCommandLineArguments, - sliceIntoChunks, - validateEnvironmentVariables, -}; diff --git a/tests/chunk.test.ts b/tests/chunk.test.ts new file mode 100644 index 0000000..8748230 --- /dev/null +++ b/tests/chunk.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from "vitest"; +import { sliceIntoChunks } from "../src/utils/chunk.ts"; + +describe("sliceIntoChunks", () => { + it("splits an array into evenly sized chunks", () => { + expect(sliceIntoChunks([1, 2, 3, 4], 2)).toEqual([ + [1, 2], + [3, 4], + ]); + }); + + it("puts the remainder in a final, smaller chunk", () => { + expect(sliceIntoChunks([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + }); + + it("returns a single chunk when the chunk size exceeds the length", () => { + expect(sliceIntoChunks([1, 2, 3], 10)).toEqual([[1, 2, 3]]); + }); + + it("returns an empty array for empty input", () => { + expect(sliceIntoChunks([], 3)).toEqual([]); + }); +}); diff --git a/tests/chunkedUpsert.test.ts b/tests/chunkedUpsert.test.ts index f5e8035..e52762a 100644 --- a/tests/chunkedUpsert.test.ts +++ b/tests/chunkedUpsert.test.ts @@ -19,9 +19,8 @@ describe("chunkedUpsert", () => { const { index, namespace, upsert } = makeMockIndex(); const vectors = Array.from({ length: 25 }, (_, i) => record(`${i}`)); - const result = await chunkedUpsert(index, vectors, "ns", 10); + await chunkedUpsert(index, vectors, "ns", 10); - expect(result).toBe(true); // 25 vectors, chunk size 10 => 3 upsert calls (10, 10, 5) expect(upsert).toHaveBeenCalledTimes(3); expect(upsert.mock.calls[0][0].records).toHaveLength(10); @@ -44,20 +43,19 @@ describe("chunkedUpsert", () => { it("does not upsert anything for an empty vector list", async () => { const { index, upsert } = makeMockIndex(); - const result = await chunkedUpsert(index, [], "ns"); + await chunkedUpsert(index, [], "ns"); - expect(result).toBe(true); expect(upsert).not.toHaveBeenCalled(); }); - it("swallows per-chunk errors and still resolves to true", async () => { + it("swallows per-chunk errors and still upserts the other chunks", async () => { const { index, upsert } = makeMockIndex(); upsert.mockRejectedValueOnce(new Error("boom")); const vectors = Array.from({ length: 15 }, (_, i) => record(`${i}`)); - const result = await chunkedUpsert(index, vectors, "ns", 10); - - expect(result).toBe(true); + await expect( + chunkedUpsert(index, vectors, "ns", 10) + ).resolves.toBeUndefined(); expect(upsert).toHaveBeenCalledTimes(2); }); }); diff --git a/tests/util.test.ts b/tests/env.test.ts similarity index 56% rename from tests/util.test.ts rename to tests/env.test.ts index a11e27e..53cb64c 100644 --- a/tests/util.test.ts +++ b/tests/env.test.ts @@ -1,26 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { getEnv, sliceIntoChunks } from "../src/utils/util.ts"; - -describe("sliceIntoChunks", () => { - it("splits an array into evenly sized chunks", () => { - expect(sliceIntoChunks([1, 2, 3, 4], 2)).toEqual([ - [1, 2], - [3, 4], - ]); - }); - - it("puts the remainder in a final, smaller chunk", () => { - expect(sliceIntoChunks([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); - }); - - it("returns a single chunk when the chunk size exceeds the length", () => { - expect(sliceIntoChunks([1, 2, 3], 10)).toEqual([[1, 2, 3]]); - }); - - it("returns an empty array for empty input", () => { - expect(sliceIntoChunks([], 3)).toEqual([]); - }); -}); +import { getEnv } from "../src/utils/env.ts"; describe("getEnv", () => { const KEY = "RECOMMENDER_TEST_ENV_VAR"; diff --git a/tsconfig.json b/tsconfig.json index f8f22f3..3e8cf43 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,6 +22,6 @@ "strictPropertyInitialization": false, "noEmit": true }, - "exclude": ["node_modules/", "dist/", "tests/"], - "include": ["./src"] + "exclude": ["node_modules/", "dist/"], + "include": ["./src", "./tests"] } From c1321dae3ce77d0497b024b1603532c99a1dfb52 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 9 Jul 2026 11:28:07 -0400 Subject: [PATCH 3/4] Drop build layer and tidy package metadata This is a runnable example, not a published library, so remove the tsup build step (tsup, tsup.config.js, main/files fields) and replace CI's Build step with a `typecheck` script (tsc --noEmit). Fix the stale package identity: rename from langchain-ts-starter to article-recommender-example, add a real description, and update keywords. Remove unused dependencies (chalk, tablemark, @types/js-yaml). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 4 +- package-lock.json | 1008 +------------------------------------- package.json | 26 +- tsup.config.js | 11 - 4 files changed, 22 insertions(+), 1027 deletions(-) delete mode 100644 tsup.config.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b3e356..519c314 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,8 @@ jobs: - name: Check formatting run: npm run format:check - - name: Build - run: npm run build + - name: Typecheck + run: npm run typecheck - name: Test run: npm test diff --git a/package-lock.json b/package-lock.json index 4133173..680ac9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,28 +1,25 @@ { - "name": "langchain-ts-starter", + "name": "article-recommender-example", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "langchain-ts-starter", + "name": "article-recommender-example", "version": "1.0.0", "license": "MIT", "dependencies": { "@huggingface/transformers": "^4.2.0", "@pinecone-database/pinecone": "^8.0.0", - "chalk": "^5.2.0", "cli-progress": "^3.12.0", "console-table-printer": "^2.11.1", "dotenv": "^16.1.3", "onnxruntime-node": "^1.15.0", - "tablemark": "^3.0.0", "yargs": "^17.7.2" }, "devDependencies": { "@tsconfig/recommended": "^1.0.2", "@types/cli-progress": "^3.11.0", - "@types/js-yaml": "^4", "@types/node": "^18.15.5", "@types/yargs": "^17.0.24", "@typescript-eslint/eslint-plugin": "^5.51.0", @@ -33,7 +30,6 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.8.3", - "tsup": "^8.5.1", "tsx": "^4.23.0", "typescript": "^5.0.4", "vitest": "^3.2.7" @@ -1287,27 +1283,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -1315,17 +1290,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1829,13 +1793,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/js-yaml": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", - "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2230,13 +2187,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true, - "license": "MIT" - }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -2450,22 +2400,6 @@ "node": ">=8" } }, - "node_modules/bundle-require": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", - "integrity": "sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==", - "dev": true, - "license": "MIT", - "dependencies": { - "load-tsconfig": "^0.2.3" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "peerDependencies": { - "esbuild": ">=0.18" - } - }, "node_modules/cac": { "version": "6.7.14", "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", @@ -2553,18 +2487,6 @@ "node": ">=18" } }, - "node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -2575,22 +2497,6 @@ "node": ">= 16" } }, - "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/cli-progress": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", @@ -2635,16 +2541,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, - "node_modules/commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2652,13 +2548,6 @@ "dev": true, "license": "MIT" }, - "node_modules/confbox": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", - "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", - "dev": true, - "license": "MIT" - }, "node_modules/confusing-browser-globals": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz", @@ -2666,16 +2555,6 @@ "dev": true, "license": "MIT" }, - "node_modules/consola": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", - "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.18.0 || >=16.10.0" - } - }, "node_modules/console-table-printer": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.16.1.tgz", @@ -3691,18 +3570,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/fix-dts-default-cjs-exports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fix-dts-default-cjs-exports/-/fix-dts-default-cjs-exports-1.0.1.tgz", - "integrity": "sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "magic-string": "^0.30.17", - "mlly": "^1.7.4", - "rollup": "^4.34.8" - } - }, "node_modules/flat-cache": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", @@ -3865,15 +3732,6 @@ "node": ">= 0.4" } }, - "node_modules/get-stdin": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", - "integrity": "sha512-jZV7n6jGE3Gt7fgSTJoz91Ak5MuTLwMwkoYdjxuJ/AmjIsE1UC03y/IWkZCQGEvVNS9qoRNwy5BCqxImv0FVeA==", - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/get-symbol-description": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", @@ -4625,16 +4483,6 @@ "dev": true, "license": "ISC" }, - "node_modules/joycon": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", - "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/js-tokens": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", @@ -4722,36 +4570,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lilconfig": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", - "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/antonk52" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/load-tsconfig": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", - "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -4782,15 +4600,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lower-case": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", - "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", - "license": "MIT", - "dependencies": { - "tslib": "^2.0.3" - } - }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4867,24 +4676,12 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/mlly": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", - "integrity": "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.16.0", - "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "ufo": "^1.6.3" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4892,18 +4689,6 @@ "dev": true, "license": "MIT" }, - "node_modules/mz": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", - "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, "node_modules/nanoid": { "version": "3.3.15", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", @@ -4937,16 +4722,6 @@ "dev": true, "license": "MIT" }, - "node_modules/no-case": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", - "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", - "license": "MIT", - "dependencies": { - "lower-case": "^2.0.2", - "tslib": "^2.0.3" - } - }, "node_modules/node-exports-info": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.2.tgz", @@ -4976,16 +4751,6 @@ "semver": "bin/semver.js" } }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -5296,28 +5061,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-types": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", - "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "confbox": "^0.1.8", - "mlly": "^1.7.4", - "pathe": "^2.0.1" - } - }, "node_modules/platform": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", @@ -5363,49 +5106,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/postcss-load-config": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", - "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "lilconfig": "^3.1.1" - }, - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "jiti": ">=1.21.0", - "postcss": ">=8.0.9", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - }, - "postcss": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -5476,20 +5176,6 @@ ], "license": "MIT" }, - "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -5764,17 +5450,6 @@ "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", "license": "MIT" }, - "node_modules/sentence-case": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/sentence-case/-/sentence-case-3.0.4.tgz", - "integrity": "sha512-8LS0JInaQMCRoQ7YUytAo/xUu5W2XnQxV2HI/6uM6U7CITS1RqPElr30V6uIqyMKM9lJGRVFy5/4CuzcixNYSg==", - "license": "MIT", - "dependencies": { - "no-case": "^3.0.4", - "tslib": "^2.0.3", - "upper-case-first": "^2.0.2" - } - }, "node_modules/serialize-error": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", @@ -5961,16 +5636,6 @@ "node": ">=8" } }, - "node_modules/source-map": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", - "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">= 12" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -5981,19 +5646,6 @@ "node": ">=0.10.0" } }, - "node_modules/split-text-to-chunks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/split-text-to-chunks/-/split-text-to-chunks-1.0.0.tgz", - "integrity": "sha512-HLtEwXK/T4l7QZSJ/kOSsZC0o5e2Xg3GzKKFxm0ZexJXw0Bo4CaEl39l7MCSRHk9EOOL5jT8JIDjmhTtcoe6lQ==", - "license": "MIT", - "dependencies": { - "get-stdin": "^5.0.1", - "minimist": "^1.2.0" - }, - "bin": { - "wordwrap": "cli.js" - } - }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", @@ -6150,29 +5802,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "node_modules/sucrase": { - "version": "3.35.1", - "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", - "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.2", - "commander": "^4.0.0", - "lines-and-columns": "^1.1.6", - "mz": "^2.7.0", - "pirates": "^4.0.1", - "tinyglobby": "^0.2.11", - "ts-interface-checker": "^0.1.9" - }, - "bin": { - "sucrase": "bin/sucrase", - "sucrase-node": "bin/sucrase-node" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -6199,19 +5828,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/tablemark": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tablemark/-/tablemark-3.1.0.tgz", - "integrity": "sha512-IwO6f0SEzp1Z+zqz/7ANUmeEac4gaNlknWyj/S9aSg11wZmWYnLeyI/xXvEOU88BYUIf8y30y0wxB58xIKrVlQ==", - "license": "MIT", - "dependencies": { - "sentence-case": "^3.0.4", - "split-text-to-chunks": "^1.0.0" - }, - "engines": { - "node": ">=14.16" - } - }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -6219,29 +5835,6 @@ "dev": true, "license": "MIT" }, - "node_modules/thenify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", - "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0" - } - }, - "node_modules/thenify-all": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "thenify": ">= 3.1.0 < 4" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -6347,23 +5940,6 @@ "node": ">=8.0" } }, - "node_modules/tree-kill": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", - "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", - "dev": true, - "license": "MIT", - "bin": { - "tree-kill": "cli.js" - } - }, - "node_modules/ts-interface-checker": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", - "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", @@ -6381,566 +5957,20 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "optional": true }, - "node_modules/tsup": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", - "integrity": "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==", + "node_modules/tsutils": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", + "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", "dev": true, "license": "MIT", "dependencies": { - "bundle-require": "^5.1.0", - "cac": "^6.7.14", - "chokidar": "^4.0.3", - "consola": "^3.4.0", - "debug": "^4.4.0", - "esbuild": "^0.27.0", - "fix-dts-default-cjs-exports": "^1.0.0", - "joycon": "^3.1.1", - "picocolors": "^1.1.1", - "postcss-load-config": "^6.0.1", - "resolve-from": "^5.0.0", - "rollup": "^4.34.8", - "source-map": "^0.7.6", - "sucrase": "^3.35.0", - "tinyexec": "^0.3.2", - "tinyglobby": "^0.2.11", - "tree-kill": "^1.2.2" - }, - "bin": { - "tsup": "dist/cli-default.js", - "tsup-node": "dist/cli-node.js" + "tslib": "^1.8.1" }, "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@microsoft/api-extractor": "^7.36.0", - "@swc/core": "^1", - "postcss": "^8.4.12", - "typescript": ">=4.5.0" - }, - "peerDependenciesMeta": { - "@microsoft/api-extractor": { - "optional": true - }, - "@swc/core": { - "optional": true - }, - "postcss": { - "optional": true - }, - "typescript": { - "optional": true - } - } - }, - "node_modules/tsup/node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsup/node_modules/esbuild": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" - } - }, - "node_modules/tsup/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/tsutils": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", - "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tslib": "^1.8.1" - }, - "engines": { - "node": ">= 6" + "node": ">= 6" }, "peerDependencies": { "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" @@ -7089,13 +6119,6 @@ "node": ">=14.17" } }, - "node_modules/ufo": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", - "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", - "dev": true, - "license": "MIT" - }, "node_modules/unbox-primitive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", @@ -7121,15 +6144,6 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "license": "MIT" }, - "node_modules/upper-case-first": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/upper-case-first/-/upper-case-first-2.0.2.tgz", - "integrity": "sha512-514ppYHBaKwfJRK/pNC6c/OxfGa0obSnAl106u97Ed0I625Nin96KAjttZF6ZL3e1XLtphxnqrOi9iWgm+u+bg==", - "license": "MIT", - "dependencies": { - "tslib": "^2.0.3" - } - }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/package.json b/package.json index 1f2fc55..aff6329 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,12 @@ { - "name": "langchain-ts-starter", + "name": "article-recommender-example", "version": "1.0.0", - "description": "", + "description": "A content-based article recommender built on Pinecone similarity search.", "type": "module", - "main": "./dist/index.js", - "files": [ - "dist" - ], "scripts": { - "build": "tsup", "index": "tsx -r dotenv/config src/index.ts", "recommend": "tsx -r dotenv/config src/recommend.ts", + "typecheck": "tsc --noEmit", "test": "vitest run", "test:watch": "vitest", "lint": "eslint src", @@ -19,30 +15,27 @@ "format:check": "prettier --list-different \"**/*.ts\"" }, "keywords": [ - "langchain", - "starter", - "template", - "node", - "typescript", - "llm" + "pinecone", + "vector-database", + "recommender", + "similarity-search", + "embeddings", + "typescript" ], "author": "", "license": "MIT", "dependencies": { "@huggingface/transformers": "^4.2.0", "@pinecone-database/pinecone": "^8.0.0", - "chalk": "^5.2.0", "cli-progress": "^3.12.0", "console-table-printer": "^2.11.1", "dotenv": "^16.1.3", "onnxruntime-node": "^1.15.0", - "tablemark": "^3.0.0", "yargs": "^17.7.2" }, "devDependencies": { "@tsconfig/recommended": "^1.0.2", "@types/cli-progress": "^3.11.0", - "@types/js-yaml": "^4", "@types/node": "^18.15.5", "@types/yargs": "^17.0.24", "@typescript-eslint/eslint-plugin": "^5.51.0", @@ -53,7 +46,6 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.8.3", - "tsup": "^8.5.1", "tsx": "^4.23.0", "typescript": "^5.0.4", "vitest": "^3.2.7" diff --git a/tsup.config.js b/tsup.config.js deleted file mode 100644 index 13f67c2..0000000 --- a/tsup.config.js +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - entry: ["src/index.ts"], - format: ["esm"], - splitting: false, - sourcemap: true, - clean: true, - bundle: true, - dts: true, -}); From 42df6fb8438f3ff2f0e9ceb38c6c1b1b7cb1ca99 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 9 Jul 2026 11:28:13 -0400 Subject: [PATCH 4/4] Update README to match current code The README still documented the old danfojs/langchain implementation. Sync all code snippets with the current source (processInChunks over an array, dropEmptyRows, clean.length, values ?? [], the empty-result guard, and the printArticleTable helper) and drop the outdated "only aws/us-west-2" region claim. Add a Project structure map and a Development section. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 190 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 117 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index e8d593a..411b325 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,28 @@ The goal is to create a recommendation engine that retrieves the best article re npm install ``` +## Project structure + +The example is two scripts backed by a handful of small, single-purpose utilities: + +``` +src/ +├── index.ts # `npm run index`: load the dataset into Pinecone +├── recommend.ts # `npm run recommend`: query for recommendations +├── embeddings.ts # wraps the local Transformers.js embedding model +├── constants.ts # the shared Pinecone namespace +├── types.ts # the ArticleRecord shape +└── utils/ + ├── csv.ts # minimal RFC 4180 CSV parser + dropEmptyRows + ├── csvLoader.ts # reads a CSV file from disk into row objects + ├── fileSplitter.ts # splits the large dataset into smaller parts + ├── document.ts # the Document value type (pageContent + metadata) + ├── chunk.ts # sliceIntoChunks batching helper + ├── chunkedUpsert.ts # upserts vectors into Pinecone in batches + ├── env.ts # reads/validates required environment variables + └── cli.ts # parses --query / --section arguments +``` + ## Required configuration In order to run this example, you have to supply the Pinecone credentials needed to interact with the Pinecone API. You can find these credentials in the [Pinecone web console](https://app.pinecone.io) under **API Keys**. This project uses `dotenv` to easily load values from the `.env` file into the environment when executing. @@ -29,7 +51,7 @@ PINECONE_REGION="us-west-2" `PINECONE_INDEX` is the name of the index where this demo will store and query embeddings. You can change `PINECONE_INDEX` to any name you like, but make sure the name not going to collide with any indexes you are already using. -`PINECONE_CLOUD` and `PINECONE_REGION` define where the index should be deployed. Currently, this is the only available cloud and region combination (`aws` and `us-west-2`), so it's recommended to leave them defaulted. +`PINECONE_CLOUD` and `PINECONE_REGION` define where the serverless index is deployed. Use a cloud/region combination that your Pinecone project supports (for example `aws` / `us-east-1`); the values above are just a starting point. ## Data preparation @@ -54,15 +76,14 @@ Since the dataset could be pretty big, this project uses a generator function th ```typescript async function* processInChunks( - dataFrame: dfd.DataFrame, + records: T[], chunkSize: number, metadataFields: M[], pageContentField: P ): AsyncGenerator { - for (let i = 0; i < dataFrame.shape[0]; i += chunkSize) { - const chunk = await getChunk(dataFrame, i, chunkSize); - const records = dfd.toJSON(chunk) as T[]; - yield records.map((record: T) => { + for (let i = 0; i < records.length; i += chunkSize) { + const chunk = records.slice(i, i + chunkSize); + yield chunk.map((record: T) => { const metadata: Partial> = {}; for (const field of metadataFields) { metadata[field] = record[field]; @@ -80,32 +101,31 @@ For each chunk, the function generates an array of `Document` objects. The funct Here are the parameters the function accepts: -- `dataFrame`: This is the DataFrame that the function will process. +- `records`: This is the array of records that the function will process. - `chunkSize`: This is the number of records that will be processed in each chunk. - `metadataFields`: This is an array of field names (which are keys of `T`) to be included in the metadata of each `Document`. - `pageContentField`: This is the field name (which is a key of `T`) to be used for the page content of each `Document`. Here's what the function does: -1. It loops over the DataFrame in chunks of size `chunkSize`. -2. For each chunk, it converts the chunk to JSON to get an array of records (of type `T`). -3. Then, for each record in the chunk, it: +1. It loops over the records in chunks of size `chunkSize`. +2. For each record in the chunk, it: - Creates a `metadata` object that includes the specified metadata fields from the record. - Creates a new `Document` with the `pageContent` from the specified field in the record, and the `metadata` object. -4. It then yields an array of the created `Document` objects for the chunk. +3. It then yields an array of the created `Document` objects for the chunk. The `yield` keyword is used here to produce a value from the generator function. This allows the function to produce a sequence of values over time, rather than computing them all at once and returning them in a single array. Next we'll create a function that will generate the embeddings and upsert them into Pinecone. We'll use the `processInChunks` generator function to process the data in chunks. We'll also use the `chunkedUpsert` method to insert the embeddings into Pinecone in batches. ```typescript -async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { +async function embedAndUpsert(records: ArticleRecord[], chunkSize: number) { const chunkGenerator = processInChunks< ArticleRecord, "section" | "url" | "title" | "publication" | "author" | "article", "article" >( - dataFrame, + records, 100, ["section", "url", "title", "publication", "author", "article"], "article" @@ -117,7 +137,7 @@ async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { documents, chunkSize, async (embeddings: PineconeRecord[]) => { - await chunkedUpsert(index, embeddings, "default"); + await chunkedUpsert(index, embeddings, DEFAULT_NAMESPACE); progressBar.increment(embeddings.length); } ); @@ -132,13 +152,18 @@ const fileParts = await splitFile("./data/all-the-news-2-1.csv", 100000); const firstFile = fileParts[0]; ``` -Next, we'll load the data into a DataFrame using `loadCSVFile` and to simplify things, we'll also drop all rows which include a null value. +Next, we'll load the CSV into an array of row objects using `loadCSVFile`, and to simplify things, we'll also drop every row that is missing a value using `dropEmptyRows`. ```typescript const data = await loadCSVFile(firstFile); -const clean = data.dropNa() as dfd.DataFrame; +const clean = dropEmptyRows(data); ``` +> **Note:** `loadCSVFile`, `dropEmptyRows`, and the `Document` type are small +> helpers in `src/utils/`. Earlier versions of this example used `danfojs` and +> `langchain` for these; they've been inlined to keep the dependency tree small +> and the data flow easy to follow. + Now we'll create the Pinecone index and kick off the embedding and upserting process. ```typescript @@ -148,14 +173,15 @@ if (!indexList.indexes?.some((index) => index.name === indexName)) { await pinecone.createIndex({ name: indexName, dimension: 384, + metric: "cosine", spec: { serverless: { region: indexRegion, cloud: indexCloud } }, waitUntilReady: true, }); } -progressBar.start(clean.shape[0], 0); +progressBar.start(clean.length, 0); await embedder.init("Xenova/all-MiniLM-L6-v2"); -await embedAndUpsert(clean, 1); +await embedAndUpsert(clean as unknown as ArticleRecord[], 1); progressBar.stop(); ``` @@ -167,7 +193,7 @@ We will query the index for the an imagined user. We'll simulate a set of the ar const indexName = getEnv("PINECONE_INDEX"); const pinecone = new Pinecone(); -// Ensure the index exists +// Ensure the index exists and is ready before we query it. try { const description = await pinecone.describeIndex(indexName); if (!description.status?.ready) { @@ -182,16 +208,19 @@ try { throw e; } -const index = pinecone.index(indexName).namespace("default"); +const index = pinecone + .index(indexName) + .namespace(DEFAULT_NAMESPACE); await embedder.init("Xenova/all-MiniLM-L6-v2"); const { query, section } = getQueryingCommandLineArguments(); -// We create a simulated user with an interest given a query and a specific section +// Simulate the articles the user has read: the closest matches to their query +// within the section they're interested in. const queryEmbedding = await embedder.embed(query); const queryResult = await index.query({ - vector: queryEmbedding.values, + vector: queryEmbedding.values ?? [], includeMetadata: true, includeValues: true, filter: { @@ -203,26 +232,35 @@ const queryResult = await index.query({ We'll calculate the **mean** vector given the results of the query. The mean vector represents the user's interests based on the articles they've read. +If the query returned no matches (for example, a section that doesn't exist), we bail out with a friendly message instead of averaging an empty list: + ```typescript -// We extract the vectors of the results -const userVectors = queryResult?.matches?.map( - (result: ScoredPineconeRecord) => result.values -); +const readArticles = queryResult.matches ?? []; +if (readArticles.length === 0) { + console.log( + `No articles found for section "${section}" matching "${query}". ` + + "Try a different --query or --section." + ); + return; +} + +// Average the read articles' vectors into a single "taste" vector. +const userVectors = readArticles + .map((match) => match.values) + .filter((values): values is number[] => values !== undefined); +const meanVec = meanVector(userVectors); +``` -// A couple of functions to calculate mean vector -const mean = (arr: number[]): number => - arr.reduce((a, b) => a + b, 0) / arr.length; +where `meanVector` averages the vectors component by component: + +```typescript const meanVector = (vectors: number[][]): number[] => { + const mean = (values: number[]): number => + values.reduce((a, b) => a + b, 0) / values.length; const { length } = vectors[0]; - return Array.from({ length }).map((_, i) => - mean(vectors.map((vec) => vec[i])) - ); + return Array.from({ length }).map((_, i) => mean(vectors.map((v) => v[i]))); }; - -// We calculate the mean vector of the results -// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -const meanVec = meanVector(userVectors!); ``` To resolve the recommendations, we'll query the index with the mean vector and filter out the articles that the user has already read. @@ -239,46 +277,41 @@ const recommendations = await index.query({ Finally, we'll use `console-table-printer` to print out the recommendations. ```typescript -const userPreferences = new Table({ - columns: [ - { name: "title", alignment: "left" }, - { name: "author", alignment: "left" }, - { name: "section", alignment: "left" }, - ], -}); - -const userRecommendations = new Table({ - columns: [ - { name: "title", alignment: "left" }, - { name: "author", alignment: "left" }, - { name: "section", alignment: "left" }, - ], -}); - -queryResult?.matches?.slice(0, 10).forEach((result: any) => { - const { title, article, publication, section } = result.metadata; - userPreferences.addRow({ - title, - article: `${article.slice(0, 70)}...`, - publication, - section, +const printArticleTable = ( + heading: string, + matches: ScoredPineconeRecord[] +) => { + const table = new Table({ + columns: [ + { name: "title", alignment: "left" }, + { name: "article", alignment: "left" }, + { name: "section", alignment: "left" }, + { name: "publication", alignment: "left" }, + ], }); -}); -console.log("========== User Preferences =========="); -userPreferences.printTable(); + for (const match of matches) { + const { metadata } = match; + if (metadata) { + const { title, article, section, publication } = metadata; + table.addRow({ + title, + article: `${article.slice(0, 70)}...`, + section, + publication, + }); + } + } -recommendations?.matches?.slice(0, 10).forEach((result: any) => { - const { title, article, publication, section } = result.metadata; - userRecommendations.addRow({ - title, - article: `${article.slice(0, 70)}...`, - publication, - section, - }); -}); -console.log("=========== Recommendations =========="); -userRecommendations.printTable(); + console.log(heading); + table.printTable(); +}; + +printArticleTable("========== User Preferences ==========", readArticles); +printArticleTable( + "=========== Recommendations ==========", + recommendations.matches ?? [] +); ``` ### Query Sports user @@ -402,3 +435,14 @@ We can see that each user's recommendations have a high similarity to what the u Since we used only the title and the content of the article to define the embeddings, and we did not take publications and sections into account, a user may get recommendations from a publication/section that they does not regularly read. You may try adding this information when creating embeddings as well and check your query results then! Also, you may notice that some articles appear in the recommendations, although the user has already read them. These articles could be removed as part of postprocessing the query results, in case you prefer not to see them in the recommendations. + +## Development + +The scripts run directly with [`tsx`](https://github.com/privatenumber/tsx), so there's no build step. The following checks run in CI and can be run locally: + +```bash +npm run typecheck # type-check with tsc (no emit) +npm run lint # eslint +npm run format:check # prettier +npm test # vitest +```