fix: Migrate from better-sqlite3 to node-sqlite3-wasm for improved compatibility

dividedmind · dividedmind · commit 4aaa5400e055 · 2025-02-17T16:53:29.000+01:00
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -103,7 +103,6 @@
     "applicationinsights": "^2.1.4",
     "async": "^3.2.4",
     "axios": "^0.27.2",
-    "better-sqlite3": "^11.5.0",
     "body-parser": "^1.20.2",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
@@ -131,6 +130,7 @@
     "lunr": "^2.3.9",
     "minimatch": "^5.1.2",
     "moo": "^0.5.1",
+    "node-sqlite3-wasm": "^0.8.34",
     "open": "^8.2.1",
     "openapi-diff": "^0.23.6",
     "openapi-types": "^12.1.3",
diff --git a/packages/cli/src/cmds/search/search.ts b/packages/cli/src/cmds/search/search.ts
@@ -1,5 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
 import yargs from 'yargs';
-import sqlite3 from 'better-sqlite3';
 import assert from 'assert';
 import { readFileSync } from 'fs';
 import { writeFile } from 'fs/promises';
@@ -186,7 +186,7 @@ export const handler = async (argv: ArgumentTypes) => {
     };
 
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(fileIndex, [process.cwd()]);
       return fileIndex;
diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
diff --git a/packages/cli/src/rpc/explain/index-snippets.ts b/packages/cli/src/rpc/explain/index-snippets.ts
@@ -1,3 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
+
 import {
   buildSnippetIndex,
   FileSearchResult,
@@ -6,7 +8,6 @@ import {
   readFileSafe,
   SnippetIndex,
 } from '@appland/search';
-import sqlite3 from 'better-sqlite3';
 
 export default async function indexSnippets(
   db: sqlite3.Database,
diff --git a/packages/cli/src/rpc/explain/index/appmap-file-index.ts b/packages/cli/src/rpc/explain/index/appmap-file-index.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { FileIndex, SessionId } from '@appland/search';
 
@@ -10,7 +10,7 @@ export async function buildAppMapFileIndex(
   appmapDirectories: string[]
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir<FileIndex>('appmaps', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     const fileIndex = new FileIndex(db);
     await buildAppMapIndex(fileIndex, appmapDirectories);
     return fileIndex;
diff --git a/packages/cli/src/rpc/explain/index/project-file-index.ts b/packages/cli/src/rpc/explain/index/project-file-index.ts
@@ -1,5 +1,5 @@
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
@@ -71,7 +71,7 @@ export async function buildProjectFileIndex(
   excludePatterns: RegExp[] | undefined
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir('files', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexFiles(db, sourceDirectories, includePatterns, excludePatterns);
   });
 }
diff --git a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileSearchResult, SessionId, SnippetIndex } from '@appland/search';
 
 import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir';
@@ -76,7 +77,7 @@ export async function buildProjectFileSnippetIndex(
   fileSearchResults: FileSearchResult[]
 ): Promise<ProjectFileSnippetIndex> {
   const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexSnippets(db, fileSearchResults);
   });
 
diff --git a/packages/cli/src/rpc/search/search.ts b/packages/cli/src/rpc/search/search.ts
@@ -1,5 +1,7 @@
 import { isAbsolute, join } from 'path';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileIndex, generateSessionId } from '@appland/search';
 import { SearchRpc } from '@appland/rpc';
 
@@ -62,7 +64,7 @@ export async function handler(
     // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
     const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS;
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(
         fileIndex,
diff --git a/packages/search/package.json b/packages/search/package.json
@@ -39,8 +39,8 @@
     "typescript-eslint": "^8.11.0"
   },
   "dependencies": {
-    "better-sqlite3": "^11.5.0",
     "isbinaryfile": "^5.0.4",
+    "node-sqlite3-wasm": "^0.8.34",
     "yargs": "^17.7.2"
   }
 }
diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts
@@ -1,7 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { fileTokens } from './tokenize';
 import FileIndex from './file-index';
@@ -62,7 +62,7 @@ const cli = yargs(hideBin(process.argv))
         return !filterRE.test(path);
       };
 
-      const db = new sqlite3(':memory:');
+      const db = new sqlite3.Database(':memory:');
       const fileIndex = new FileIndex(db);
       const sessionId = generateSessionId();
 
diff --git a/packages/search/src/file-index.ts b/packages/search/src/file-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_TABLE_SQL = `CREATE VIRTUAL TABLE file_content USING fts5(
@@ -74,22 +75,22 @@ export type FileSearchResult = {
 export default class FileIndex {
   #insert: sqlite3.Statement;
   #updateBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<string>;
-  #search: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #search: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_TABLE_SQL);
     this.database.exec(CREATE_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insert = this.database.prepare(INSERT_SQL);
     this.#updateBoost = this.database.prepare(UPDATE_BOOST_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
     this.#search = this.database.prepare(SEARCH_SQL);
   }
 
   indexFile(directory: string, filePath: string, symbols: string, words: string): void {
-    this.#insert.run(directory, filePath, symbols, words);
+    this.#insert.run([directory, filePath, symbols, words]);
   }
 
   /**
@@ -99,7 +100,7 @@ export default class FileIndex {
    * @param boostFactor - The factor by which to boost the file's relevance.
    */
   boostFile(sessionId: SessionId, filePath: string, boostFactor: number): void {
-    this.#updateBoost.run(sessionId, filePath, boostFactor);
+    this.#updateBoost.run([sessionId, filePath, boostFactor]);
   }
 
   /**
@@ -118,7 +119,7 @@ export default class FileIndex {
    * @returns An array of search results with directory, file path, and score.
    */
   search(sessionId: SessionId, query: string, limit = 10): FileSearchResult[] {
-    const rows = this.#search.all(sessionId, query, limit) as FileIndexRow[];
+    const rows = this.#search.all([sessionId, query, limit]) as FileIndexRow[];
     return rows.map((row) => ({
       directory: row.directory,
       filePath: row.file_path,
diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts
@@ -1,5 +1,7 @@
 import assert from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5(
@@ -115,14 +117,14 @@ type SnippetSearchRow = {
 export default class SnippetIndex {
   #insertSnippet: sqlite3.Statement;
   #updateSnippetBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<[string]>;
-  #searchSnippet: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #searchSnippet: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
     this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insertSnippet = this.database.prepare(INSERT_SNIPPET_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
     this.#updateSnippetBoost = this.database.prepare(UPDATE_SNIPPET_BOOST_SQL);
@@ -152,7 +154,7 @@ export default class SnippetIndex {
     words: string,
     content: string
   ): void {
-    this.#insertSnippet.run(encodeSnippetId(snippetId), directory, symbols, words, content);
+    this.#insertSnippet.run([encodeSnippetId(snippetId), directory, symbols, words, content]);
   }
 
   /**
@@ -162,11 +164,11 @@ export default class SnippetIndex {
    * @param boostFactor - The factor by which to boost the snippet's relevance.
    */
   boostSnippet(sessionId: SessionId, snippetId: SnippetId, boostFactor: number): void {
-    this.#updateSnippetBoost.run(sessionId, encodeSnippetId(snippetId), boostFactor);
+    this.#updateSnippetBoost.run([sessionId, encodeSnippetId(snippetId), boostFactor]);
   }
 
   searchSnippets(sessionId: SessionId, query: string, limit = 10): SnippetSearchResult[] {
-    const rows = this.#searchSnippet.all(sessionId, query, limit) as SnippetSearchRow[];
+    const rows = this.#searchSnippet.all([sessionId, query, limit]) as SnippetSearchRow[];
     return rows.map((row) => ({
       directory: row.directory,
       snippetId: parseSnippetId(row.snippet_id),
diff --git a/yarn.lock b/yarn.lock
@@ -182,7 +182,6 @@ __metadata:
     applicationinsights: ^2.1.4
     async: ^3.2.4
     axios: ^0.27.2
-    better-sqlite3: ^11.5.0
     body-parser: ^1.20.2
     boxen: ^5.0.1
     chalk: ^4.1.2
@@ -221,6 +220,7 @@ __metadata:
     minimatch: ^5.1.2
     moo: ^0.5.1
     node-fetch: 2.6.7
+    node-sqlite3-wasm: ^0.8.34
     open: ^8.2.1
     openapi-diff: ^0.23.6
     openapi-types: ^12.1.3
@@ -595,7 +595,6 @@ __metadata:
     "@types/better-sqlite3": ^7.6.11
     "@types/jest": ^29.5.4
     "@types/node": ^16
-    better-sqlite3: ^11.5.0
     eslint: ^9
     eslint-config-prettier: ^9
     eslint-plugin-eslint-comments: ^3.2.0
@@ -605,6 +604,7 @@ __metadata:
     eslint-plugin-promise: ^7.1.0
     isbinaryfile: ^5.0.4
     jest: ^29.7.0
+    node-sqlite3-wasm: ^0.8.34
     prettier: ^3.3.3
     ts-jest: ^29.2.5
     tsc: ^2.0.4
@@ -15901,17 +15901,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"better-sqlite3@npm:^11.5.0":
-  version: 11.5.0
-  resolution: "better-sqlite3@npm:11.5.0"
-  dependencies:
-    bindings: ^1.5.0
-    node-gyp: latest
-    prebuild-install: ^7.1.1
-  checksum: 37acef8d4272ad57fe211aa5a2e177f95443eafb794c7db6c2d0dae0a4fe4c2ef508b8b970d82f1d9744d009677d82067279f45e27d4155a5e68a578f5c5c051
-  languageName: node
-  linkType: hard
-
 "bfj@npm:^6.1.1":
   version: 6.1.2
   resolution: "bfj@npm:6.1.2"
@@ -32664,6 +32653,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"node-sqlite3-wasm@npm:^0.8.34":
+  version: 0.8.34
+  resolution: "node-sqlite3-wasm@npm:0.8.34"
+  checksum: c6c9b2c75d714badaa41ac66a9ed6a30a718252d9259268bb96d899308aa9024c8766a0995c10e5225e7c57cc8c1483ef9279c3b6e97ddd45b6dba3af3e6e551
+  languageName: node
+  linkType: hard
+
 "non-layered-tidy-tree-layout@npm:^2.0.2":
   version: 2.0.2
   resolution: "non-layered-tidy-tree-layout@npm:2.0.2"
@@ -35376,28 +35372,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prebuild-install@npm:^7.1.1":
-  version: 7.1.2
-  resolution: "prebuild-install@npm:7.1.2"
-  dependencies:
-    detect-libc: ^2.0.0
-    expand-template: ^2.0.3
-    github-from-package: 0.0.0
-    minimist: ^1.2.3
-    mkdirp-classic: ^0.5.3
-    napi-build-utils: ^1.0.1
-    node-abi: ^3.3.0
-    pump: ^3.0.0
-    rc: ^1.2.7
-    simple-get: ^4.0.0
-    tar-fs: ^2.0.0
-    tunnel-agent: ^0.6.0
-  bin:
-    prebuild-install: bin.js
-  checksum: 543dadf8c60e004ae9529e6013ca0cbeac8ef38b5f5ba5518cb0b622fe7f8758b34e4b5cb1a791db3cdc9d2281766302df6088bd1a225f206925d6fee17d6c5c
-  languageName: node
-  linkType: hard
-
 "prelude-ls@npm:^1.2.1":
   version: 1.2.1
   resolution: "prelude-ls@npm:1.2.1"

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import sqlite3 from 'better-sqlite3';`
	`1`	`+import type sqlite3 from 'node-sqlite3-wasm';`
`2`	`2`
`3`	`3`	`import {`
`4`	`4`	`buildFileIndex,`
Original file line number	Diff line number	Diff line change
`@@ -39,8 +39,8 @@`
`39`	`39`	`"typescript-eslint": "^8.11.0"`
`40`	`40`	`},`
`41`	`41`	`"dependencies": {`
`42`		`- "better-sqlite3": "^11.5.0",`
`43`	`42`	`"isbinaryfile": "^5.0.4",`
	`43`	`+ "node-sqlite3-wasm": "^0.8.34",`
`44`	`44`	`"yargs": "^17.7.2"`
`45`	`45`	`}`
`46`	`46`	`}`