implement a shutdown protocol

.github/workflows/node.js.yml

@@ -12,12 +12,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        node: [18.17.1, 20.x]
-        exclude:
-          - platform: windows-latest
-            node: 20.x
-          - platform: macos-latest
-            node: 20.x
+        node: [18.x, 20.x]
 
     steps:
     - uses: actions/checkout@v3
@@ -38,10 +33,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Use Node.js 18.17.1
+    - name: Use Node.js 18.x
       uses: actions/setup-node@v3
       with:
-        node-version: 18.17.1
+        node-version: 18.x
     - run: npm ci
     - run: npm run build -if-present
     - run: npm run c8

CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.2.1 2024-01-09
+
+- Fix [#65](https://github.com/mmomtchev/sqlite-wasm-http/issues/65), a minor issue preventing proper shutdown when using pools in Node.js
+
 ## 1.2.0 2023-12-12
 
 - Update the SQLite WASM distribution to an official release, 3.44.2 built to WASM with emscripten 3.1.46

src/index.ts

@@ -55,8 +55,26 @@ export function createSQLiteThread(options?: SQLiteOptions): Promise<SQLite.Prom
       }
     });
   }).then((p) => {
+    // The original SQLite WASM Promiser interface is enriched with a close
+    // method that can properly dispose of a thread
     p.close = () => {
-      worker.terminate();
+      debug['threads']('Sending destroy to SQL Worker');
+      return new Promise<void>((resolve, reject) => {
+        const tmout = setTimeout(() => {
+          // .terminate() should be used only as a last resort since there seems to
+          // be an issue in recent Node.js versions when using WASM in a worker_thread
+          worker.terminate();
+          reject('Timeout when destroying worker');
+        }, VFSHTTP.defaultOptions.timeout);
+        worker.onmessage = ({ data }) => {
+          debug['threads']('Received response from SQL Worker being destroyed', data);
+          if (data.type === 'ack') {
+            clearTimeout(tmout);
+            resolve();
+          }
+        };
+        worker.postMessage({ type: 'destroy' });
+      });
     };
     return p;
   });

src/sqlite-worker.ts

@@ -1,4 +1,8 @@
 // This is the entry point for an SQLite worker thread
+// It launches the WASM binary in each thread and installs the HTTP VFS driver
+// It also intercepts the incoming messages to implement a close protocol
+// (as this feature is missing from SQLite WASM)
+
 import { installHttpVfs } from './vfs-http.js';
 import { installSyncHttpVfs } from './vfs-sync-http.js';
 import * as VFSHTTP from './vfs-http-types.js';
@@ -19,6 +23,19 @@ globalThis.onmessage = ({ data }) => {
     .then((sqlite3) => {
       debug['threads']('SQLite init');
       sqlite3.initWorker1API();
+
+      // Install the close interceptor
+      let sqlite3Handler = globalThis.onmessage;
+      globalThis.onmessage = function (ev) {
+        if (ev.data && ev.data.type === 'destroy') {
+          debug['threads']('SQLite Worker received destroy');
+          sqlite3Handler = null;
+          globalThis.postMessage({ type: 'ack' });
+          this.setTimeout(() => globalThis.close(), 1000);
+        }
+        if (sqlite3Handler) sqlite3Handler.call(this, ev);
+      };
+
       if (typeof msg.httpChannel === 'object') {
         installHttpVfs(sqlite3, msg.httpChannel, msg.httpOptions);
       } else if (msg.httpChannel === true) {

src/vfs-http-types.ts

@@ -83,7 +83,7 @@ const debugOptions = (typeof SQLITE_DEBUG !== 'undefined' && SQLITE_DEBUG) ||
   (typeof process !== 'undefined' && typeof process?.env?.SQLITE_DEBUG !== 'undefined' && process.env.SQLITE_DEBUG) ||
   '';
 
-export const debugSys = ['threads', 'vfs', 'cache', 'http'] as const;
+export const debugSys = ['threads', 'io', 'vfs', 'cache', 'http'] as const;
 export const debug = {} as Record<typeof debugSys[number], (...args: unknown[]) => void>;
 for (const d of debugSys) {
   debug[d] = debugOptions.includes(d) ?

src/vfs-http-worker.ts

@@ -1,4 +1,4 @@
-// This is the entry point for an HTTP backend thread
+// This is the entry point the shared HTTP backend thread
 // It can serve multiple SQLite worker threads
 
 import LRUCache from 'lru-cache';
@@ -237,13 +237,13 @@ const backendAsyncMethods:
   }
 };
 
-async function workMessage(this: Consumer, { data }: { data: VFSHTTP.Message }) {
-  debug['threads']('Received new work message', this, data);
+async function workMessage(this: Consumer, { data }: { data: VFSHTTP.Message; }) {
+  debug['io']('Received new work message', this, data);
   let r;
   try {
     r = await backendAsyncMethods[data.msg](data, this);
 
-    debug['threads']('operation successful', this, r);
+    debug['io']('operation successful', this, r);
     Atomics.store(this.lock, 0, r);
   } catch (e) {
     console.error(e);

src/vfs-http.ts

@@ -1,6 +1,7 @@
 // This is the VFS layer for the shared backend
-// It run in each SQLite worker thread that uses it
-// and it is fully synchronous
+// It runs in each SQLite worker thread and exposes a synchronous interface
+// the the SQLite WASM
+// It uses vfs-http-worker of which there is a single instance
 
 import * as VFSHTTP from './vfs-http-types.js';
 import { debug } from './vfs-http-types.js';
@@ -63,7 +64,7 @@ export function installHttpVfs(
     if (r === 'timed-out') {
       console.error('Backend timeout', r, lock, msg);
       return -1;
-    } 
+    }
     return rc;
   };
 

src/vfs-sync-http.ts

@@ -1,6 +1,6 @@
 // This is the ersatz HTTP backend
 // It does not require SharedArrayBuffer and does not share its cache
-// It runs in the SQLite worker thread
+// It runs in each SQLite worker thread
 
 import LRUCache from 'lru-cache';
 import { ntoh16 } from './endianness.js';

test/http-pool.test.ts

@@ -75,15 +75,11 @@ for (const type of ['sync', 'shared'] as const) {
             done('beyond the event horizon');
           })
           .catch((e) => {
-            pool.close();
             assert.include(e.result.message, 'sqlite3 result code 14: unable to open database file');
-            done();
+            pool.close().then(done);
           })
-          .catch((e) => {
-            // Obviously a major malfunction, the above code should never throw
-            done(e);
-          })
-        );
+        )
+        .catch(done);
     });
   });
 }

test/vfs-http.test.ts

@@ -296,10 +296,10 @@ for (const back of Object.keys(backTests) as (keyof typeof backTests)[]) {
           .then(() => {
             assert.strictEqual(tiles, concurrentDb.length);
           })
-          .finally(() => {
-            Promise.all(concurrentDb.map((dbq) => dbq.then((db) => db.close())));
-            done();
-          })
+          .finally(() =>
+            Promise.all(concurrentDb.map((dbq) => dbq.then((db) => db.close())))
+          )
+          .then(done)
           .catch(done);
       });
     }