Optionally use range to get file size

src/vfs-http-types.ts

@@ -22,13 +22,19 @@ export interface Options {
    * Force the type of backend
    */
   backendType?: 'sync' | 'shared';
+  /**
+   * Open method to determine file size. Use GET if server uses
+   * compression for HEAD responses (e.g. GitHub)
+   */
+  openMethod?: 'GET' | 'HEAD';
 }
 
 export const defaultOptions = {
   timeout: 20000,
   maxPageSize: 4096,
   cacheSize: 1024,
-  headers: {} as Record<string, string>
+  headers: {} as Record<string, string>,
+  openMethod: 'HEAD'
 };
 
 export interface BackendChannel {

src/vfs-http-worker.ts

@@ -56,17 +56,31 @@ const backendAsyncMethods:
       return 0;
 
     // Set a promise for the next opener of the same file to await upon
-    entry = fetch(msg.url, { method: 'HEAD', headers: { ...options?.headers } })
-      .then((head) => {
-        if (head.headers.get('Accept-Ranges') !== 'bytes') {
+    const method = options?.openMethod ?? VFSHTTP.defaultOptions.openMethod;
+    const headers = { ...options?.headers };
+    if (method === 'GET') {
+      headers['Range'] = 'bytes=0-1';
+    }
+    entry = fetch(msg.url, { method, headers })
+      .then((openResp) => {
+        if (openResp.headers.get('Accept-Ranges') !== 'bytes') {
           console.warn(`Server for ${msg.url} does not advertise 'Accept-Ranges'. ` +
             'If the server supports it, in order to remove this message, add "Accept-Ranges: bytes". ' +
             'Additionally, if using CORS, add "Access-Control-Expose-Headers: *".');
         }
+        const fileSize = (() => {
+          if (method === 'GET') {
+              const contentRange = openResp.headers.get('Content-Range') ?? '';
+              const rangeData = contentRange.split('/', 2);
+              return rangeData.length == 2 ? rangeData[1] : 0;
+            } else {
+              return openResp.headers.get('Content-Length') ?? 0;
+            }
+          })();
         return {
           url: msg.url,
           id: nextId++,
-          size: BigInt(head.headers.get('Content-Length') ?? 0),
+          size: BigInt(fileSize),
           // This will be determined on the first read
           pageSize: null
         };

src/vfs-sync-http.ts

@@ -287,17 +287,31 @@ export function installSyncHttpVfs(sqlite3: SQLite3, options?: VFSHTTP.Options)
       let valid = false;
       let err: Error | null = null;
       try {
+        const openMethod = options?.openMethod ?? VFSHTTP.defaultOptions.openMethod;
         const xhr = new XMLHttpRequest();
-        xhr.open('HEAD', url, false);
+        xhr.open(openMethod, url, false);
+        if (openMethod === 'GET') {
+          // use Range to get file size without compression
+          xhr.setRequestHeader('Range', 'bytes=0-1');
+        }
         for (const h of Object.keys(options?.headers ?? VFSHTTP.defaultOptions.headers))
           xhr.setRequestHeader(h, (options?.headers ?? VFSHTTP.defaultOptions.headers)[h]);
         xhr.onload = () => {
+          const fileSize = (() => {
+            if (openMethod === 'GET') {
+              const contentRange = xhr.getResponseHeader('Content-Range') ?? '';
+              const rangeData = contentRange.split('/', 2);
+              return rangeData.length == 2 ? rangeData[1] : 0;
+            } else {
+              return xhr.getResponseHeader('Content-Length') ?? 0;
+            }
+          })();
           const fh = Object.create(null) as FileDescriptor;
           fh.fid = fid;
           fh.url = url;
           fh.sq3File = new sqlite3_file(fid);
           fh.sq3File.$pMethods = httpIoMethods.pointer;
-          fh.size = BigInt(xhr.getResponseHeader('Content-Length') ?? 0);
+          fh.size = BigInt(fileSize);
           fh.pageCache = new LRUCache({
             maxSize: (options?.cacheSize ?? VFSHTTP.defaultOptions.cacheSize) * 1024,
             sizeCalculation: (value) => (value as Uint8Array).byteLength ?? 4

test/vfs-http.test.ts

@@ -195,6 +195,53 @@ for (const back of Object.keys(backTests) as (keyof typeof backTests)[]) {
         });
     });
 
+    it('should support GET open method', (done) => {
+      const githubURL = 'https://phiresky.github.io/world-development-indicators-sqlite/split-db/dbstat.sqlite3';
+
+      const backend = createHttpBackend({
+        backendType: back === 'sync' ? 'sync' : undefined,
+        openMethod: 'GET',
+      });
+      const dbGithubq = createSQLiteThread({ http: backend });
+      const rows: SQLite.ResultArray[] = [];
+      dbGithubq
+        .then((dbGithub) => dbGithub('open', {
+          filename: 'file:' + encodeURI(githubURL),
+          vfs: 'http'
+        })
+        .then(() => dbGithub))
+        .then((dbGithub) => dbGithub('exec', {
+          sql: 'select id, name from names ' +
+            'where name = $name',
+          bind: {$name: 'wdi_country'},
+          callback: (msg) => {
+            rows.push(msg);
+          }
+        }))
+        .then((msg) => {
+          assert.strictEqual(msg.type, 'exec');
+          assert.sameMembers(msg.result.columnNames, ['id', 'name']);
+          assert.lengthOf(rows, 2);
+          rows.forEach((row, idx) => {
+            assert.sameMembers(row.columnNames, ['id', 'name']);
+            if (row.row) {
+              assert.isAtMost(idx, 0);
+              assert.isNumber(row.rowNumber);
+              assert.strictEqual(row.row[0], 2);
+              assert.strictEqual(row.row[1], 'wdi_country');
+            } else {
+              assert.isNull(row.rowNumber);
+              assert.strictEqual(idx, 1);
+            }
+          });
+          done();
+        })
+        .catch(done)
+        .finally(() => {
+          dbGithubq.then((dbGithub) => dbGithub.close()).then(() => backend.close());
+        });
+    });
+
     it('should support object row mode', (done) => {
       const rows: SQLite.ResultObject[] = [];
       db('exec', {