From 606e87d1dd92f92b2d02581115b1d6c2e715878d Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:31:01 +0100 Subject: [PATCH 01/11] Update drive-gateway.ts --- src/drive/drive-gateway.ts | 131 +++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 64 deletions(-) diff --git a/src/drive/drive-gateway.ts b/src/drive/drive-gateway.ts index 075346e5..dded5e71 100644 --- a/src/drive/drive-gateway.ts +++ b/src/drive/drive-gateway.ts @@ -65,30 +65,31 @@ export class GDriveGateway implements bs.Gateway { } async put(url: URI, body: Uint8Array): Promise { - const { pathPart } = getStore(url, this.sthis, (...args) => args.join("/")); - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("auth", "name", "store"); if (rParams.isErr()) { return this.logger.Error().Url(url).Err(rParams).Msg("Put Error").ResultError(); } - const { auth } = rParams.Ok(); + const { auth, store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); - if (!index) { + if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); - if (fileId.Err()) { + const fileId = await this.#search(name, auth, store); + if (fileId.isErr()) { if (isNotFoundError(fileId.Err())) { - const done = await this.#insert(name, body, auth, pathPart); + const fileData = new Blob([body], { type: "application/octet-stream" }); + + const done = await this.#insert(name, body, auth, store); if (done.isErr()) { return done; } return Result.Ok(undefined); } } - const fileMetadata = await this.#get(fileId.Ok(), pathPart, auth); - const fileData = new Blob([body], { type: "application/json" }); - const done = await this.#update(fileId.Ok(), fileMetadata, fileData, auth, pathPart); + const fileData = new Blob([body], { type: "application/octet-stream" }); + + const done = await this.#update(fileId.Ok(), fileData, auth, store); if (done.isErr()) { return done; } @@ -101,9 +102,9 @@ export class GDriveGateway implements bs.Gateway { Authorization: `Bearer ${auth}`, }; try { - const response = await fetch(url + fileId, { + const response = await fetch(BuildURI.from(url).appendRelative('drive/v3/files') + fileId, { method: "DELETE", - headers: headers, + headers }); return await response.json(); } catch (err) { @@ -111,7 +112,7 @@ export class GDriveGateway implements bs.Gateway { } } - async #get(fileId: string, type: "data" | "wal" | "meta", auth: string): Promise> { + async #get(fileId: string, auth: string): Promise> { let response; let headers; const url = BuildURI.from(this.params.driveURL); @@ -119,57 +120,50 @@ export class GDriveGateway implements bs.Gateway { Authorization: `Bearer ${auth}`, "Content-Type": "application/json", }; - if (type == "meta") { - response = await fetch(url + fileId, { - method: "GET", - headers, - }); - return Result.Ok(new Uint8Array(await response.arrayBuffer())); - } else { - headers = { - Authorization: `Bearer ${auth}`, - }; - response = await fetch(url.appendRelative(fileId).setParam("alt", "media").toString(), { - method: "GET", - headers, - }); - return Result.Ok(new Uint8Array(await response.arrayBuffer())); - } + + headers = { + Authorization: `Bearer ${auth}`, + }; + response = await fetch(url.appendRelative(`drive/v3/files/${fileId}`).setParam("alt", "media").toString(), { + method: "GET", + headers, + }); + return Result.Ok(new Uint8Array(await response.arrayBuffer())); + } async get(url: URI): Promise { - const { pathPart } = getStore(url, this.sthis, (...args) => args.join("/")); - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("auth", "name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } let { name } = rParams.Ok(); - const { auth } = rParams.Ok(); + const { auth, store } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); - if (fileId.Err()) { + const fileId = await this.#search(name, auth, store); + if (fileId.isErr()) { return Result.Err(fileId.Err()); } - const response = await this.#get(fileId.Ok(), pathPart, auth); + const response = await this.#get(fileId.Ok(), auth); return response; } async delete(url: URI): Promise { - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("auth", "name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } - const { auth } = rParams.Ok(); + const { auth, store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); + const fileId = await this.#search(name, auth, store); if (fileId.isErr()) { return fileId; } @@ -212,35 +206,22 @@ export class GDriveGateway implements bs.Gateway { async #update( fileId: string, - fileMetadata: object, fileData: Blob, auth: string, - store: "data" | "wal" | "meta" + store: string ): Promise> { const url = BuildURI.from(this.params.driveURL); - const boundary = "-------" + this.sthis.nextId(16).str; + const headers = { Authorization: `Bearer ${auth}`, - "Content-Type": `multipart/related; boundary="${boundary}"`, + "Content-Type": `fireproof/${store}`, }; const response = await fetch( - url.appendRelative(fileId).setParam("uploadType", "multipart").setParam("fields", "id").toString(), + url.appendRelative(`upload/drive/v3/files/${fileId}`).setParam("uploadType", "media").toString(), { method: "PATCH", headers, - body: new ReadableStream({ - start: (controller) => { - controller.enqueue(this.sthis.txt.encode(`--${boundary}\r\n`)); - controller.enqueue(this.sthis.txt.encode("Content-Type: application/json\r\n\r\n")); - controller.enqueue(this.sthis.txt.encode(JSON.stringify(fileMetadata) + "\r\n")); - controller.enqueue(this.sthis.txt.encode(`--${boundary}\r\n`)); - controller.enqueue(this.sthis.txt.encode(`Content-Type: fireproof/${store}\r\n\r\n`)); - controller.enqueue(fileData.arrayBuffer()); - controller.enqueue(this.sthis.txt.encode("\r\n")); - controller.enqueue(this.sthis.txt.encode(`--${boundary}--\r\n`)); - controller.close(); - }, - }), + body: fileData } ); if (!response.ok) { @@ -261,7 +242,7 @@ export class GDriveGateway implements bs.Gateway { form.append("file", file); try { const response = await fetch( - url.setParam("uploadType", "multipart").setParam("supportsAllDrives", "true").toString(), + url.appendRelative('upload/drive/v3/files').setParam("uploadType", "multipart").setParam("supportsAllDrives", "true").toString(), { method: "POST", headers: { Authorization: "Bearer " + auth }, @@ -274,10 +255,10 @@ export class GDriveGateway implements bs.Gateway { return this.logger.Error().Any({ auth, store }).Err(err).Msg("Insert Error").ResultError(); } } - async #search(fileName: string, auth: string): Promise> { + async #search(fileName: string, auth: string, store: string): Promise> { try { const response = await fetch( - BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=name", `"${fileName}"`).toString(), + BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=mimeType", `"fireproof/${store}" and name="${fileName}"`).toString(), { headers: { Authorization: "Bearer " + auth, @@ -305,19 +286,41 @@ export class GDriveGateway implements bs.Gateway { // return num.toString(); // } +export class GDriveTestStore implements bs.TestGateway { + readonly logger: Logger; + readonly sthis: SuperThis; + readonly gateway: bs.Gateway; + + constructor(sthis: SuperThis, gw: bs.Gateway) { + this.sthis = ensureSuperLog(sthis, "GDriveTestStore"); + this.logger = this.sthis.logger; + this.gateway = gw; + } + + async get(iurl: URI, key: string): Promise { + const url = iurl.build().setParam("key", key).URI(); + const buffer = await this.gateway.get(url); + return buffer.Ok(); + } +} + const onceregisterGDriveStoreProtocol = new KeyedResolvOnce<() => void>(); -export function registerGDriveStoreProtocol(protocol = "gdrive:") { +export function registerGDriveStoreProtocol(protocol = "gdrive:", overrideBaseURL?: string) { return onceregisterGDriveStoreProtocol.get(protocol).once(() => { URI.protocolHasHostpart(protocol); return bs.registerStoreProtocol({ protocol, - defaultURI: (): URI => { - return URI.from("gdrive://"); - }, + overrideBaseURL, gateway: async (sthis): Promise => { return new GDriveGateway(sthis, { - driveURL: "https://www.googleapis.com/upload/drive/v3/files/", + driveURL: "https://www.googleapis.com/", + }); + }, + test: async (sthis: SuperThis) => { + const gateway = new GDriveGateway(sthis, { + driveURL: "https://www.googleapis.com/", }); + return new GDriveTestStore(sthis, gateway); }, }); }); From e6c30f013caf9735b3d1c8327fb5330ff7b6116d Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:31:25 +0100 Subject: [PATCH 02/11] Update drive-gateway.test.ts --- src/drive/drive-gateway.test.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/drive/drive-gateway.test.ts b/src/drive/drive-gateway.test.ts index ff4e4cb5..5f9eb273 100644 --- a/src/drive/drive-gateway.test.ts +++ b/src/drive/drive-gateway.test.ts @@ -1,20 +1,12 @@ import { fireproof } from "@fireproof/core"; -import { registerGDriveStoreProtocol } from "./drive-gateway.js"; +import { connect } from "./index.js"; import { describe, it } from "vitest"; import { smokeDB } from "../../tests/helper.js"; describe("store-register", () => { it("should store and retrieve data", async () => { - const unreg = registerGDriveStoreProtocol("gdrive:"); - const db = fireproof("my-database", { - store: { - stores: { - base: "gdrive://www.googleapis.com/?auth=testtoken", - }, - }, - }); + const db = fireproof("my-database"); + connect(db, "ya29.a0AeXRPp4bASF7IqO0gx7MYSjgV8zMixuMR3z4rWvuqYvNUKv9pDuPumJIvmaXzmNE7k-7xArIdN9-VRippEG5ByIoqinkRlyNxunGmLq9oGZvgAiQ6_zUxHBHm1XL2Nx2lBjILtZZkJlFRNc_02wE5GZJ2VkLQar_rS-BKOqPaCgYKAfcSAQ8SFQHGX2Mia-YkdQYWm7KsWLbfF2ZwOQ0175") await smokeDB(db); - await db.destroy(); - unreg(); }); }); From 2cb94f9d932c658a5d11adedf6070d818357f633 Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:32:08 +0100 Subject: [PATCH 03/11] Update helper.ts --- tests/helper.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helper.ts b/tests/helper.ts index c8100280..e31cbf12 100644 --- a/tests/helper.ts +++ b/tests/helper.ts @@ -1,4 +1,5 @@ import { Database } from "@fireproof/core"; +import {expect} from "vitest" export async function smokeDB(db: Database) { const ran = db.sthis.nextId().str; From 08320c2741ebbc4075192b98c8c0098740ba834c Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:33:03 +0100 Subject: [PATCH 04/11] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c497bc0d..5027e767 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "@aws-sdk/s3-request-presigner": "^3.750.0", "@cloudflare/vitest-pool-workers": "^0.7.4", "@cloudflare/workers-types": "^4.20250224.0", - "@fireproof/core": "0.20.0-dev-preview-50", + "@fireproof/core": "0.19.122", "@fireproof/vendor": "~2.0.0", "@ipld/dag-ucan": "^3.4.5", "@jspm/core": "^2.1.0", From abf231bd62e38719897c88c8645e988c7d91a1bb Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:33:41 +0100 Subject: [PATCH 05/11] Update connection-from-store.ts --- src/connection-from-store.ts | 118 ++++++++++++++++------------------- 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/src/connection-from-store.ts b/src/connection-from-store.ts index 89a9d50e..bdf6bba7 100644 --- a/src/connection-from-store.ts +++ b/src/connection-from-store.ts @@ -1,5 +1,5 @@ -import { BuildURI, runtimeFn } from "@adviser/cement"; -import { bs, Database, SuperThis } from "@fireproof/core"; +import { BuildURI, CoerceURI, runtimeFn, URI } from "@adviser/cement"; +import { bs, Database, ensureLogger, SuperThis } from "@fireproof/core"; // export interface StoreOptions { // readonly data: bs.DataStore; @@ -7,71 +7,63 @@ import { bs, Database, SuperThis } from "@fireproof/core"; // readonly wal: bs.WALState; // } -// export class ConnectionFromStore extends bs.ConnectionBase { -// stores?: { -// readonly data: bs.DataStore; -// readonly meta: bs.MetaStore; -// } = undefined; +export class ConnectionFromStore extends bs.ConnectionBase { + stores?: { + readonly data: bs.DataStore; + readonly meta: bs.MetaStore; + } = undefined; -// // readonly urlData: URI; -// // readonly urlMeta: URI; + // readonly urlData: URI; + // readonly urlMeta: URI; -// readonly sthis: SuperThis; -// constructor(sthis: SuperThis, url: URI) { -// const logger = ensureLogger(sthis, "ConnectionFromStore", { -// url: () => url.toString(), -// this: 1, -// log: 1, -// }); -// super(url, logger); -// this.sthis = sthis; -// // this.urlData = url; -// // this.urlMeta = url; -// } -// async onConnect(): Promise { -// this.logger.Debug().Msg("onConnect-start"); -// // const stores = { -// // base: this.url, -// // // data: this.urlData, -// // // meta: this.urlMeta, -// // }; -// const rName = this.url.getParamResult("name"); -// if (rName.isErr()) { -// throw this.logger.Error().Err(rName).Msg("missing Parameter").AsError(); -// } -// const storeRuntime = bs.toStoreRuntime(this.sthis); -// const loader: bs.StoreFactoryItem = { -// url: this.url, -// loader: { -// ebOpts: { -// logger: this.logger, -// storeUrls: { -// data: this.url, -// meta: this.url, -// file: this.url, -// wal: this.url, -// }, -// // store: { stores }, -// storeRuntime, -// } as bs.Loadable["ebOpts"], -// sthis: this.sthis, -// } as bs.Loadable, -// }; + readonly sthis: SuperThis; + constructor(sthis: SuperThis, url: URI) { + const logger = ensureLogger(sthis, "ConnectionFromStore", { + url: () => url.toString(), + this: 1, + log: 1, + }); + super(url, logger); + this.sthis = sthis; + // this.urlData = url; + // this.urlMeta = url; + } + async onConnect(): Promise { + this.logger.Debug().Msg("onConnect-start"); + const stores = { + base: this.url, + // data: this.urlData, + // meta: this.urlMeta, + }; + const rName = this.url.getParamResult("name"); + if (rName.isErr()) { + throw this.logger.Error().Err(rName).Msg("missing Parameter").AsError(); + } + const storeRuntime = bs.toStoreRuntime({ stores }, this.sthis); + const loader = { + name: rName.Ok(), + ebOpts: { + logger: this.logger, + store: { stores }, + storeRuntime, + }, + sthis: this.sthis, + } as bs.Loadable; -// this.stores = { -// data: await storeRuntime.makeDataStore(loader), -// meta: await storeRuntime.makeMetaStore(loader), -// }; -// // await this.stores.data.start(); -// // await this.stores.meta.start(); -// this.logger.Debug().Msg("onConnect-done"); -// return; -// } -// } + this.stores = { + data: await storeRuntime.makeDataStore(loader), + meta: await storeRuntime.makeMetaStore(loader), + }; + // await this.stores.data.start(); + // await this.stores.meta.start(); + this.logger.Debug().Msg("onConnect-done"); + return; + } +} -// export function connectionFactory(sthis: SuperThis, iurl: CoerceURI): bs.ConnectionBase { -// return new ConnectionFromStore(sthis, URI.from(iurl)); -// } +export function connectionFactory(sthis: SuperThis, iurl: CoerceURI): bs.ConnectionBase { + return new ConnectionFromStore(sthis, URI.from(iurl)); +} export function makeKeyBagUrlExtractable(sthis: SuperThis) { let base = sthis.env.get("FP_KEYBAG_URL"); From 76c8a3f00c80f51ae0c1445f81816362298f58ac Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:03:37 +0100 Subject: [PATCH 06/11] Update package.json --- package.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package.json b/package.json index 5027e767..84b62f34 100644 --- a/package.json +++ b/package.json @@ -135,6 +135,17 @@ "webdriverio": "^9.10.1", "ws": "^8.18.0" }, + "exports":{ + "./ucan": { + "browser": "./dist/ucan/web/index.js", + "node": "./dist/ucan/index.js", + "require": "./dist/ucan/index.cjs", + "script": "./dist/ucan/web/index.global.js", + "types": "./dist/ucan/index.d.ts" + }, + "./drive": "./dist/tsc/src/drive/index.js", + "./drive-gateway": "./dist/tsc/src/drive/drive-gateway.js" + }, "pnpm": { "onlyBuiltDependencies": [ "aws-sdk", From b7b1bf4f03d582fd0128d19fe76f64d08cf5ffab Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:48:18 +0100 Subject: [PATCH 07/11] Update drive-gateway.ts --- src/drive/drive-gateway.ts | 90 +++++++++++++++----------------------- 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/src/drive/drive-gateway.ts b/src/drive/drive-gateway.ts index dded5e71..ba23cb37 100644 --- a/src/drive/drive-gateway.ts +++ b/src/drive/drive-gateway.ts @@ -2,6 +2,7 @@ import { BuildURI, KeyedResolvOnce, Logger, Result, URI } from "@adviser/cement" import { bs, getStore, SuperThis, ensureSuperLog, NotFoundError, isNotFoundError } from "@fireproof/core"; interface GDriveGatewayParams { + readonly auth: string; readonly driveURL: string; } @@ -65,22 +66,22 @@ export class GDriveGateway implements bs.Gateway { } async put(url: URI, body: Uint8Array): Promise { - const rParams = url.getParamsResult("auth", "name", "store"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return this.logger.Error().Url(url).Err(rParams).Msg("Put Error").ResultError(); } - const { auth, store } = rParams.Ok(); + const { store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth, store); + const fileId = await this.#search(name, store); if (fileId.isErr()) { if (isNotFoundError(fileId.Err())) { const fileData = new Blob([body], { type: "application/octet-stream" }); - const done = await this.#insert(name, body, auth, store); + const done = await this.#insert(name, body, store); if (done.isErr()) { return done; } @@ -89,17 +90,17 @@ export class GDriveGateway implements bs.Gateway { } const fileData = new Blob([body], { type: "application/octet-stream" }); - const done = await this.#update(fileId.Ok(), fileData, auth, store); + const done = await this.#update(fileId.Ok(), fileData, store); if (done.isErr()) { return done; } return Result.Ok(undefined); } - async #delete(fileId: string, auth: string): Promise> { + async #delete(fileId: string): Promise> { const url = this.params.driveURL; const headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, }; try { const response = await fetch(BuildURI.from(url).appendRelative('drive/v3/files') + fileId, { @@ -108,21 +109,21 @@ export class GDriveGateway implements bs.Gateway { }); return await response.json(); } catch (err) { - return this.logger.Error().Url(url).Any("init", auth).Err(err).Msg("Could not delete").ResultError(); + return this.logger.Error().Url(url).Any("init", this.params.auth).Err(err).Msg("Could not delete").ResultError(); } } - async #get(fileId: string, auth: string): Promise> { + async #get(fileId: string): Promise> { let response; let headers; const url = BuildURI.from(this.params.driveURL); headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, "Content-Type": "application/json", }; headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, }; response = await fetch(url.appendRelative(`drive/v3/files/${fileId}`).setParam("alt", "media").toString(), { method: "GET", @@ -133,41 +134,41 @@ export class GDriveGateway implements bs.Gateway { } async get(url: URI): Promise { - const rParams = url.getParamsResult("auth", "name", "store"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } let { name } = rParams.Ok(); - const { auth, store } = rParams.Ok(); + const { store } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth, store); + const fileId = await this.#search(name, store); if (fileId.isErr()) { return Result.Err(fileId.Err()); } - const response = await this.#get(fileId.Ok(), auth); + const response = await this.#get(fileId.Ok()); return response; } async delete(url: URI): Promise { - const rParams = url.getParamsResult("auth", "name", "store"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } - const { auth, store } = rParams.Ok(); + const { store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth, store); + const fileId = await this.#search(name, store); if (fileId.isErr()) { return fileId; } - return await this.#delete(fileId.Ok(), auth); + return await this.#delete(fileId.Ok()); } async subscribe(url: URI, callback: (msg: Uint8Array) => void): Promise { @@ -207,13 +208,12 @@ export class GDriveGateway implements bs.Gateway { async #update( fileId: string, fileData: Blob, - auth: string, store: string ): Promise> { const url = BuildURI.from(this.params.driveURL); const headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, "Content-Type": `fireproof/${store}`, }; const response = await fetch( @@ -225,11 +225,11 @@ export class GDriveGateway implements bs.Gateway { } ); if (!response.ok) { - return this.logger.Error().Any({ auth, store }).Msg("Insert Error").ResultError(); + return this.logger.Error().Any({ store }).Msg("Insert Error").ResultError(); } return Result.Ok(fileId); } - async #insert(fileName: string, content: Uint8Array, auth: string, store: string): Promise> { + async #insert(fileName: string, content: Uint8Array, store: string): Promise> { const url = BuildURI.from(this.params.driveURL); const mime = `fireproof/${store}`; const file = new Blob([content], { type: mime }); @@ -245,23 +245,23 @@ export class GDriveGateway implements bs.Gateway { url.appendRelative('upload/drive/v3/files').setParam("uploadType", "multipart").setParam("supportsAllDrives", "true").toString(), { method: "POST", - headers: { Authorization: "Bearer " + auth }, + headers: { Authorization: "Bearer " + this.params.auth }, body: form, } ); const jsonRes = (await response.json()) as { id: string }; return Result.Ok(jsonRes.id); } catch (err) { - return this.logger.Error().Any({ auth, store }).Err(err).Msg("Insert Error").ResultError(); + return this.logger.Error().Any({ store }).Err(err).Msg("Insert Error").ResultError(); } } - async #search(fileName: string, auth: string, store: string): Promise> { + async #search(fileName: string, store: string): Promise> { try { const response = await fetch( BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=mimeType", `"fireproof/${store}" and name="${fileName}"`).toString(), { headers: { - Authorization: "Bearer " + auth, + Authorization: "Bearer " + this.params.auth, }, } ); @@ -274,7 +274,7 @@ export class GDriveGateway implements bs.Gateway { } return Result.Err(new NotFoundError("File not found")); } catch (err) { - return this.logger.Error().Any({ auth, fileName }).Err(err).Msg("Fetch Error").ResultError(); + return this.logger.Error().Any({ fileName }).Err(err).Msg("Fetch Error").ResultError(); } } } @@ -286,42 +286,24 @@ export class GDriveGateway implements bs.Gateway { // return num.toString(); // } -export class GDriveTestStore implements bs.TestGateway { - readonly logger: Logger; - readonly sthis: SuperThis; - readonly gateway: bs.Gateway; - - constructor(sthis: SuperThis, gw: bs.Gateway) { - this.sthis = ensureSuperLog(sthis, "GDriveTestStore"); - this.logger = this.sthis.logger; - this.gateway = gw; - } - async get(iurl: URI, key: string): Promise { - const url = iurl.build().setParam("key", key).URI(); - const buffer = await this.gateway.get(url); - return buffer.Ok(); - } -} const onceregisterGDriveStoreProtocol = new KeyedResolvOnce<() => void>(); -export function registerGDriveStoreProtocol(protocol = "gdrive:", overrideBaseURL?: string) { +export function registerGDriveStoreProtocol(protocol = "gdrive:", auth: string) { return onceregisterGDriveStoreProtocol.get(protocol).once(() => { URI.protocolHasHostpart(protocol); return bs.registerStoreProtocol({ - protocol, - overrideBaseURL, + protocol: protocol, + isDefault: false, + defaultURI: (): URI => { + return URI.from("gdrive://"); + }, gateway: async (sthis): Promise => { return new GDriveGateway(sthis, { + auth: auth, driveURL: "https://www.googleapis.com/", }); - }, - test: async (sthis: SuperThis) => { - const gateway = new GDriveGateway(sthis, { - driveURL: "https://www.googleapis.com/", - }); - return new GDriveTestStore(sthis, gateway); - }, + } }); }); } From 226992d93cbbc1305564718f46c327025cd2333f Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Wed, 19 Mar 2025 09:21:46 +0100 Subject: [PATCH 08/11] Update drive-gateway.test.ts --- src/drive/drive-gateway.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/drive/drive-gateway.test.ts b/src/drive/drive-gateway.test.ts index 5f9eb273..26524e82 100644 --- a/src/drive/drive-gateway.test.ts +++ b/src/drive/drive-gateway.test.ts @@ -1,12 +1,16 @@ import { fireproof } from "@fireproof/core"; -import { connect } from "./index.js"; +import {registerGDriveStoreProtocol} from "./drive-gateway.ts" import { describe, it } from "vitest"; import { smokeDB } from "../../tests/helper.js"; -describe("store-register", () => { - it("should store and retrieve data", async () => { - const db = fireproof("my-database"); - connect(db, "ya29.a0AeXRPp4bASF7IqO0gx7MYSjgV8zMixuMR3z4rWvuqYvNUKv9pDuPumJIvmaXzmNE7k-7xArIdN9-VRippEG5ByIoqinkRlyNxunGmLq9oGZvgAiQ6_zUxHBHm1XL2Nx2lBjILtZZkJlFRNc_02wE5GZJ2VkLQar_rS-BKOqPaCgYKAfcSAQ8SFQHGX2Mia-YkdQYWm7KsWLbfF2ZwOQ0175") +describe("store-register", { timeout: 100000 }, () => { + it("should store and retrieve data", { timeout: 100000 }, async () => { + registerGDriveStoreProtocol("gdrive:", "ya29.a0AeXRPp7h4fANs0x2Y9nL3TCvL96bAnUJmwQ0oOlXPcKSmnajd_h8X2yxA8vdUo62CiKySJzrhYHMhwQVqvLnNVrHaSgR23PuF6rZXLXMApAu6rfRWtVFFKS8pYjEm36VW5csE656Z5bHXzWLXitqz9we-7zskpMNbak_NWOMaCgYKAXoSAQ8SFQHGX2MiXQzoB1I3l33KyL1DJICHNw0175") + const db = fireproof("diy-0.0", { + storeUrls: { + base: "gdrive://home-improvement", + }, + }); await smokeDB(db); }); }); From 1add3056370fd39ed28451526c8f9581f273412c Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Wed, 19 Mar 2025 09:40:49 +0100 Subject: [PATCH 09/11] Update package.json --- package.json | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/package.json b/package.json index 84b62f34..c497bc0d 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "@aws-sdk/s3-request-presigner": "^3.750.0", "@cloudflare/vitest-pool-workers": "^0.7.4", "@cloudflare/workers-types": "^4.20250224.0", - "@fireproof/core": "0.19.122", + "@fireproof/core": "0.20.0-dev-preview-50", "@fireproof/vendor": "~2.0.0", "@ipld/dag-ucan": "^3.4.5", "@jspm/core": "^2.1.0", @@ -135,17 +135,6 @@ "webdriverio": "^9.10.1", "ws": "^8.18.0" }, - "exports":{ - "./ucan": { - "browser": "./dist/ucan/web/index.js", - "node": "./dist/ucan/index.js", - "require": "./dist/ucan/index.cjs", - "script": "./dist/ucan/web/index.global.js", - "types": "./dist/ucan/index.d.ts" - }, - "./drive": "./dist/tsc/src/drive/index.js", - "./drive-gateway": "./dist/tsc/src/drive/drive-gateway.js" - }, "pnpm": { "onlyBuiltDependencies": [ "aws-sdk", From dcaef46895eb28a35bd4a9c3a332390c8075f90f Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:41:53 +0100 Subject: [PATCH 10/11] Update drive-gateway.ts --- src/drive/drive-gateway.ts | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/drive/drive-gateway.ts b/src/drive/drive-gateway.ts index ba23cb37..4e0984f6 100644 --- a/src/drive/drive-gateway.ts +++ b/src/drive/drive-gateway.ts @@ -76,12 +76,12 @@ export class GDriveGateway implements bs.Gateway { if (index) { name += `-${index}`; } - const fileId = await this.#search(name, store); + const fileId = await this.#search(name, this.#toStore(store)); if (fileId.isErr()) { if (isNotFoundError(fileId.Err())) { const fileData = new Blob([body], { type: "application/octet-stream" }); - const done = await this.#insert(name, body, store); + const done = await this.#insert(name, body, this.#toStore(store)); if (done.isErr()) { return done; } @@ -90,7 +90,7 @@ export class GDriveGateway implements bs.Gateway { } const fileData = new Blob([body], { type: "application/octet-stream" }); - const done = await this.#update(fileId.Ok(), fileData, store); + const done = await this.#update(fileId.Ok(), fileData, this.#toStore(store)); if (done.isErr()) { return done; } @@ -103,7 +103,7 @@ export class GDriveGateway implements bs.Gateway { Authorization: `Bearer ${this.params.auth}`, }; try { - const response = await fetch(BuildURI.from(url).appendRelative('drive/v3/files') + fileId, { + const response = await fetch(BuildURI.from(url).appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).toString(), { method: "DELETE", headers }); @@ -125,7 +125,7 @@ export class GDriveGateway implements bs.Gateway { headers = { Authorization: `Bearer ${this.params.auth}`, }; - response = await fetch(url.appendRelative(`drive/v3/files/${fileId}`).setParam("alt", "media").toString(), { + response = await fetch(url.appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).setParam("alt", "media").toString(), { method: "GET", headers, }); @@ -145,7 +145,7 @@ export class GDriveGateway implements bs.Gateway { if (index) { name += `-${index}`; } - const fileId = await this.#search(name, store); + const fileId = await this.#search(name, this.#toStore(store)); if (fileId.isErr()) { return Result.Err(fileId.Err()); } @@ -164,7 +164,7 @@ export class GDriveGateway implements bs.Gateway { if (index) { name += `-${index}`; } - const fileId = await this.#search(name, store); + const fileId = await this.#search(name, this.#toStore(store)); if (fileId.isErr()) { return fileId; } @@ -208,7 +208,7 @@ export class GDriveGateway implements bs.Gateway { async #update( fileId: string, fileData: Blob, - store: string + store: "data" | "wal" | "meta" ): Promise> { const url = BuildURI.from(this.params.driveURL); @@ -217,7 +217,7 @@ export class GDriveGateway implements bs.Gateway { "Content-Type": `fireproof/${store}`, }; const response = await fetch( - url.appendRelative(`upload/drive/v3/files/${fileId}`).setParam("uploadType", "media").toString(), + url.appendRelative('upload').appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).setParam("uploadType", "media").toString(), { method: "PATCH", headers, @@ -229,7 +229,7 @@ export class GDriveGateway implements bs.Gateway { } return Result.Ok(fileId); } - async #insert(fileName: string, content: Uint8Array, store: string): Promise> { + async #insert(fileName: string, content: Uint8Array, store: "data" | "wal" | "meta"): Promise> { const url = BuildURI.from(this.params.driveURL); const mime = `fireproof/${store}`; const file = new Blob([content], { type: mime }); @@ -255,7 +255,7 @@ export class GDriveGateway implements bs.Gateway { return this.logger.Error().Any({ store }).Err(err).Msg("Insert Error").ResultError(); } } - async #search(fileName: string, store: string): Promise> { + async #search(fileName: string, store: "data" | "wal" | "meta"): Promise> { try { const response = await fetch( BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=mimeType", `"fireproof/${store}" and name="${fileName}"`).toString(), @@ -277,6 +277,10 @@ export class GDriveGateway implements bs.Gateway { return this.logger.Error().Any({ fileName }).Err(err).Msg("Fetch Error").ResultError(); } } + #toStore(s: string): "data" | "wal" | "meta" { + if (["data", "wal", "meta"].includes(s)) { return s as "data"|"wal"|"meta" } + throw new Error(`unknown Store:${s}`) + } } // function generateRandom21DigitNumber() { // let num = Math.floor(Math.random() * 9) + 1; // First digit can't be 0 From ec9506cdf1aac19b378de85f7dfcd12cfadb4342 Mon Sep 17 00:00:00 2001 From: Benraouane Soufiane <110255764+BenraouaneSoufiane@users.noreply.github.com> Date: Wed, 19 Mar 2025 13:58:00 +0100 Subject: [PATCH 11/11] Create README.md --- src/drive/README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/drive/README.md diff --git a/src/drive/README.md b/src/drive/README.md new file mode 100644 index 00000000..0b0c9bd4 --- /dev/null +++ b/src/drive/README.md @@ -0,0 +1,44 @@ + +# `@fireproof/drive` + +[Fireproof](https://use-fireproof.com) is an embedded JavaScript document database that runs in the browser (or anywhere with JavaScript) and **[connects to any cloud](https://www.npmjs.com/package/@fireproof/connect)**. + +This module, `@fireproof/drive`, allows you to connect your Fireproof database to google drive via pre defined google drive api endpoints, enabling you to sync your data across multiple users in real-time. + +## Get started + +We assume you already have an app that uses Fireproof in the browser, and you want to setup collaboration among multiple users via the cloud. To write your first Fireproof app, see the [Fireproof quickstart](https://use-fireproof.com/docs/react-tutorial), otherwise read on. + +### 1. Install + +In your existing Fireproof app install the connector: + +```sh +npm install @fireproof/drive +``` + +### 2. Connect + +You're all done on the server, and ready to develop locally and then deploy with no further changes. Now you just need to register the google drive gateway in your client code. Fireproof already deployed the google drive api endpoints, so you don't need anything except fresh access token to sync data with your drive + +```js +// you already have this in your app +import { useFireproof } from "use-fireproof"; +// add this line +import { registerGDriveStoreProtocol } from "@fireproof/drive"; +``` + +You should call registerGDriveStoreProtocol('gdrive:', access_token) before calling useFireproof() hook + +```js +registerGDriveStoreProtocol('gdrive:', access_token) +const { database } = useFireproof("my-app-database-name", { + storeUrls: { + base: "gdrive://", + }, + }); +``` + +### 3. Collaborate + +Now you can use Fireproof as you normally would, and it will sync in realtime with other users. Any existing apps you have that use the [live query](https://use-fireproof.com/docs/react-hooks/use-live-query) or [subscription](https://use-fireproof.com/docs/database-api/database#subscribe) APIs will automatically render multi-user updates.