[POC] TanStackDB Attachments

demos/react-supabase-todolist-tanstackdb/pnpm-workspace.yaml

@@ -4,3 +4,7 @@ allowBuilds:
   '@journeyapps/wa-sqlite': true
   '@swc/core': true
   esbuild: true
+
+trustPolicyExclude:
+  - rollup@2.80.0
+  - semver@6.3.1

demos/react-supabase-todolist-tanstackdb/src/app/views/todo-lists/page.tsx

@@ -1,5 +1,5 @@
 import { NavigationPage } from '@/components/navigation/NavigationPage';
-import { listsCollection, useSupabase } from '@/components/providers/SystemProvider';
+import { attachmentQueue, listsCollection, useSupabase } from '@/components/providers/SystemProvider';
 import { GuardBySync } from '@/components/widgets/GuardBySync';
 import { SearchBarWidget } from '@/components/widgets/SearchBarWidget';
 import { TodoListsWidget } from '@/components/widgets/TodoListsWidget';
@@ -31,13 +31,21 @@ export default function TodoListsPage() {
       throw new Error(`Could not create new lists, no userID found`);
     }
 
-    // This could alternatively be synchronous and use optimistic updates
-    await listsCollection.insert({
-      id: crypto.randomUUID(),
-      name,
-      created_at: new Date(),
-      owner_id: userID
-    }).isPersisted.promise;
+    await attachmentQueue.saveFileTanStack({
+      // This is just random file data for this poc, this could be an image from a camera etc
+      data: btoa(crypto.randomUUID()),
+      fileExtension: 'jpg',
+      updateHook: async (attachmentRecord) => {
+        // This should happen in the same transaction as creating the attachment
+        listsCollection.insert({
+          id: crypto.randomUUID(),
+          name,
+          created_at: new Date(),
+          owner_id: userID,
+          photo_id: attachmentRecord.id // make the association for related data
+        });
+      }
+    });
   };
 
   return (

demos/react-supabase-todolist-tanstackdb/src/components/providers/Attachments.ts

@@ -0,0 +1,168 @@
+import { AppSchema } from '@/library/powersync/AppSchema';
+import {
+  AbstractPowerSyncDatabase,
+  AttachmentData,
+  AttachmentErrorHandler,
+  AttachmentQueue,
+  AttachmentRecord,
+  AttachmentService,
+  AttachmentState,
+  ILogger,
+  IndexDBFileSystemStorageAdapter,
+  LocalStorageAdapter,
+  RemoteStorageAdapter,
+  WatchedAttachmentItem
+} from '@powersync/web';
+import { Collection, createTransaction } from '@tanstack/db';
+import { PowerSyncTransactor } from '@tanstack/powersync-db-collection';
+
+export const LocalAttachmentStoage = new IndexDBFileSystemStorageAdapter('my-app-files');
+
+export const RemoteAttachmentStorage = {
+  async uploadFile(fileData: ArrayBuffer, attachment: AttachmentRecord) {
+    // no-op for poc
+  },
+
+  async downloadFile(attachment: AttachmentRecord): Promise<ArrayBuffer> {
+    // no-op for poc
+    return new ArrayBuffer();
+  },
+
+  async deleteFile(attachment: AttachmentRecord) {
+    // no-op for poc
+  }
+};
+
+/**
+ * This extends the default AttachmentQueue constructor params
+ * FIXME(powersync) we should export this type from the common SDK.
+ */
+type TanStackDBAttachmentQueueParams = {
+  db: AbstractPowerSyncDatabase;
+  /**
+   * For TanStack, we want access to the synced TanStackDB collection.
+   * In order to have the same relational data be set in a single transaction.
+   * This also allows for joining both TanStackDB collections.
+   */
+  attachmentsCollection: Collection<AttachmentQueueRow>;
+  remoteStorage: RemoteStorageAdapter;
+  localStorage: LocalStorageAdapter;
+  watchAttachments: (onUpdate: (attachment: WatchedAttachmentItem[]) => Promise<void>, signal: AbortSignal) => void;
+  tableName?: string;
+  logger?: ILogger;
+  syncIntervalMs?: number;
+  syncThrottleDuration?: number;
+  downloadAttachments?: boolean;
+  archivedCacheLimit?: number;
+  errorHandler?: AttachmentErrorHandler;
+};
+
+/**
+ * The PowerSync table row type
+ */
+type AttachmentQueueRow = (typeof AppSchema)['types']['attachments'];
+
+/**
+ * A custom extension of the PowerSyncAttachmentQueue.
+ * We could export something like this in the TanStackDB integration
+ */
+export class TanStackDBAttachmentQueue extends AttachmentQueue {
+  readonly powersync: AbstractPowerSyncDatabase;
+  readonly collection: Collection<AttachmentQueueRow>;
+
+  constructor(params: TanStackDBAttachmentQueueParams) {
+    super(params);
+    this.powersync = params.db;
+    this.collection = params.attachmentsCollection;
+  }
+
+  /**
+   * HACK: The AttachmentQueue should make this protected instead,
+   * in order for extensions to use it.
+   */
+  get _attachmentService(): AttachmentService {
+    // This is not protected, it's private and should be protected
+    return this['attachmentService'] as AttachmentService;
+  }
+
+  /**
+   * Saves a new attachment given the input data.
+   * Provides an updateHook which is called inside a TanStackDB transaction.
+   * Relational associataions with the provded attachment ID should be made in this hook.
+   */
+  async saveFileTanStack({
+    data,
+    fileExtension,
+    mediaType,
+    metaData,
+    id,
+    updateHook
+  }: {
+    data: AttachmentData;
+    fileExtension: string;
+    mediaType?: string;
+    metaData?: string;
+    id?: string;
+    // Note that this is called inside a synchronous TanStackDB transaction
+    // any mutations made to other collections, will be in the same transaction.
+    updateHook?: (attachment: AttachmentQueueRow) => Promise<void>;
+  }): Promise<AttachmentQueueRow> {
+    const resolvedId = id ?? (await this.generateAttachmentId());
+    const filename = `${resolvedId}.${fileExtension}`;
+    const localUri = this.localStorage.getLocalUri(filename);
+    const size = await this.localStorage.saveFile(localUri, data);
+
+    const attachment: AttachmentQueueRow = {
+      id: resolvedId,
+      filename,
+      media_type: mediaType ?? null,
+      local_uri: localUri,
+      state: AttachmentState.QUEUED_UPLOAD,
+      has_synced: 0,
+      size,
+      timestamp: new Date().getTime(),
+      meta_data: metaData ?? null
+    };
+
+    /**
+     * The use the attachmentService lock to prevent potential attachment queue race conditions.
+     * This specicifally prevents assuming a newly watched attachment record is one to download.
+     *  */
+    await this._attachmentService.withContext(async (ctx) => {
+      // Create a TanStackDB transaction context, the mutation will happen later
+      const tanStackDBTransaction = createTransaction({
+        autoCommit: false,
+        mutationFn: async ({ transaction }) => {
+          // Now we should apply the actual operations.
+          // We can save the attachment using dedicated APIs
+          await new PowerSyncTransactor({
+            database: ctx.db
+          }).applyTransaction(transaction);
+
+          // We don't need to explicitly use this here, the default transactor should
+          // be able to handle this (but it could be more future proof if we did support it later)
+          // await ctx.upsertAttachment(attachment, tx);
+        }
+      });
+
+      /**
+       * TODO, does the user want to have the attachment record peristed in this transaction or not?
+       * The implementation can be done according to the users's needs, devs should
+       * implement this saveFile override themselves, this is just an example.
+       *
+       * In this example, we write the attachment record first.
+       */
+      tanStackDBTransaction.mutate(() => {
+        // save the attachment record
+        this.collection.insert(attachment);
+        // allow the user to associate values in this transaction
+        updateHook?.(attachment);
+      });
+
+      // Actually perform the transaction
+      await tanStackDBTransaction.commit();
+    });
+
+    return attachment;
+  }
+}

demos/react-supabase-todolist-tanstackdb/src/components/providers/SystemProvider.tsx

@@ -4,19 +4,27 @@ import { SupabaseConnector } from '@/library/powersync/SupabaseConnector';
 import { TodosDeserializationSchema, TodosSchema } from '@/library/powersync/TodosSchema';
 import { CircularProgress } from '@mui/material';
 import { PowerSyncContext } from '@powersync/react';
-import { createBaseLogger, LogLevel, PowerSyncDatabase, WASQLiteOpenFactory, WASQLiteVFS } from '@powersync/web';
-import { createCollection } from '@tanstack/db';
+import {
+  createBaseLogger,
+  LogLevel,
+  PowerSyncDatabase,
+  WASQLiteOpenFactory,
+  WASQLiteVFS,
+  WatchedAttachmentItem
+} from '@powersync/web';
+import { createCollection, isNull, liveQueryCollectionOptions, not } from '@tanstack/db';
 import { powerSyncCollectionOptions } from '@tanstack/powersync-db-collection';
 import React, { Suspense } from 'react';
 import { NavigationPanelContextProvider } from '../navigation/NavigationPanelContext';
+import { LocalAttachmentStoage, RemoteAttachmentStorage, TanStackDBAttachmentQueue } from './Attachments';
 
 const SupabaseContext = React.createContext<SupabaseConnector | null>(null);
 export const useSupabase = () => React.useContext(SupabaseContext);
 
 export const db = new PowerSyncDatabase({
   schema: AppSchema,
   database: new WASQLiteOpenFactory({
-    dbFilename: 'example.db',
+    dbFilename: 'example-v2.db',
     vfs: WASQLiteVFS.OPFSCoopSyncVFS
   })
 });
@@ -47,6 +55,68 @@ export const todosCollection = createCollection(
   })
 );
 
+// Keep the local only attachment records in sync with TanStackDB
+export const attachmentsCollection = createCollection(
+  powerSyncCollectionOptions({
+    database: db,
+    table: AppSchema.props.attachments
+  })
+);
+
+export const attachmentQueue = new TanStackDBAttachmentQueue({
+  db: db, // PowerSync database instance
+  attachmentsCollection: attachmentsCollection as any, //TODO better typing,
+  localStorage: LocalAttachmentStoage,
+  remoteStorage: RemoteAttachmentStorage,
+
+  // Define which attachments exist in your data model
+  watchAttachments: async (onUpdate, abortSignal) => {
+    const livePhotoIds = createCollection(
+      liveQueryCollectionOptions({
+        query: (q) =>
+          q
+            .from({ document: listsCollection })
+            .where(({ document }) => not(isNull(document.photo_id)))
+            .select(({ document }) => ({
+              photo_id: document.photo_id
+            }))
+      })
+    );
+
+    const initialState = await livePhotoIds.stateWhenReady();
+
+    type LivePhotoId = { photo_id: string | null };
+    const mapper = (item: Partial<LivePhotoId>) =>
+      ({ id: item.photo_id!, fileExtension: 'jpg' }) satisfies WatchedAttachmentItem;
+
+    // report the initial state of all active attachment IDs
+    onUpdate(Array.from(initialState.values()).map(mapper));
+
+    // Subscribe for future changes
+    livePhotoIds.subscribeChanges((changes) => {
+      // we need the wholistic state for at every change
+      const allPhotoIds = livePhotoIds.map(mapper);
+      onUpdate(allPhotoIds);
+    });
+
+    abortSignal.addEventListener(
+      'abort',
+      () => {
+        // Stop the watched operations
+        livePhotoIds.cleanup();
+      },
+      { once: true }
+    );
+  },
+
+  // Optional configuration
+  syncIntervalMs: 30000, // Sync every 30 seconds
+  downloadAttachments: true, // Auto-download referenced files
+  archivedCacheLimit: 100 // Keep 100 archived files before cleanup
+});
+
+attachmentQueue.startSync();
+
 export type EnhancedListRecord = ListRecord & { total_tasks: number; completed_tasks: number };
 
 export const SystemProvider = ({ children }: { children: React.ReactNode }) => {

demos/react-supabase-todolist-tanstackdb/src/components/widgets/ListItemWidget.tsx

@@ -25,11 +25,12 @@ export type ListItemWidgetProps = {
   id: string;
   title: string;
   description: string;
+  localUri?: string | null;
   selected?: boolean;
 };
 
 export const ListItemWidget: React.FC<ListItemWidgetProps> = React.memo((props) => {
-  const { id, title, description, selected } = props;
+  const { id, title, description, localUri, selected } = props;
 
   const navigate = useNavigate();
 
@@ -79,14 +80,24 @@ export const ListItemWidget: React.FC<ListItemWidgetProps> = React.memo((props)
               <RightIcon />
             </IconButton>
           </Box>
-        }>
+        }
+      >
         <ListItemButton onClick={openList} selected={selected}>
           <ListItemAvatar>
             <Avatar>
               <ListIcon />
             </Avatar>
           </ListItemAvatar>
-          <ListItemText primary={title} secondary={description} />
+          <ListItemText
+            primary={title}
+            secondary={
+              <>
+                {description}
+                <br />
+                local_uri: {localUri ?? 'none'}
+              </>
+            }
+          />
         </ListItemButton>
       </ListItem>
     </S.MainPaper>

demos/react-supabase-todolist-tanstackdb/src/components/widgets/SearchBarWidget.tsx

@@ -25,7 +25,7 @@ export const SearchBarWidget: React.FC<any> = () => {
       q
         .from({ todos: todosCollection })
         .where(({ todos }) => like(todos.description, `%${searchInput}%`))
-        .join({ lists: listsCollection }, ({ todos, lists }) => eq(todos.list_id, lists.id))
+        .innerJoin({ lists: listsCollection }, ({ todos, lists }) => eq(todos.list_id, lists.id))
         .select(({ todos, lists }) => ({
           id: todos.id,
           list_id: todos.list_id,

demos/react-supabase-todolist-tanstackdb/src/components/widgets/TodoListsWidget.tsx

@@ -1,6 +1,6 @@
 import { List } from '@mui/material';
 import { count, eq, sum, useLiveQuery } from '@tanstack/react-db';
-import { listsCollection, todosCollection } from '../providers/SystemProvider';
+import { attachmentsCollection, listsCollection, todosCollection } from '../providers/SystemProvider';
 import { ListItemWidget } from './ListItemWidget';
 
 export type TodoListsWidgetProps = {
@@ -16,10 +16,12 @@ export function TodoListsWidget(props: TodoListsWidgetProps) {
     q
       .from({ lists: listsCollection })
       .leftJoin({ todos: todosCollection }, ({ lists, todos }) => eq(lists.id, todos.list_id))
-      .groupBy(({ lists }) => [lists.id, lists.name])
-      .select(({ lists, todos }) => ({
+      .leftJoin({ attachment: attachmentsCollection }, ({ lists, attachment }) => eq(lists.photo_id, attachment.id))
+      .groupBy(({ lists, attachment }) => [lists.id, lists.name, attachment.local_uri])
+      .select(({ lists, todos, attachment }) => ({
         id: lists.id,
         name: lists.name,
+        attachment_local_uri: attachment?.local_uri,
         total_tasks: count(todos?.id),
         completed_tasks: sum(todos?.completed as number)
       }))
@@ -41,6 +43,7 @@ export function TodoListsWidget(props: TodoListsWidgetProps) {
           id={r.id}
           title={r.name ?? ''}
           description={description(r.total_tasks, r.completed_tasks)}
+          localUri={r.attachment_local_uri}
           selected={r.id == props.selectedId}
         />
       ))}