From 23548744ebdaab53c6493df27febede2b4bfc7ac Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 11:55:01 +0100 Subject: [PATCH 01/26] Handle tags and move out events --- .../electric-db-collection/src/electric.ts | 281 +++++++++++++++++- .../electric-db-collection/src/tagIndex.ts | 162 ++++++++++ 2 files changed, 428 insertions(+), 15 deletions(-) create mode 100644 packages/electric-db-collection/src/tagIndex.ts diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 4598768fe..585b14d18 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -14,8 +14,19 @@ import { TimeoutWaitingForTxIdError, } from './errors' import { compileSQL } from './sql-compiler' +import { + addTagToIndex, + findRowsMatchingPattern, + getTagLength, + isMoveOutMessage, + removeTagFromIndex, + tagMatchesPattern, +} from './tagIndex' +import type { MoveOutPattern, MoveTag, RowId, TagIndex } from './tagIndex' import type { BaseCollectionConfig, + ChangeMessage, + Collection, CollectionConfig, DeleteMutationFnParams, InsertMutationFnParams, @@ -880,6 +891,185 @@ function createElectricSync>( // Store for the relation schema information const relationSchema = new Store(undefined) + // Tag tracking state + const rowTagSets = new Map>() + const tagIndex: TagIndex = [] + let tagLength: number | undefined = undefined + + /** + * Initialize the tag index with the correct length + */ + const initializeTagIndex = (length: number): void => { + if (tagIndex.length < length) { + // Extend the index array to the required length + for (let i = tagIndex.length; i < length; i++) { + tagIndex[i] = new Map() + } + } + } + + /** + * Add tags to a row and update the tag index + */ + const addTagsToRow = ( + tags: Array, + rowId: RowId, + rowTagSet: Set, + ): void => { + for (const tag of tags) { + // Infer tag length from first tag + if (tagLength === undefined) { + tagLength = getTagLength(tag) + initializeTagIndex(tagLength) + } + + // Validate tag length matches + const currentTagLength = getTagLength(tag) + if (currentTagLength !== tagLength) { + debug( + `${collectionId ? `[${collectionId}] ` : ``}Tag length mismatch: expected ${tagLength}, got ${currentTagLength}`, + ) + continue + } + + rowTagSet.add(tag) + addTagToIndex(tag, rowId, tagIndex, tagLength) + } + } + + /** + * Remove tags from a row and update the tag index + */ + const removeTagsFromRow = ( + removedTags: Array, + rowId: RowId, + rowTagSet: Set, + ): void => { + if (tagLength === undefined) { + return + } + + for (const tag of removedTags) { + const currentTagLength = getTagLength(tag) + if (currentTagLength === tagLength) { + rowTagSet.delete(tag) + removeTagFromIndex(tag, rowId, tagIndex, tagLength) + } + } + } + + /** + * Process tags for a change message (add and remove tags) + */ + const processTagsForChangeMessage = ( + tags: Array | undefined, + removedTags: Array | undefined, + rowId: RowId, + ): Set => { + // Initialize tag set for this row if it doesn't exist (needed for checking deletion) + if (!rowTagSets.has(rowId)) { + rowTagSets.set(rowId, new Set()) + } + const rowTagSet = rowTagSets.get(rowId)! + + // Add new tags + if (tags) { + addTagsToRow(tags, rowId, rowTagSet) + } + + // Remove tags + if (removedTags) { + removeTagsFromRow(removedTags, rowId, rowTagSet) + } + + return rowTagSet + } + + /** + * Clear all tag tracking state (used when truncating) + */ + const clearTagTrackingState = (): void => { + rowTagSets.clear() + tagIndex.length = 0 + tagLength = undefined + } + + /** + * Remove matching tags from a row based on a pattern + * Returns true if the row's tag set is now empty + */ + const removeMatchingTagsFromRow = ( + rowId: RowId, + pattern: MoveOutPattern, + ): boolean => { + const rowTagSet = rowTagSets.get(rowId) + if (!rowTagSet) { + return false + } + + // Find tags that match this pattern and remove them + for (const tag of rowTagSet) { + if (tagMatchesPattern(tag, pattern)) { + rowTagSet.delete(tag) + removeTagFromIndex(tag, rowId, tagIndex, tagLength!) + } + } + + // Check if row's tag set is now empty + if (rowTagSet.size === 0) { + rowTagSets.delete(rowId) + return true + } + + return false + } + + /** + * Process move-out event: remove matching tags from rows and delete rows with empty tag sets + */ + const processMoveOutEvent = ( + patterns: Array, + collection: Collection, + begin: () => void, + write: (message: Omit, `key`>) => void, + transactionStarted: boolean, + ): boolean => { + if (tagLength === undefined) { + debug( + `${collectionId ? `[${collectionId}] ` : ``}Received move-out message but no tag length set yet, ignoring`, + ) + return transactionStarted + } + + let txStarted = transactionStarted + + // Process all patterns and collect rows to delete + for (const pattern of patterns) { + // Find all rows that match this pattern + const affectedRowIds = findRowsMatchingPattern(pattern, tagIndex) + + for (const rowId of affectedRowIds) { + if (removeMatchingTagsFromRow(rowId, pattern)) { + // Delete rows with empty tag sets + if (!txStarted) { + begin() + txStarted = true + } + + const rowValue = collection.get(rowId) + if (rowValue !== undefined) { + write({ + type: `delete`, + value: rowValue, + }) + } + } + } + } + + return txStarted + } + /** * Get the sync metadata for insert operations * @returns Record containing relation information @@ -1074,14 +1264,38 @@ function createElectricSync>( transactionStarted = true } - write({ - type: message.headers.operation, - value: message.value, - // Include the primary key and relation info in the metadata - metadata: { - ...message.headers, - }, - }) + // Process tags if present + const tags = message.headers.tags + const removedTags = message.headers.removed_tags + const hasTags = tags || removedTags + + const rowId = collection.getKeyFromItem(message.value) + const rowTagSet = () => + processTagsForChangeMessage(tags, removedTags, rowId) + + // Check if row should be deleted (empty tag set) + // but only if the message includes tags + // because shapes without subqueries don't contain tags + // so we should keep those around + if (hasTags && rowTagSet().size === 0) { + rowTagSets.delete(rowId) + write({ + type: `delete`, + value: message.value, + metadata: { + ...message.headers, + }, + }) + } else { + write({ + type: message.headers.operation, + value: message.value, + // Include the primary key and relation info in the metadata + metadata: { + ...message.headers, + }, + }) + } } } else if (isSnapshotEndMessage(message)) { // Track postgres snapshot metadata for resolving awaiting mutations @@ -1097,6 +1311,15 @@ function createElectricSync>( if (commitPoint !== `up-to-date`) { commitPoint = `subset-end` } + } else if (isMoveOutMessage(message)) { + // Handle move-out event: remove matching tags from rows + transactionStarted = processMoveOutEvent( + message.headers.patterns, + collection, + begin, + write, + transactionStarted, + ) } else if (isMustRefetchMessage(message)) { debug( `${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate`, @@ -1110,6 +1333,9 @@ function createElectricSync>( truncate() + // Clear tag tracking state + clearTagTrackingState() + // Reset the loadSubset deduplication state since we're starting fresh // This ensures that previously loaded predicates don't prevent refetching after truncate loadSubsetDedupe?.reset() @@ -1134,16 +1360,41 @@ function createElectricSync>( // Truncate to clear all snapshot data truncate() + // Clear tag tracking state for atomic swap + clearTagTrackingState() + // Apply all buffered change messages and extract txids/snapshots for (const bufferedMsg of bufferedMessages) { if (isChangeMessage(bufferedMsg)) { - write({ - type: bufferedMsg.headers.operation, - value: bufferedMsg.value, - metadata: { - ...bufferedMsg.headers, - }, - }) + // Process tags for buffered messages + const tags = bufferedMsg.headers.tags + const removedTags = bufferedMsg.headers.removed_tags + const rowId = collection.getKeyFromItem(bufferedMsg.value) + + const rowTagSet = processTagsForChangeMessage( + tags, + removedTags, + rowId, + ) + + // Check if row should be deleted (empty tag set) + if (rowTagSet.size === 0) { + write({ + type: `delete`, + value: bufferedMsg.value, + metadata: { + ...bufferedMsg.headers, + }, + }) + } else { + write({ + type: bufferedMsg.headers.operation, + value: bufferedMsg.value, + metadata: { + ...bufferedMsg.headers, + }, + }) + } // Extract txids from buffered messages (will be committed to store after transaction) if (hasTxids(bufferedMsg)) { diff --git a/packages/electric-db-collection/src/tagIndex.ts b/packages/electric-db-collection/src/tagIndex.ts new file mode 100644 index 000000000..6632d1dca --- /dev/null +++ b/packages/electric-db-collection/src/tagIndex.ts @@ -0,0 +1,162 @@ +// Import Row and Message types for the isEventMessage function +import type { Message, Row } from '@electric-sql/client' + +export type RowId = string | number +export type MoveTag = Array +export type Position = number +export type Value = string +export type MoveOutPattern = { + position: Position + value: Value +} + +const TAG_WILDCARD = `_` + +/** + * Event message type for move-out events + */ +export interface EventMessage { + headers: { + event: `move-out` + patterns: Array + } +} + +/** + * Tag index structure: array indexed by position, maps value to set of row IDs. + * For example: + * ```example + * const tag1 = [a, b, c] + * const tag2 = [a, b, d] + * const tag3 = [a, d, e] + * + * // Index is: + * [ + * new Map([a -> ]) + * new Map([b -> , d -> ]) + * new Map([c -> , d -> , e -> ]) + * ] + * ``` + */ +export type TagIndex = Array>> + +/** + * Abstraction to get the value at a specific position in a tag + */ +export function getValue(tag: MoveTag, position: Position): Value { + if (position >= tag.length) { + throw new Error(`Position out of bounds`) + } + return tag[position]! +} + +/** + * Abstraction to extract position and value from a pattern. + */ +export function getPositionalValue(pattern: MoveOutPattern): { + position: number + value: string +} { + return pattern +} + +/** + * Abstraction to get the length of a tag + */ +export function getTagLength(tag: MoveTag): number { + return tag.length +} + +/** + * Check if a tag matches a pattern. + * A tag matches if the value at the pattern's position equals the pattern's value, + * or if the value at that position is "_" (wildcard). + */ +export function tagMatchesPattern( + tag: MoveTag, + pattern: MoveOutPattern, +): boolean { + const { position, value } = getPositionalValue(pattern) + const tagValue = getValue(tag, position) + return tagValue === value || tagValue === TAG_WILDCARD +} + +/** + * Add a tag to the index for efficient pattern matching + */ +export function addTagToIndex( + tag: MoveTag, + rowId: RowId, + index: TagIndex, + tagLength: number, +): void { + for (let i = 0; i < tagLength; i++) { + const value = getValue(tag, i) + + // Only index non-wildcard values + if (value !== TAG_WILDCARD) { + const positionIndex = index[i]! + if (!positionIndex.has(value)) { + positionIndex.set(value, new Set()) + } + + const tags = positionIndex.get(value)! + tags.add(rowId) + } + } +} + +/** + * Remove a tag from the index + */ +export function removeTagFromIndex( + tag: MoveTag, + rowId: RowId, + index: TagIndex, + tagLength: number, +): void { + for (let i = 0; i < tagLength; i++) { + const value = getValue(tag, i) + + // Only remove non-wildcard values + if (value !== TAG_WILDCARD) { + const positionIndex = index[i] + if (positionIndex) { + const rowSet = positionIndex.get(value) + if (rowSet) { + rowSet.delete(rowId) + + // Clean up empty sets + if (rowSet.size === 0) { + positionIndex.delete(value) + } + } + } + } + } +} + +/** + * Find all rows that match a given pattern + */ +export function findRowsMatchingPattern( + pattern: MoveOutPattern, + index: TagIndex, +): Set { + const { position, value } = getPositionalValue(pattern) + // the index for this position exists + // because we initialize the index with the right length + // as soon as we know the tag length (i.e. when we first receive a tag) + const positionIndex = index[position]! + const rowSet = positionIndex.get(value) + return rowSet ?? new Set() +} + +/** + * Check if a message is an event message with move-out event + */ +export function isMoveOutMessage>( + message: Message, +): message is Message & EventMessage { + return message.headers.event === `move-out` +} From e787eb57c4a26e3cb49895cac59836fdf0fc46af Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 11:56:48 +0100 Subject: [PATCH 02/26] Unit tests for tags --- .../electric-db-collection/tests/tags.test.ts | 1118 +++++++++++++++++ 1 file changed, 1118 insertions(+) create mode 100644 packages/electric-db-collection/tests/tags.test.ts diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts new file mode 100644 index 000000000..5d47e704f --- /dev/null +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -0,0 +1,1118 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createCollection } from '@tanstack/db' +import { electricCollectionOptions } from '../src/electric' +import type { ElectricCollectionUtils } from '../src/electric' +import type { Collection } from '@tanstack/db' +import type { Message, Row } from '@electric-sql/client' +import type { StandardSchemaV1 } from '@standard-schema/spec' +import type { MoveOutPattern, MoveTag } from '../src/tagIndex' + +// Mock the ShapeStream module +const mockSubscribe = vi.fn() +const mockRequestSnapshot = vi.fn() +const mockFetchSnapshot = vi.fn() +const mockStream = { + subscribe: mockSubscribe, + requestSnapshot: mockRequestSnapshot, + fetchSnapshot: mockFetchSnapshot, +} + +vi.mock(`@electric-sql/client`, async () => { + const actual = await vi.importActual(`@electric-sql/client`) + return { + ...actual, + ShapeStream: vi.fn(() => mockStream), + } +}) + +describe(`Electric Tag Tracking and GC`, () => { + let collection: Collection< + Row, + string | number, + ElectricCollectionUtils, + StandardSchemaV1, + Row + > + let subscriber: (messages: Array>) => void + + beforeEach(() => { + vi.clearAllMocks() + + // Reset mock subscriber + mockSubscribe.mockImplementation((callback) => { + subscriber = callback + return () => {} + }) + + // Reset mock requestSnapshot + mockRequestSnapshot.mockResolvedValue(undefined) + + // Create collection with Electric configuration + const config = { + id: `test`, + shapeOptions: { + url: `http://test-url`, + params: { + table: `test_table`, + }, + }, + startSync: true, + getKey: (item: Row) => item.id as number, + } + + // Get the options with utilities + const options = electricCollectionOptions(config) + + // Create collection with Electric configuration + collection = createCollection(options) as unknown as Collection< + Row, + string | number, + ElectricCollectionUtils, + StandardSchemaV1, + Row + > + }) + + it(`should track tags when rows are inserted with tags`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + + // Insert row with tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1, tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + expect(collection.status).toEqual(`ready`) + + // Remove first tag - row should still exist + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `update`, + removed_tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Remove last tag - row should be garbage collected + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `update`, + removed_tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual(new Map()) + }) + + it(`should track tags when rows are updated with new tags`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + + // Insert row with tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Update with additional tags + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + subscriber([ + { + key: `1`, + value: { id: 1, name: `Updated User` }, + headers: { + operation: `update`, + tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Updated User` }]]), + ) + + // Remove first tag - row should still exist + subscriber([ + { + key: `1`, + value: { id: 1, name: `Updated User` }, + headers: { + operation: `update`, + removed_tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Updated User` }]]), + ) + + // Remove last tag - row should be garbage collected + subscriber([ + { + key: `1`, + value: { id: 1, name: `Updated User` }, + headers: { + operation: `update`, + removed_tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual(new Map()) + }) + + it(`should not interfere between rows with distinct tags`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag3: MoveTag = [`hash7`, `hash8`, `hash9`] + const tag4: MoveTag = [`hash10`, `hash11`, `hash12`] + + // Insert multiple rows with some shared tags + // Row 1: tag1, tag2 + // Row 2: tag2 (shared with row 1), tag3 + // Row 3: tag3 (shared with row 2), tag4 + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [tag1, tag2], + }, + }, + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [tag2, tag3], + }, + }, + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `insert`, + tags: [tag3, tag4], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // All rows should exist + expect(collection.state.size).toBe(3) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove tag1 from row 1 - row 1 should still exist (has tag2), others unaffected + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `update`, + removed_tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 1 should still exist (has tag2), rows 2 and 3 unaffected + expect(collection.state.size).toBe(3) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove tag2 from row 1 (shared tag) - row 1 should be deleted + // Row 2 should still exist because it has tag3 (tag2 removal only affects row 1) + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `update`, + removed_tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 1 should be garbage collected, rows 2 and 3 should remain + // Row 2 still has tag2 and tag3, so removing tag2 from row 1 doesn't affect it + expect(collection.state.size).toBe(2) + expect(collection.state.has(1)).toBe(false) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove tag3 from row 2 - row 2 should still exist (has tag2) + // Row 3 should still exist because it has tag4 (tag3 removal only affects row 2) + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `update`, + removed_tags: [tag3], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 2 should still exist (has tag3), row 3 unaffected + expect(collection.state.size).toBe(2) + expect(collection.state.has(1)).toBe(false) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove tag2 from row 2 (shared tag) - row 2 should be deleted + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `update`, + removed_tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 2 should be garbage collected, row 3 should remain + // Row 3 still has tag3 and tag4 + expect(collection.state.size).toBe(1) + expect(collection.state.has(1)).toBe(false) + expect(collection.state.has(2)).toBe(false) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + }) + + it(`should require exact match in removed_tags for tags with wildcards (underscore)`, () => { + const tagWithWildcard: MoveTag = [`hash1`, `_`, `hash3`] + const tagWithoutWildcard: MoveTag = [`hash1`, `hash2`, `hash3`] + + // Insert row with wildcard tag + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [tagWithWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + + // Try to remove with non-matching tag (has specific value instead of wildcard) + // Should NOT remove because it doesn't match exactly + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `update`, + removed_tags: [tagWithoutWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist because the tag didn't match exactly + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + + // Remove with exact match (wildcard tag) + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `update`, + removed_tags: [tagWithWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be garbage collected because exact match was removed + expect(collection.state.size).toBe(0) + expect(collection.state.has(1)).toBe(false) + + // Insert row with specific value tag (no wildcard) + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [tagWithoutWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + + // Try to remove with wildcard tag - should NOT match + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `update`, + removed_tags: [tagWithWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist because wildcard doesn't match specific value + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + + // Remove with exact match (specific value tag) + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `update`, + removed_tags: [tagWithoutWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be garbage collected + expect(collection.state.size).toBe(0) + expect(collection.state.has(2)).toBe(false) + + // Test with multiple tags including wildcards + const tagWildcard1: MoveTag = [`hash1`, `_`, `hash3`] + const tagWildcard2: MoveTag = [`hash4`, `_`, `hash6`] + const tagSpecific: MoveTag = [`hash1`, `hash2`, `hash3`] + + subscriber([ + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `insert`, + tags: [tagWildcard1, tagWildcard2, tagSpecific], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove one wildcard tag with exact match + subscriber([ + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `update`, + removed_tags: [tagWildcard1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist (has tagWildcard2 and tagSpecific) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Try to remove wildcard tag with non-matching specific value + subscriber([ + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `update`, + removed_tags: [tagWithoutWildcard], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist because tagWithoutWildcard doesn't match tagWildcard2 + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove specific tag with exact match + subscriber([ + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `update`, + removed_tags: [tagSpecific], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist (has tagWildcard2) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Remove last wildcard tag with exact match + subscriber([ + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `update`, + removed_tags: [tagWildcard2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be garbage collected + expect(collection.state.size).toBe(0) + expect(collection.state.has(3)).toBe(false) + }) + + it(`should handle move-out events that remove matching tags`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash1`, `hash2`, `hash4`] + const tag3: MoveTag = [`hash5`, `hash6`, `hash1`] + + // Insert rows with tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [tag2], + }, + }, + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `insert`, + tags: [tag3], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.size).toBe(3) + + // Send move-out event with pattern matching hash1 at position 0 + const pattern: MoveOutPattern = { + position: 0, + value: `hash1`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [pattern], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Rows 1 and 2 should be deleted (they have hash1 at position 0) + // Row 3 should remain (has hash5 at position 0) + expect(collection.state.size).toBe(1) + expect(collection.state.has(3)).toBe(true) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + }) + + it(`should remove shared tags from all rows when move-out pattern matches`, () => { + // Create tags where some are shared between rows + const sharedTag1: MoveTag = [`hash1`, `hash2`, `hash3`] // Shared by rows 1 and 2 + const sharedTag2: MoveTag = [`hash4`, `hash5`, `hash6`] // Shared by rows 2 and 3 + const uniqueTag1: MoveTag = [`hash7`, `hash8`, `hash9`] // Only in row 1 + const uniqueTag2: MoveTag = [`hash10`, `hash11`, `hash12`] // Only in row 3 + + // Insert rows with multiple tags, some shared + // Row 1: sharedTag1, uniqueTag1 + // Row 2: sharedTag1, sharedTag2 + // Row 3: sharedTag2, uniqueTag2 + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [sharedTag1, uniqueTag1], + }, + }, + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [sharedTag1, sharedTag2], + }, + }, + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `insert`, + tags: [sharedTag2, uniqueTag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.size).toBe(3) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Send move-out event matching sharedTag1 (hash1 at position 0) + // This should remove sharedTag1 from both row 1 and row 2 + const pattern: MoveOutPattern = { + position: 0, + value: `hash1`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [pattern], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 1 should be deleted (only had sharedTag1 and uniqueTag1, sharedTag1 removed, but uniqueTag1 should remain... wait) + // Actually, if sharedTag1 matches the pattern, it should be removed from row 1 + // Row 1 has [sharedTag1, uniqueTag1], so after removing sharedTag1, it still has uniqueTag1 + // Row 2 has [sharedTag1, sharedTag2], so after removing sharedTag1, it still has sharedTag2 + // So both rows should still exist + expect(collection.state.size).toBe(3) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Send move-out event matching sharedTag2 (hash4 at position 0) + // This should remove sharedTag2 from both row 2 and row 3 + const pattern2: MoveOutPattern = { + position: 0, + value: `hash4`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [pattern2], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 2 should be deleted (had sharedTag1 and sharedTag2, both removed) + // Row 3 should still exist (has uniqueTag2) + // Row 1 should still exist (has uniqueTag1) + expect(collection.state.size).toBe(2) + expect(collection.state.has(2)).toBe(false) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + + // Send move-out event matching uniqueTag1 (hash7 at position 0) + // This should remove uniqueTag1 from row 1 + const pattern3: MoveOutPattern = { + position: 0, + value: `hash7`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [pattern3], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row 1 should be deleted (no tags left) + // Row 3 should still exist (has uniqueTag2) + expect(collection.state.size).toBe(1) + expect(collection.state.has(1)).toBe(false) + expect(collection.state.has(2)).toBe(false) + expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) + }) + + it(`should not remove tags with underscores when pattern matches non-indexed position`, () => { + // Tag with underscore at position 1: [a, _, c] + // This tag is NOT indexed at position 1 (because of underscore) + const tagWithUnderscore: MoveTag = [`a`, `_`, `c`] + + // Insert row with tag containing underscore + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [tagWithUnderscore], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.size).toBe(1) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + + // Send move-out event with pattern matching position 1 (where underscore is) + // Since the tag is not indexed at position 1, it won't be found in the index + // and the tag should remain + const patternNonIndexed: MoveOutPattern = { + position: 1, + value: `b`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [patternNonIndexed], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist because the tag wasn't found in the index + expect(collection.state.size).toBe(1) + expect(collection.state.get(1)).toEqual({ id: 1, name: `User 1` }) + + // Send move-out event with pattern matching position 2 (where 'c' is) + // Position 2 is indexed (has value 'c'), so it will be found in the index + // The pattern [*, *, c] matches the tag [a, _, c], so the tag is removed + const patternIndexed: MoveOutPattern = { + position: 2, + value: `c`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [patternIndexed], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be garbage collected because the tag was removed + // (tagset becomes empty) + expect(collection.state.size).toBe(0) + expect(collection.state.has(1)).toBe(false) + }) + + it(`should handle move-out events with multiple patterns`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag3: MoveTag = [`hash7`, `hash8`, `hash9`] + + // Insert rows with tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `User 1` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [tag2], + }, + }, + { + key: `3`, + value: { id: 3, name: `User 3` }, + headers: { + operation: `insert`, + tags: [tag3], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state.size).toBe(3) + + // Send move-out event with multiple patterns + const pattern1: MoveOutPattern = { + position: 0, + value: `hash1`, + } + const pattern2: MoveOutPattern = { + position: 0, + value: `hash4`, + } + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [pattern1, pattern2], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Rows 1 and 2 should be deleted, row 3 should remain + expect(collection.state.size).toBe(1) + expect(collection.state.has(3)).toBe(true) + }) + + it(`should clear tag state on must-refetch`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + + // Insert row with tag + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Send must-refetch + subscriber([ + { + headers: { control: `must-refetch` }, + }, + ]) + + // The collection should still have old data because truncate is in pending + // transaction. This is the intended behavior of the collection, you should have + // the old data until the next up-to-date message. + expect(collection.state.size).toBe(1) + expect(collection.state.has(1)).toBe(true) + expect(collection.state.get(1)).toEqual({ id: 1, name: `Test User` }) + + // Send new data after must-refetch + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { operation: `insert` }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Collection should now have the new data + expect(collection.state).toEqual(new Map([[2, { id: 2, name: `User 2` }]])) + + // Re-insert with new tag + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag2], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([ + [1, { id: 1, name: `Test User` }], + [2, { id: 2, name: `User 2` }], + ]), + ) + + // Remove tag2 and check that the row is gone + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `update`, + removed_tags: [tag2], + } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be garbage collected + expect(collection.state.size).toBe(1) + expect(collection.state.has(1)).toBe(false) + expect(collection.state.has(2)).toBe(true) + expect(collection.state.get(2)).toEqual({ id: 2, name: `User 2` }) + }) + + it(`should handle rows with no tags (not deleted)`, () => { + // Insert row without tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should exist even without tags + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Update the row without tags + subscriber([ + { + key: `1`, + old_value: { id: 1, name: `Test User` }, + value: { id: 1, name: `Updated Test User` }, + headers: { + operation: `update`, + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Updated Test User` }]]), + ) + + // Insert a row with tags + const tag: MoveTag = [`hash1`, `hash2`, `hash3`] + subscriber([ + { + key: `2`, + value: { id: 2, name: `User 2` }, + headers: { + operation: `insert`, + tags: [tag], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should exist + expect(collection.state).toEqual( + new Map([ + [1, { id: 1, name: `Updated Test User` }], + [2, { id: 2, name: `User 2` }], + ]), + ) + + // Move out that matches the tag + const pattern: MoveOutPattern = { + position: 1, + value: `hash2`, + } + + subscriber([ + { + headers: { event: `move-out`, patterns: [pattern] } as any, // TODO: remove this when pushing to CI + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // User 2 should be gine but user 1 should still exist because it was never tagged + expect(collection.state.size).toBe(1) + expect(collection.state.has(1)).toBe(true) + expect(collection.state.has(2)).toBe(false) + expect(collection.state.get(1)).toEqual({ + id: 1, + name: `Updated Test User`, + }) + }) + + it(`should handle adding and removing tags in same update`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + + // Insert row with tag1 + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Update: remove tag1, add tag2 + subscriber([ + { + key: `1`, + value: { id: 1, name: `Updated User` }, + headers: { + operation: `update`, + tags: [tag2], + removed_tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should still exist (has tag2) + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Updated User` }]]), + ) + }) +}) From 5a947575d5113dace8e4da581c5667113f26c56e Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 12:04:14 +0100 Subject: [PATCH 03/26] Changeset --- .changeset/witty-animals-agree.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/witty-animals-agree.md diff --git a/.changeset/witty-animals-agree.md b/.changeset/witty-animals-agree.md new file mode 100644 index 000000000..be350c75e --- /dev/null +++ b/.changeset/witty-animals-agree.md @@ -0,0 +1,5 @@ +--- +"@tanstack/electric-db-collection": patch +--- + +Support tagged rows and move out events in Electric collection. From a5aeecd99c8a15a1dc4e2ea23903434b9a3f0f30 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 14:22:52 +0100 Subject: [PATCH 04/26] Add unit test to check with tags that are structurally equal but different references --- .../electric-db-collection/tests/tags.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index 5d47e704f..e6d0b6a95 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -213,6 +213,47 @@ describe(`Electric Tag Tracking and GC`, () => { expect(collection.state).toEqual(new Map()) }) + it(`should track tags that are structurally equal`, () => { + const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag1Copy: MoveTag = [`hash1`, `hash2`, `hash3`] + + // Insert row with tags + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Remove first tag - row should be gone + subscriber([ + { + key: `1`, + value: { id: 1, name: `Updated User` }, + headers: { + operation: `update`, + removed_tags: [tag1Copy], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual(new Map()) + }) + it(`should not interfere between rows with distinct tags`, () => { const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] From d03074ab9550550281913987957189525964d776 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 15:46:40 +0100 Subject: [PATCH 05/26] Switch to string tags with values delimited by pipe character --- .../electric-db-collection/src/electric.ts | 49 ++++++++++--- .../src/{tagIndex.ts => tag-index.ts} | 13 ++-- .../electric-db-collection/tests/tags.test.ts | 68 +++++++++---------- 3 files changed, 81 insertions(+), 49 deletions(-) rename packages/electric-db-collection/src/{tagIndex.ts => tag-index.ts} (93%) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 585b14d18..9f01177c6 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -21,8 +21,14 @@ import { isMoveOutMessage, removeTagFromIndex, tagMatchesPattern, -} from './tagIndex' -import type { MoveOutPattern, MoveTag, RowId, TagIndex } from './tagIndex' +} from './tag-index' +import type { + MoveOutPattern, + MoveTag, + ParsedMoveTag, + RowId, + TagIndex, +} from './tag-index' import type { BaseCollectionConfig, ChangeMessage, @@ -891,6 +897,27 @@ function createElectricSync>( // Store for the relation schema information const relationSchema = new Store(undefined) + // //////// + // TODO: move this to the bottom of this file + const tagCache = new Map() + + /** + * Parses a tag string into a MoveTag. + * It memoizes the result parsed tag such that future calls + * for the same tag string return the same MoveTag array. + * @param tag - The tag string to parse. + * @returns The parsed MoveTag. + */ + const parseTag = (tag: MoveTag): ParsedMoveTag => { + if (tagCache.has(tag)) { + return tagCache.get(tag)! + } + const parsedTag = tag.split(`|`) + tagCache.set(tag, parsedTag) + return parsedTag + } + // //////////// + // Tag tracking state const rowTagSets = new Map>() const tagIndex: TagIndex = [] @@ -917,14 +944,16 @@ function createElectricSync>( rowTagSet: Set, ): void => { for (const tag of tags) { + const parsedTag = parseTag(tag) + // Infer tag length from first tag if (tagLength === undefined) { - tagLength = getTagLength(tag) + tagLength = getTagLength(parsedTag) initializeTagIndex(tagLength) } // Validate tag length matches - const currentTagLength = getTagLength(tag) + const currentTagLength = getTagLength(parsedTag) if (currentTagLength !== tagLength) { debug( `${collectionId ? `[${collectionId}] ` : ``}Tag length mismatch: expected ${tagLength}, got ${currentTagLength}`, @@ -933,7 +962,7 @@ function createElectricSync>( } rowTagSet.add(tag) - addTagToIndex(tag, rowId, tagIndex, tagLength) + addTagToIndex(parsedTag, rowId, tagIndex, tagLength) } } @@ -950,10 +979,11 @@ function createElectricSync>( } for (const tag of removedTags) { - const currentTagLength = getTagLength(tag) + const parsedTag = parseTag(tag) + const currentTagLength = getTagLength(parsedTag) if (currentTagLength === tagLength) { rowTagSet.delete(tag) - removeTagFromIndex(tag, rowId, tagIndex, tagLength) + removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength) } } } @@ -1009,9 +1039,10 @@ function createElectricSync>( // Find tags that match this pattern and remove them for (const tag of rowTagSet) { - if (tagMatchesPattern(tag, pattern)) { + const parsedTag = parseTag(tag) + if (tagMatchesPattern(parsedTag, pattern)) { rowTagSet.delete(tag) - removeTagFromIndex(tag, rowId, tagIndex, tagLength!) + removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength!) } } diff --git a/packages/electric-db-collection/src/tagIndex.ts b/packages/electric-db-collection/src/tag-index.ts similarity index 93% rename from packages/electric-db-collection/src/tagIndex.ts rename to packages/electric-db-collection/src/tag-index.ts index 6632d1dca..fc0dbdc89 100644 --- a/packages/electric-db-collection/src/tagIndex.ts +++ b/packages/electric-db-collection/src/tag-index.ts @@ -2,7 +2,8 @@ import type { Message, Row } from '@electric-sql/client' export type RowId = string | number -export type MoveTag = Array +export type MoveTag = string +export type ParsedMoveTag = Array export type Position = number export type Value = string export type MoveOutPattern = { @@ -43,7 +44,7 @@ export type TagIndex = Array>> /** * Abstraction to get the value at a specific position in a tag */ -export function getValue(tag: MoveTag, position: Position): Value { +export function getValue(tag: ParsedMoveTag, position: Position): Value { if (position >= tag.length) { throw new Error(`Position out of bounds`) } @@ -63,7 +64,7 @@ export function getPositionalValue(pattern: MoveOutPattern): { /** * Abstraction to get the length of a tag */ -export function getTagLength(tag: MoveTag): number { +export function getTagLength(tag: ParsedMoveTag): number { return tag.length } @@ -73,7 +74,7 @@ export function getTagLength(tag: MoveTag): number { * or if the value at that position is "_" (wildcard). */ export function tagMatchesPattern( - tag: MoveTag, + tag: ParsedMoveTag, pattern: MoveOutPattern, ): boolean { const { position, value } = getPositionalValue(pattern) @@ -85,7 +86,7 @@ export function tagMatchesPattern( * Add a tag to the index for efficient pattern matching */ export function addTagToIndex( - tag: MoveTag, + tag: ParsedMoveTag, rowId: RowId, index: TagIndex, tagLength: number, @@ -110,7 +111,7 @@ export function addTagToIndex( * Remove a tag from the index */ export function removeTagFromIndex( - tag: MoveTag, + tag: ParsedMoveTag, rowId: RowId, index: TagIndex, tagLength: number, diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index e6d0b6a95..dd484819d 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -5,7 +5,7 @@ import type { ElectricCollectionUtils } from '../src/electric' import type { Collection } from '@tanstack/db' import type { Message, Row } from '@electric-sql/client' import type { StandardSchemaV1 } from '@standard-schema/spec' -import type { MoveOutPattern, MoveTag } from '../src/tagIndex' +import type { MoveOutPattern } from '../src/tag-index' // Mock the ShapeStream module const mockSubscribe = vi.fn() @@ -74,8 +74,8 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should track tags when rows are inserted with tags`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` // Insert row with tags subscriber([ @@ -135,7 +135,7 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should track tags when rows are updated with new tags`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag1 = `hash1|hash2|hash3` // Insert row with tags subscriber([ @@ -157,7 +157,7 @@ describe(`Electric Tag Tracking and GC`, () => { ) // Update with additional tags - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag2 = `hash4|hash5|hash6` subscriber([ { key: `1`, @@ -214,8 +214,8 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should track tags that are structurally equal`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag1Copy: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag1 = `hash1|hash2|hash3` + const tag1Copy = `hash1|hash2|hash3` // Insert row with tags subscriber([ @@ -255,10 +255,10 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should not interfere between rows with distinct tags`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] - const tag3: MoveTag = [`hash7`, `hash8`, `hash9`] - const tag4: MoveTag = [`hash10`, `hash11`, `hash12`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` + const tag3 = `hash7|hash8|hash9` + const tag4 = `hash10|hash11|hash12` // Insert multiple rows with some shared tags // Row 1: tag1, tag2 @@ -390,8 +390,8 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should require exact match in removed_tags for tags with wildcards (underscore)`, () => { - const tagWithWildcard: MoveTag = [`hash1`, `_`, `hash3`] - const tagWithoutWildcard: MoveTag = [`hash1`, `hash2`, `hash3`] + const tagWithWildcard = `hash1|_|hash3` + const tagWithoutWildcard = `hash1|hash2|hash3` // Insert row with wildcard tag subscriber([ @@ -503,9 +503,9 @@ describe(`Electric Tag Tracking and GC`, () => { expect(collection.state.has(2)).toBe(false) // Test with multiple tags including wildcards - const tagWildcard1: MoveTag = [`hash1`, `_`, `hash3`] - const tagWildcard2: MoveTag = [`hash4`, `_`, `hash6`] - const tagSpecific: MoveTag = [`hash1`, `hash2`, `hash3`] + const tagWildcard1 = `hash1|_|hash3` + const tagWildcard2 = `hash4|_|hash6` + const tagSpecific = `hash1|hash2|hash3` subscriber([ { @@ -598,9 +598,9 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should handle move-out events that remove matching tags`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash1`, `hash2`, `hash4`] - const tag3: MoveTag = [`hash5`, `hash6`, `hash1`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash1|hash2|hash4` + const tag3 = `hash5|hash6|hash1` // Insert rows with tags subscriber([ @@ -662,10 +662,10 @@ describe(`Electric Tag Tracking and GC`, () => { it(`should remove shared tags from all rows when move-out pattern matches`, () => { // Create tags where some are shared between rows - const sharedTag1: MoveTag = [`hash1`, `hash2`, `hash3`] // Shared by rows 1 and 2 - const sharedTag2: MoveTag = [`hash4`, `hash5`, `hash6`] // Shared by rows 2 and 3 - const uniqueTag1: MoveTag = [`hash7`, `hash8`, `hash9`] // Only in row 1 - const uniqueTag2: MoveTag = [`hash10`, `hash11`, `hash12`] // Only in row 3 + const sharedTag1 = `hash1|hash2|hash3` // Shared by rows 1 and 2 + const sharedTag2 = `hash4|hash5|hash6` // Shared by rows 2 and 3 + const uniqueTag1 = `hash7|hash8|hash9` // Only in row 1 + const uniqueTag2 = `hash10|hash11|hash12` // Only in row 3 // Insert rows with multiple tags, some shared // Row 1: sharedTag1, uniqueTag1 @@ -790,9 +790,9 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should not remove tags with underscores when pattern matches non-indexed position`, () => { - // Tag with underscore at position 1: [a, _, c] + // Tag with underscore at position 1: a|_|c // This tag is NOT indexed at position 1 (because of underscore) - const tagWithUnderscore: MoveTag = [`a`, `_`, `c`] + const tagWithUnderscore = `a|_|c` // Insert row with tag containing underscore subscriber([ @@ -838,7 +838,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event with pattern matching position 2 (where 'c' is) // Position 2 is indexed (has value 'c'), so it will be found in the index - // The pattern [*, *, c] matches the tag [a, _, c], so the tag is removed + // The pattern matching position 2 with value 'c' matches the tag a|_|c, so the tag is removed const patternIndexed: MoveOutPattern = { position: 2, value: `c`, @@ -863,9 +863,9 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should handle move-out events with multiple patterns`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] - const tag3: MoveTag = [`hash7`, `hash8`, `hash9`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` + const tag3 = `hash7|hash8|hash9` // Insert rows with tags subscriber([ @@ -928,8 +928,8 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should clear tag state on must-refetch`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` // Insert row with tag subscriber([ @@ -1064,7 +1064,7 @@ describe(`Electric Tag Tracking and GC`, () => { ) // Insert a row with tags - const tag: MoveTag = [`hash1`, `hash2`, `hash3`] + const tag = `hash1|hash2|hash3` subscriber([ { key: `2`, @@ -1113,8 +1113,8 @@ describe(`Electric Tag Tracking and GC`, () => { }) it(`should handle adding and removing tags in same update`, () => { - const tag1: MoveTag = [`hash1`, `hash2`, `hash3`] - const tag2: MoveTag = [`hash4`, `hash5`, `hash6`] + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` // Insert row with tag1 subscriber([ From 9b57784747673db91b2527c9769f08464bae39c7 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 15:52:22 +0100 Subject: [PATCH 06/26] Remove tag from local cache when it is removed from a tag set --- .../electric-db-collection/src/electric.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 9f01177c6..b80e110ae 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -897,17 +897,11 @@ function createElectricSync>( // Store for the relation schema information const relationSchema = new Store(undefined) - // //////// - // TODO: move this to the bottom of this file const tagCache = new Map() - /** - * Parses a tag string into a MoveTag. - * It memoizes the result parsed tag such that future calls - * for the same tag string return the same MoveTag array. - * @param tag - The tag string to parse. - * @returns The parsed MoveTag. - */ + // Parses a tag string into a MoveTag. + // It memoizes the result parsed tag such that future calls + // for the same tag string return the same MoveTag array. const parseTag = (tag: MoveTag): ParsedMoveTag => { if (tagCache.has(tag)) { return tagCache.get(tag)! @@ -916,7 +910,6 @@ function createElectricSync>( tagCache.set(tag, parsedTag) return parsedTag } - // //////////// // Tag tracking state const rowTagSets = new Map>() @@ -984,6 +977,11 @@ function createElectricSync>( if (currentTagLength === tagLength) { rowTagSet.delete(tag) removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength) + // We aggresively evict the tag from the cache + // if this tag is shared with another row + // and is not removed from that other row + // then next time we encounter the tag it will be parsed again + tagCache.delete(tag) } } } From 25fce7de5cd6d78fe9b9106ba3e60faf78c372c4 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 16:12:39 +0100 Subject: [PATCH 07/26] Correctly handle rows without tags in progressive mode --- packages/electric-db-collection/src/electric.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index b80e110ae..a662c714e 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1398,16 +1398,17 @@ function createElectricSync>( // Process tags for buffered messages const tags = bufferedMsg.headers.tags const removedTags = bufferedMsg.headers.removed_tags + const hasTags = tags || removedTags const rowId = collection.getKeyFromItem(bufferedMsg.value) - const rowTagSet = processTagsForChangeMessage( - tags, - removedTags, - rowId, - ) + const rowTagSet = () => + processTagsForChangeMessage(tags, removedTags, rowId) // Check if row should be deleted (empty tag set) - if (rowTagSet.size === 0) { + // but only if the message includes tags + // because shapes without subqueries don't contain tags + // so we should keep those around + if (hasTags && rowTagSet().size === 0) { write({ type: `delete`, value: bufferedMsg.value, From c1457248bc92745e7005aa859ddc7e405e451a8b Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 17:01:17 +0100 Subject: [PATCH 08/26] Add test to check that delete cleans up tags --- .../electric-db-collection/tests/tags.test.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index dd484819d..4adfd8cf1 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -1156,4 +1156,86 @@ describe(`Electric Tag Tracking and GC`, () => { new Map([[1, { id: 1, name: `Updated User` }]]), ) }) + + it(`should not recover old tags when row is deleted and re-inserted`, () => { + const tag1 = `hash1|hash2|hash3` + const tag2 = `hash4|hash5|hash6` + + // Insert row with tag1 + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `insert`, + tags: [tag1], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Test User` }]]), + ) + + // Delete the row (without tags) + subscriber([ + { + key: `1`, + value: { id: 1, name: `Test User` }, + headers: { + operation: `delete`, + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be deleted + expect(collection.state.size).toBe(0) + expect(collection.state.has(1)).toBe(false) + + // Insert the row again with a new tag (tag2) + subscriber([ + { + key: `1`, + value: { id: 1, name: `Re-inserted User` }, + headers: { + operation: `insert`, + tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should exist with new tag + expect(collection.state).toEqual( + new Map([[1, { id: 1, name: `Re-inserted User` }]]), + ) + + // Update the row with removed_tags including its new tag (tag2) + // The row should NOT have the old tag1, only tag2 + subscriber([ + { + key: `1`, + value: { id: 1, name: `Re-inserted User` }, + headers: { + operation: `update`, + removed_tags: [tag2], + }, + }, + { + headers: { control: `up-to-date` }, + }, + ]) + + // Row should be gone because tag2 was removed and it doesn't have old tag1 + expect(collection.state.size).toBe(0) + expect(collection.state.has(1)).toBe(false) + }) }) From bf523ee57affc74aea42a475b3a6ea0149fe127e Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 1 Dec 2025 17:02:18 +0100 Subject: [PATCH 09/26] Clean tracked tags for row when row is deleted --- .../electric-db-collection/src/electric.ts | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index a662c714e..6887b6a9d 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1022,6 +1022,34 @@ function createElectricSync>( tagLength = undefined } + /** + * Remove all tags for a row from both the tag set and the index + * Used when a row is deleted + */ + const clearTagsForRow = (rowId: RowId): void => { + if (tagLength === undefined) { + return + } + + const rowTagSet = rowTagSets.get(rowId) + if (!rowTagSet) { + return + } + + // Remove each tag from the index + for (const tag of rowTagSet) { + const parsedTag = parseTag(tag) + const currentTagLength = getTagLength(parsedTag) + if (currentTagLength === tagLength) { + removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength) + } + tagCache.delete(tag) + } + + // Remove the row from the tag sets map + rowTagSets.delete(rowId) + } + /** * Remove matching tags from a row based on a pattern * Returns true if the row's tag set is now empty @@ -1307,7 +1335,7 @@ function createElectricSync>( // because shapes without subqueries don't contain tags // so we should keep those around if (hasTags && rowTagSet().size === 0) { - rowTagSets.delete(rowId) + clearTagsForRow(rowId) write({ type: `delete`, value: message.value, @@ -1316,6 +1344,9 @@ function createElectricSync>( }, }) } else { + if (message.headers.operation === `delete`) { + clearTagsForRow(rowId) + } write({ type: message.headers.operation, value: message.value, @@ -1409,6 +1440,7 @@ function createElectricSync>( // because shapes without subqueries don't contain tags // so we should keep those around if (hasTags && rowTagSet().size === 0) { + clearTagsForRow(rowId) write({ type: `delete`, value: bufferedMsg.value, @@ -1417,6 +1449,9 @@ function createElectricSync>( }, }) } else { + if (bufferedMsg.headers.operation === `delete`) { + clearTagsForRow(rowId) + } write({ type: bufferedMsg.headers.operation, value: bufferedMsg.value, From 9663f5041c7b95fdbe249804ac8a13fbbc5571de Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 13:59:26 +0100 Subject: [PATCH 10/26] e2e tests for move ins and out --- .../docker/docker-compose.yml | 1 + packages/db-collection-e2e/src/index.ts | 1 + .../src/suites/moves.suite.ts | 862 ++++++++++++++++++ .../e2e/electric.e2e.test.ts | 19 +- 4 files changed, 882 insertions(+), 1 deletion(-) create mode 100644 packages/db-collection-e2e/src/suites/moves.suite.ts diff --git a/packages/db-collection-e2e/docker/docker-compose.yml b/packages/db-collection-e2e/docker/docker-compose.yml index c2668b309..a756912c2 100644 --- a/packages/db-collection-e2e/docker/docker-compose.yml +++ b/packages/db-collection-e2e/docker/docker-compose.yml @@ -29,6 +29,7 @@ services: environment: DATABASE_URL: postgresql://postgres:password@postgres:5432/e2e_test?sslmode=disable ELECTRIC_INSECURE: true + ELECTRIC_FEATURE_FLAGS: "allow_subqueries,tagged_subqueries" ports: - '3000:3000' depends_on: diff --git a/packages/db-collection-e2e/src/index.ts b/packages/db-collection-e2e/src/index.ts index 5682bcc40..0e6bc0dcc 100644 --- a/packages/db-collection-e2e/src/index.ts +++ b/packages/db-collection-e2e/src/index.ts @@ -26,3 +26,4 @@ export { createCollationTestSuite } from './suites/collation.suite' export { createMutationsTestSuite } from './suites/mutations.suite' export { createLiveUpdatesTestSuite } from './suites/live-updates.suite' export { createProgressiveTestSuite } from './suites/progressive.suite' +export { createMovesTestSuite as createTagsTestSuite } from './suites/moves.suite' diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts new file mode 100644 index 000000000..74cdefbfe --- /dev/null +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -0,0 +1,862 @@ +/** + * Tags Test Suite + * + * Tests Electric collection tag behavior with subqueries + * Only Electric collection supports tags (via shapes with subqueries) + */ + +import { randomUUID } from 'node:crypto' +import { describe, expect, it, beforeAll } from 'vitest' +import { createCollection } from '@tanstack/db' +import { electricCollectionOptions } from '@tanstack/electric-db-collection' +import { waitFor } from '../utils/helpers' +import type { E2ETestConfig } from '../types' +import type { Client } from 'pg' +import type { Collection } from '@tanstack/db' +import type { ElectricCollectionUtils } from '@tanstack/electric-db-collection' + +interface TagsTestConfig extends E2ETestConfig { + tagsTestSetup: { + dbClient: Client + baseUrl: string + testSchema: string + usersTable: string + postsTable: string + } +} + +export function createMovesTestSuite(getConfig: () => Promise) { + describe(`Tags Suite`, () => { + let usersTable: string + let postsTable: string + let dbClient: Client + let baseUrl: string + let testSchema: string + let config: TagsTestConfig + + beforeAll(async () => { + const testConfig = await getConfig() + if (!testConfig.tagsTestSetup) { + throw new Error(`Tags test setup not configured`) + } + + config = testConfig as TagsTestConfig + const setup = config.tagsTestSetup + dbClient = setup.dbClient + baseUrl = setup.baseUrl + testSchema = setup.testSchema + usersTable = setup.usersTable + postsTable = setup.postsTable + }) + + // Helper to create a collection on posts table with WHERE clause that has nested subquery + // This creates a shape: posts WHERE userId IN (SELECT id FROM users WHERE isActive = true) + // When a user's isActive changes, posts will move in/out of this shape + function createPostsByActiveUsersCollection( + id: string = `tags-posts-active-users-${Date.now()}`, + ): Collection { + // Remove quotes from table names for the WHERE clause SQL + const usersTableUnquoted = usersTable.replace(/"/g, ``) + + return createCollection( + electricCollectionOptions({ + id, + shapeOptions: { + url: `${baseUrl}/v1/shape`, + params: { + table: `${testSchema}.${postsTable}`, + // WHERE clause with nested subquery + // Posts will move in/out when users' isActive changes + // Column reference should be just the column name, not the full table path + where: `"userId" IN (SELECT id FROM ${testSchema}.${usersTableUnquoted} WHERE "isActive" = true)`, + }, + }, + syncMode: `eager`, + getKey: (item: any) => item.id, + startSync: true, + }), + ) as any + } + + // Helper to wait for collection to be ready + async function waitForReady( + collection: Collection, + ) { + await collection.preload() + await waitFor(() => collection.status === `ready`, { + timeout: 30000, + message: `Collection did not become ready`, + }) + } + + // Helper to wait for a specific item to appear + async function waitForItem( + collection: Collection, + itemId: string, + timeout: number = 10000, + ) { + await waitFor(() => collection.has(itemId), { + timeout, + message: `Item ${itemId} did not appear in collection`, + }) + } + + // Helper to wait for a specific item to disappear + async function waitForItemRemoved( + collection: Collection, + itemId: string, + timeout: number = 10000, + ) { + await waitFor(() => !collection.has(itemId), { + timeout, + message: `Item ${itemId} was not removed from collection`, + }) + } + + it.only(`1. Initial snapshot contains only posts from active users`, async () => { + // Create collection on posts with WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) + const collection = createPostsByActiveUsersCollection() + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert 2 active users and 1 inactive user + const userId1 = randomUUID() + const userId2 = randomUUID() + const userId3 = randomUUID() + + await config.mutations.insertUser({ + id: userId1, + name: `Active User 1`, + email: `user1@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId2, + name: `Active User 2`, + email: `user2@test.com`, + age: 30, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId3, + name: `Inactive User`, + email: `user3@test.com`, + age: 42, + isActive: false, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert posts for these users + const postId1 = randomUUID() + const postId2 = randomUUID() + const postId3 = randomUUID() + + await config.mutations.insertPost({ + id: postId1, + userId: userId1, + title: `Post 1`, + content: `Content 1`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId2, + userId: userId2, + title: `Post 2`, + content: `Content 2`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId3, + userId: userId3, + title: `Post 3`, + content: `Content 3`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for collection to sync + await waitForReady(collection) + + // Wait for both posts to appear (users are active, so posts match the subquery) + await waitForItem(collection, postId1) + await waitForItem(collection, postId2) + + // Verify only posts 1 and 2 are in the collection + expect(collection.has(postId1)).toBe(true) + expect(collection.has(postId2)).toBe(true) + expect(collection.has(postId3)).toBe(false) + + // Wait a bit to make sure post 3 is not coming in later + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(collection.has(postId3)).toBe(false) + + // Note: Tags are internal to Electric and may not be directly accessible + // The test verifies that posts with matching conditions appear in snapshot + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable}`) + await dbClient.query(`DELETE FROM ${usersTable}`) + await collection.cleanup() + }) + + it(`2. Move-in: row becomes eligible for subquery`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = false + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Inactive User`, + email: `inactive@test.com`, + age: 25, + isActive: false, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Inactive User Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait a bit to ensure post doesn't appear (user is inactive, so post doesn't match subquery) + await new Promise((resolve) => setTimeout(resolve, 2000)) + expect(collection.has(postId)).toBe(false) + + // Update user to isActive = true (move-in for the post) + await config.mutations.updateUser(userId, { isActive: true }) + + // Wait for post to appear (move-in) + await waitForItem(collection, postId, 15000) + expect(collection.has(postId)).toBe(true) + expect(collection.get(postId)?.title).toBe(`Inactive User Post`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`3. Move-out: row becomes ineligible for subquery`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Active User Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for post to appear (user is active, so post matches subquery) + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update user to isActive = false (move-out for the post) + await config.mutations.updateUser(userId, { isActive: false }) + + // Wait for post to be removed (move-out) + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`4. Move-out → move-in cycle ("flapping row")`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Flapping User`, + email: `flapping@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Flapping Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Move-out: isActive = false + await config.mutations.updateUser(userId, { isActive: false }) + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Move-in: isActive = true + await config.mutations.updateUser(userId, { isActive: true }) + await waitForItem(collection, postId, 15000) + expect(collection.has(postId)).toBe(true) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`5. Tags-only update (row stays within subquery)`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Tagged Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update post title (tags might change but post stays in subquery since user is still active) + await dbClient.query( + `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, + [`Updated Tagged Post`, postId], + ) + + // Wait a bit and verify post still exists + await new Promise((resolve) => setTimeout(resolve, 2000)) + expect(collection.has(postId)).toBe(true) + expect(collection.get(postId)?.title).toBe(`Updated Tagged Post`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`6. Database DELETE triggers removed_at`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `To Be Deleted`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Delete post in Postgres + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + + // Wait for post to be removed + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Clean up + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`7. Join-based subquery: move-out when join breaks`, async () => { + // This test uses the same pattern as others - posts with WHERE clause referencing users + // The WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) + // acts as a join condition + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User for Join`, + email: `join@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post referencing the user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Join Test Post`, + content: `Test content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update user.isActive = false (breaks subquery condition, post moves out) + await config.mutations.updateUser(userId, { isActive: false }) + + // Wait for post to be removed (move-out) + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Update user.isActive = true again (post moves in) + await config.mutations.updateUser(userId, { isActive: true }) + await waitForItem(collection, postId, 15000) + expect(collection.has(postId)).toBe(true) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + + await collection.cleanup() + }) + + it(`8. Concurrent/rapid updates must not cause 409 conflicts`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Rapid Update User`, + email: `rapid@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Rapid Update Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + + // Apply rapid sequence in a transaction (changing user isActive) + await dbClient.query(`BEGIN`) + try { + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [false, userId], + ) + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [true, userId], + ) + await dbClient.query( + `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, + [`Updated Title`, postId], + ) + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [false, userId], + ) + await dbClient.query(`COMMIT`) + } catch (error) { + await dbClient.query(`ROLLBACK`) + throw error + } + + // Wait for final state (post should be removed since user is inactive) + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Verify no errors occurred (collection should still be ready) + expect(collection.status).toBe(`ready`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`9. Snapshot after move-out should not re-include removed rows`, async () => { + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Create first collection + const collection1 = createPostsByActiveUsersCollection() + await waitForReady(collection1) + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Snapshot Test User`, + email: `snapshot@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Snapshot Test Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection1, postId) + expect(collection1.has(postId)).toBe(true) + + // Update user → post moves out + await config.mutations.updateUser(userId, { isActive: false }) + + await waitForItemRemoved(collection1, postId, 15000) + expect(collection1.has(postId)).toBe(false) + + // Clean up first collection + await collection1.cleanup() + + // Create fresh collection (new subscription) + const collection2 = createPostsByActiveUsersCollection() + await waitForReady(collection2) + + // Wait a bit to ensure snapshot is complete + await new Promise((resolve) => setTimeout(resolve, 2000)) + + // Snapshot should NOT include the removed post (user is inactive) + expect(collection2.has(postId)).toBe(false) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection2.cleanup() + }) + + it(`10. Multi-row batch: some rows move in, some move out`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert 3 users all with isActive = true + const userId1 = randomUUID() + const userId2 = randomUUID() + const userId3 = randomUUID() + + await config.mutations.insertUser({ + id: userId1, + name: `User 1`, + email: `user1@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId2, + name: `User 2`, + email: `user2@test.com`, + age: 30, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId3, + name: `User 3`, + email: `user3@test.com`, + age: 35, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert posts for these users + const postId1 = randomUUID() + const postId2 = randomUUID() + const postId3 = randomUUID() + + await config.mutations.insertPost({ + id: postId1, + userId: userId1, + title: `Post 1`, + content: `Content 1`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId2, + userId: userId2, + title: `Post 2`, + content: `Content 2`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId3, + userId: userId3, + title: `Post 3`, + content: `Content 3`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for all posts to appear + await waitForItem(collection, postId1) + await waitForItem(collection, postId2) + await waitForItem(collection, postId3) + + // In one SQL transaction: + // user1: isActive → false (post1 moves out) + // post2: title change (stays in since user2 is still active) + // user3/post3: no change + await dbClient.query(`BEGIN`) + try { + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [false, userId1], + ) + await dbClient.query( + `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, + [`Updated Post 2`, postId2], + ) + // user3/post3: no change + await dbClient.query(`COMMIT`) + } catch (error) { + await dbClient.query(`ROLLBACK`) + throw error + } + + // Wait for changes to propagate + await waitForItemRemoved(collection, postId1, 15000) + expect(collection.has(postId1)).toBe(false) // post1: moved out (user1 inactive) + expect(collection.has(postId2)).toBe(true) // post2: still in (user2 active) + expect(collection.get(postId2)?.title).toBe(`Updated Post 2`) + expect(collection.has(postId3)).toBe(true) // post3: still in (user3 active) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId1]) + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId2]) + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId3]) + await config.mutations.deleteUser(userId1) + await config.mutations.deleteUser(userId2) + await config.mutations.deleteUser(userId3) + await collection.cleanup() + }) + + it(`11. Tags = null / empty array must not trigger move-out`, async () => { + const collection = createPostsByActiveUsersCollection() + await waitForReady(collection) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Tags Test Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update post content (non-filtering field) - should not cause move-out + // The post stays in because the user is still active + await dbClient.query( + `UPDATE ${postsTable} SET content = $1 WHERE id = $2`, + [`Updated Content`, postId], + ) + + // Wait a bit and verify post still exists (no move-out) + await new Promise((resolve) => setTimeout(resolve, 2000)) + expect(collection.has(postId)).toBe(true) + expect(collection.get(postId)?.content).toBe(`Updated Content`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + }) +} diff --git a/packages/electric-db-collection/e2e/electric.e2e.test.ts b/packages/electric-db-collection/e2e/electric.e2e.test.ts index c8ef05e44..395a52163 100644 --- a/packages/electric-db-collection/e2e/electric.e2e.test.ts +++ b/packages/electric-db-collection/e2e/electric.e2e.test.ts @@ -17,6 +17,7 @@ import { createPaginationTestSuite, createPredicatesTestSuite, createProgressiveTestSuite, + createTagsTestSuite, generateSeedData, } from '../../db-collection-e2e/src/index' import { waitFor } from '../../db-collection-e2e/src/utils/helpers' @@ -31,7 +32,15 @@ declare module 'vitest' { } describe(`Electric Collection E2E Tests`, () => { - let config: E2ETestConfig + let config: E2ETestConfig & { + tagsTestSetup?: { + dbClient: Client + baseUrl: string + testSchema: string + usersTable: string + postsTable: string + } + } let dbClient: Client let usersTable: string let postsTable: string @@ -433,6 +442,13 @@ describe(`Electric Collection E2E Tests`, () => { commentsUpToDateControl.current?.() }, }, + tagsTestSetup: { + dbClient, + baseUrl, + testSchema, + usersTable, + postsTable, + }, getTxid: async () => { // Get the current transaction ID from the last operation // This uses pg_current_xact_id_if_assigned() which returns the txid @@ -578,4 +594,5 @@ describe(`Electric Collection E2E Tests`, () => { createMutationsTestSuite(getConfig) createLiveUpdatesTestSuite(getConfig) createProgressiveTestSuite(getConfig) + createTagsTestSuite(getConfig as any) }) From 28b06297ad8b4746084ea9f8e97cb1305e7aed42 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 14:46:21 +0100 Subject: [PATCH 11/26] Fix tag indexing --- packages/electric-db-collection/src/tag-index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/electric-db-collection/src/tag-index.ts b/packages/electric-db-collection/src/tag-index.ts index fc0dbdc89..4c3a44aa2 100644 --- a/packages/electric-db-collection/src/tag-index.ts +++ b/packages/electric-db-collection/src/tag-index.ts @@ -145,11 +145,8 @@ export function findRowsMatchingPattern( index: TagIndex, ): Set { const { position, value } = getPositionalValue(pattern) - // the index for this position exists - // because we initialize the index with the right length - // as soon as we know the tag length (i.e. when we first receive a tag) - const positionIndex = index[position]! - const rowSet = positionIndex.get(value) + const positionIndex = index[position] + const rowSet = positionIndex?.get(value) return rowSet ?? new Set() } From 7facdcba6e9d6d3be57c76a9cfbbf6d98b1161c5 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 14:46:42 +0100 Subject: [PATCH 12/26] Fix tests --- .../src/suites/moves.suite.ts | 255 +++--------------- 1 file changed, 33 insertions(+), 222 deletions(-) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 74cdefbfe..58ac7461e 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -6,7 +6,7 @@ */ import { randomUUID } from 'node:crypto' -import { describe, expect, it, beforeAll } from 'vitest' +import { beforeAll, describe, expect, it } from 'vitest' import { createCollection } from '@tanstack/db' import { electricCollectionOptions } from '@tanstack/electric-db-collection' import { waitFor } from '../utils/helpers' @@ -26,7 +26,7 @@ interface TagsTestConfig extends E2ETestConfig { } export function createMovesTestSuite(getConfig: () => Promise) { - describe(`Tags Suite`, () => { + describe(`Moves Suite`, () => { let usersTable: string let postsTable: string let dbClient: Client @@ -35,12 +35,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { let config: TagsTestConfig beforeAll(async () => { - const testConfig = await getConfig() - if (!testConfig.tagsTestSetup) { - throw new Error(`Tags test setup not configured`) - } - - config = testConfig as TagsTestConfig + config = await getConfig() const setup = config.tagsTestSetup dbClient = setup.dbClient baseUrl = setup.baseUrl @@ -105,7 +100,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { async function waitForItemRemoved( collection: Collection, itemId: string, - timeout: number = 10000, + timeout: number = 2000, ) { await waitFor(() => !collection.has(itemId), { timeout, @@ -113,7 +108,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { }) } - it.only(`1. Initial snapshot contains only posts from active users`, async () => { + it(`Initial snapshot contains only posts from active users`, async () => { // Create collection on posts with WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) const collection = createPostsByActiveUsersCollection() @@ -222,7 +217,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`2. Move-in: row becomes eligible for subquery`, async () => { + it(`Move-in: row becomes eligible for subquery`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -257,14 +252,14 @@ export function createMovesTestSuite(getConfig: () => Promise) { }) // Wait a bit to ensure post doesn't appear (user is inactive, so post doesn't match subquery) - await new Promise((resolve) => setTimeout(resolve, 2000)) + await new Promise((resolve) => setTimeout(resolve, 500)) expect(collection.has(postId)).toBe(false) // Update user to isActive = true (move-in for the post) await config.mutations.updateUser(userId, { isActive: true }) // Wait for post to appear (move-in) - await waitForItem(collection, postId, 15000) + await waitForItem(collection, postId, 1000) expect(collection.has(postId)).toBe(true) expect(collection.get(postId)?.title).toBe(`Inactive User Post`) @@ -274,7 +269,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`3. Move-out: row becomes ineligible for subquery`, async () => { + it(`Move-out: row becomes ineligible for subquery`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -316,7 +311,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await config.mutations.updateUser(userId, { isActive: false }) // Wait for post to be removed (move-out) - await waitForItemRemoved(collection, postId, 15000) + await waitForItemRemoved(collection, postId) expect(collection.has(postId)).toBe(false) // Clean up @@ -325,7 +320,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`4. Move-out → move-in cycle ("flapping row")`, async () => { + it(`Move-out → move-in cycle ("flapping row")`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -378,7 +373,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`5. Tags-only update (row stays within subquery)`, async () => { + it(`Tags-only update (row stays within subquery)`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -422,7 +417,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { ) // Wait a bit and verify post still exists - await new Promise((resolve) => setTimeout(resolve, 2000)) + await new Promise((resolve) => setTimeout(resolve, 500)) expect(collection.has(postId)).toBe(true) expect(collection.get(postId)?.title).toBe(`Updated Tagged Post`) @@ -432,7 +427,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`6. Database DELETE triggers removed_at`, async () => { + it(`Database DELETE leads to row being removed from collection`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -473,7 +468,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) // Wait for post to be removed - await waitForItemRemoved(collection, postId, 15000) + await waitForItemRemoved(collection, postId) expect(collection.has(postId)).toBe(false) // Clean up @@ -481,140 +476,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection.cleanup() }) - it(`7. Join-based subquery: move-out when join breaks`, async () => { - // This test uses the same pattern as others - posts with WHERE clause referencing users - // The WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) - // acts as a join condition - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Active User for Join`, - email: `join@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post referencing the user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Join Test Post`, - content: `Test content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Update user.isActive = false (breaks subquery condition, post moves out) - await config.mutations.updateUser(userId, { isActive: false }) - - // Wait for post to be removed (move-out) - await waitForItemRemoved(collection, postId, 15000) - expect(collection.has(postId)).toBe(false) - - // Update user.isActive = true again (post moves in) - await config.mutations.updateUser(userId, { isActive: true }) - await waitForItem(collection, postId, 15000) - expect(collection.has(postId)).toBe(true) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - - await collection.cleanup() - }) - - it(`8. Concurrent/rapid updates must not cause 409 conflicts`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Rapid Update User`, - email: `rapid@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Rapid Update Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - - // Apply rapid sequence in a transaction (changing user isActive) - await dbClient.query(`BEGIN`) - try { - await dbClient.query( - `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [false, userId], - ) - await dbClient.query( - `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [true, userId], - ) - await dbClient.query( - `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, - [`Updated Title`, postId], - ) - await dbClient.query( - `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [false, userId], - ) - await dbClient.query(`COMMIT`) - } catch (error) { - await dbClient.query(`ROLLBACK`) - throw error - } - - // Wait for final state (post should be removed since user is inactive) - await waitForItemRemoved(collection, postId, 15000) - expect(collection.has(postId)).toBe(false) - - // Verify no errors occurred (collection should still be ready) - expect(collection.status).toBe(`ready`) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`9. Snapshot after move-out should not re-include removed rows`, async () => { + it(`Snapshot after move-out should not re-include removed rows`, async () => { if (!config.mutations) { throw new Error(`Mutations not configured`) } @@ -655,7 +517,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { // Update user → post moves out await config.mutations.updateUser(userId, { isActive: false }) - await waitForItemRemoved(collection1, postId, 15000) + await waitForItemRemoved(collection1, postId) expect(collection1.has(postId)).toBe(false) // Clean up first collection @@ -666,7 +528,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await waitForReady(collection2) // Wait a bit to ensure snapshot is complete - await new Promise((resolve) => setTimeout(resolve, 2000)) + await new Promise((resolve) => setTimeout(resolve, 1000)) // Snapshot should NOT include the removed post (user is inactive) expect(collection2.has(postId)).toBe(false) @@ -677,7 +539,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection2.cleanup() }) - it(`10. Multi-row batch: some rows move in, some move out`, async () => { + it(`Multi-row batch: some rows move in, some move out`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) @@ -695,7 +557,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { name: `User 1`, email: `user1@test.com`, age: 25, - isActive: true, + isActive: false, createdAt: new Date(), metadata: null, deletedAt: null, @@ -761,26 +623,30 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) - // Wait for all posts to appear - await waitForItem(collection, postId1) + // Wait for posts 2 and 3 to appear await waitForItem(collection, postId2) await waitForItem(collection, postId3) + expect(collection.has(postId1)).toBe(false) + // In one SQL transaction: - // user1: isActive → false (post1 moves out) + // user1: isActive → true (post1 moves in) // post2: title change (stays in since user2 is still active) - // user3/post3: no change + // user3: isActive → false (post3 moves out) await dbClient.query(`BEGIN`) try { await dbClient.query( `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [false, userId1], + [true, userId1], ) await dbClient.query( `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, [`Updated Post 2`, postId2], ) - // user3/post3: no change + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [false, userId3], + ) await dbClient.query(`COMMIT`) } catch (error) { await dbClient.query(`ROLLBACK`) @@ -788,11 +654,11 @@ export function createMovesTestSuite(getConfig: () => Promise) { } // Wait for changes to propagate - await waitForItemRemoved(collection, postId1, 15000) - expect(collection.has(postId1)).toBe(false) // post1: moved out (user1 inactive) + await waitForItemRemoved(collection, postId3) + expect(collection.has(postId1)).toBe(true) // post1: moved in (user1 active) expect(collection.has(postId2)).toBe(true) // post2: still in (user2 active) expect(collection.get(postId2)?.title).toBe(`Updated Post 2`) - expect(collection.has(postId3)).toBe(true) // post3: still in (user3 active) + expect(collection.has(postId3)).toBe(false) // post3: moved out (user3 inactive) // Clean up await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId1]) @@ -803,60 +669,5 @@ export function createMovesTestSuite(getConfig: () => Promise) { await config.mutations.deleteUser(userId3) await collection.cleanup() }) - - it(`11. Tags = null / empty array must not trigger move-out`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Active User`, - email: `active@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Tags Test Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Update post content (non-filtering field) - should not cause move-out - // The post stays in because the user is still active - await dbClient.query( - `UPDATE ${postsTable} SET content = $1 WHERE id = $2`, - [`Updated Content`, postId], - ) - - // Wait a bit and verify post still exists (no move-out) - await new Promise((resolve) => setTimeout(resolve, 2000)) - expect(collection.has(postId)).toBe(true) - expect(collection.get(postId)?.content).toBe(`Updated Content`) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) }) } From 8a42bd919c92fe179a60ee1482a0b4752101c77f Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 15:18:57 +0100 Subject: [PATCH 13/26] Renamed test --- packages/db-collection-e2e/src/suites/moves.suite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 58ac7461e..348592a0d 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -539,7 +539,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { await collection2.cleanup() }) - it(`Multi-row batch: some rows move in, some move out`, async () => { + it(`Multi-row transaction: some rows move in, some move out`, async () => { const collection = createPostsByActiveUsersCollection() await waitForReady(collection) From 9f184980fbea80ae43ee1fcf64a94e62bd081cd1 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 15:35:14 +0100 Subject: [PATCH 14/26] Address feedback (part 1) --- .../electric-db-collection/src/electric.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 6887b6a9d..994bf367d 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -903,9 +903,11 @@ function createElectricSync>( // It memoizes the result parsed tag such that future calls // for the same tag string return the same MoveTag array. const parseTag = (tag: MoveTag): ParsedMoveTag => { - if (tagCache.has(tag)) { - return tagCache.get(tag)! + const cachedTag = tagCache.get(tag) + if (cachedTag) { + return cachedTag } + const parsedTag = tag.split(`|`) tagCache.set(tag, parsedTag) return parsedTag @@ -973,16 +975,13 @@ function createElectricSync>( for (const tag of removedTags) { const parsedTag = parseTag(tag) - const currentTagLength = getTagLength(parsedTag) - if (currentTagLength === tagLength) { - rowTagSet.delete(tag) - removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength) - // We aggresively evict the tag from the cache - // if this tag is shared with another row - // and is not removed from that other row - // then next time we encounter the tag it will be parsed again - tagCache.delete(tag) - } + rowTagSet.delete(tag) + removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength) + // We aggresively evict the tag from the cache + // if this tag is shared with another row + // and is not removed from that other row + // then next time we encounter the tag it will be parsed again + tagCache.delete(tag) } } From a8eb10b9b89e812dedd01290e44097a7e3b71cd1 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 15:48:47 +0100 Subject: [PATCH 15/26] Extract duplicated code --- .../electric-db-collection/src/electric.ts | 115 +++++++----------- 1 file changed, 46 insertions(+), 69 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 994bf367d..5debf7276 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1238,6 +1238,50 @@ function createElectricSync>( syncMode === `progressive` && !hasReceivedUpToDate const bufferedMessages: Array> = [] // Buffer change messages during initial sync + /** + * Process a change message: handle tags and write the mutation + */ + const processChangeMessage = (changeMessage: Message) => { + if (!isChangeMessage(changeMessage)) { + return + } + + // Process tags if present + const tags = changeMessage.headers.tags + const removedTags = changeMessage.headers.removed_tags + const hasTags = tags || removedTags + + const rowId = collection.getKeyFromItem(changeMessage.value) + const rowTagSet = () => + processTagsForChangeMessage(tags, removedTags, rowId) + + // Check if row should be deleted (empty tag set) + // but only if the message includes tags + // because shapes without subqueries don't contain tags + // so we should keep those around + if (hasTags && rowTagSet().size === 0) { + clearTagsForRow(rowId) + write({ + type: `delete`, + value: changeMessage.value, + metadata: { + ...changeMessage.headers, + }, + }) + } else { + if (changeMessage.headers.operation === `delete`) { + clearTagsForRow(rowId) + } + write({ + type: changeMessage.headers.operation, + value: changeMessage.value, + metadata: { + ...changeMessage.headers, + }, + }) + } + } + // Create deduplicated loadSubset wrapper for non-eager modes // This prevents redundant snapshot requests when multiple concurrent // live queries request overlapping or subset predicates @@ -1320,41 +1364,7 @@ function createElectricSync>( transactionStarted = true } - // Process tags if present - const tags = message.headers.tags - const removedTags = message.headers.removed_tags - const hasTags = tags || removedTags - - const rowId = collection.getKeyFromItem(message.value) - const rowTagSet = () => - processTagsForChangeMessage(tags, removedTags, rowId) - - // Check if row should be deleted (empty tag set) - // but only if the message includes tags - // because shapes without subqueries don't contain tags - // so we should keep those around - if (hasTags && rowTagSet().size === 0) { - clearTagsForRow(rowId) - write({ - type: `delete`, - value: message.value, - metadata: { - ...message.headers, - }, - }) - } else { - if (message.headers.operation === `delete`) { - clearTagsForRow(rowId) - } - write({ - type: message.headers.operation, - value: message.value, - // Include the primary key and relation info in the metadata - metadata: { - ...message.headers, - }, - }) - } + processChangeMessage(message) } } else if (isSnapshotEndMessage(message)) { // Track postgres snapshot metadata for resolving awaiting mutations @@ -1425,40 +1435,7 @@ function createElectricSync>( // Apply all buffered change messages and extract txids/snapshots for (const bufferedMsg of bufferedMessages) { if (isChangeMessage(bufferedMsg)) { - // Process tags for buffered messages - const tags = bufferedMsg.headers.tags - const removedTags = bufferedMsg.headers.removed_tags - const hasTags = tags || removedTags - const rowId = collection.getKeyFromItem(bufferedMsg.value) - - const rowTagSet = () => - processTagsForChangeMessage(tags, removedTags, rowId) - - // Check if row should be deleted (empty tag set) - // but only if the message includes tags - // because shapes without subqueries don't contain tags - // so we should keep those around - if (hasTags && rowTagSet().size === 0) { - clearTagsForRow(rowId) - write({ - type: `delete`, - value: bufferedMsg.value, - metadata: { - ...bufferedMsg.headers, - }, - }) - } else { - if (bufferedMsg.headers.operation === `delete`) { - clearTagsForRow(rowId) - } - write({ - type: bufferedMsg.headers.operation, - value: bufferedMsg.value, - metadata: { - ...bufferedMsg.headers, - }, - }) - } + processChangeMessage(bufferedMsg) // Extract txids from buffered messages (will be committed to store after transaction) if (hasTxids(bufferedMsg)) { From f55ca94e9ae1a916601f1cf807bf1769d3c6478e Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 16:03:22 +0100 Subject: [PATCH 16/26] Removed impossible branch and fix tag unit tests to mock correct Electric behavior when all tags are removed --- .../electric-db-collection/src/electric.ts | 36 ++++++------------- .../electric-db-collection/tests/tags.test.ts | 20 +++++------ 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 5debf7276..ca835bc76 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1252,34 +1252,20 @@ function createElectricSync>( const hasTags = tags || removedTags const rowId = collection.getKeyFromItem(changeMessage.value) - const rowTagSet = () => - processTagsForChangeMessage(tags, removedTags, rowId) - // Check if row should be deleted (empty tag set) - // but only if the message includes tags - // because shapes without subqueries don't contain tags - // so we should keep those around - if (hasTags && rowTagSet().size === 0) { + if (changeMessage.headers.operation === `delete`) { clearTagsForRow(rowId) - write({ - type: `delete`, - value: changeMessage.value, - metadata: { - ...changeMessage.headers, - }, - }) - } else { - if (changeMessage.headers.operation === `delete`) { - clearTagsForRow(rowId) - } - write({ - type: changeMessage.headers.operation, - value: changeMessage.value, - metadata: { - ...changeMessage.headers, - }, - }) + } else if (hasTags) { + processTagsForChangeMessage(tags, removedTags, rowId) } + + write({ + type: changeMessage.headers.operation, + value: changeMessage.value, + metadata: { + ...changeMessage.headers, + }, + }) } // Create deduplicated loadSubset wrapper for non-eager modes diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index 4adfd8cf1..823f76b1a 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -122,7 +122,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `Test User` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], }, }, @@ -201,7 +201,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `Updated User` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], }, }, @@ -242,7 +242,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `Updated User` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag1Copy], }, }, @@ -328,7 +328,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `User 1` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], }, }, @@ -372,7 +372,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `2`, value: { id: 2, name: `User 2` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], }, }, @@ -435,7 +435,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `User 1` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tagWithWildcard], }, }, @@ -489,7 +489,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `2`, value: { id: 2, name: `User 2` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tagWithoutWildcard], }, }, @@ -583,7 +583,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `3`, value: { id: 3, name: `User 3` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tagWildcard2], }, }, @@ -1007,7 +1007,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `Test User` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], } as any, // TODO: remove this when pushing to CI }, @@ -1225,7 +1225,7 @@ describe(`Electric Tag Tracking and GC`, () => { key: `1`, value: { id: 1, name: `Re-inserted User` }, headers: { - operation: `update`, + operation: `delete`, removed_tags: [tag2], }, }, From 333cc5010f5a35c7c5f14b4a5c8d12d07470cf8c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 2 Dec 2025 16:27:06 +0100 Subject: [PATCH 17/26] Fix types in tests --- .../electric-db-collection/src/tag-index.ts | 14 +++---- .../electric-db-collection/tests/tags.test.ts | 38 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/electric-db-collection/src/tag-index.ts b/packages/electric-db-collection/src/tag-index.ts index 4c3a44aa2..eeecab67f 100644 --- a/packages/electric-db-collection/src/tag-index.ts +++ b/packages/electric-db-collection/src/tag-index.ts @@ -7,7 +7,7 @@ export type ParsedMoveTag = Array export type Position = number export type Value = string export type MoveOutPattern = { - position: Position + pos: Position value: Value } @@ -54,8 +54,8 @@ export function getValue(tag: ParsedMoveTag, position: Position): Value { /** * Abstraction to extract position and value from a pattern. */ -export function getPositionalValue(pattern: MoveOutPattern): { - position: number +function getPositionalValue(pattern: MoveOutPattern): { + pos: number value: string } { return pattern @@ -77,8 +77,8 @@ export function tagMatchesPattern( tag: ParsedMoveTag, pattern: MoveOutPattern, ): boolean { - const { position, value } = getPositionalValue(pattern) - const tagValue = getValue(tag, position) + const { pos, value } = getPositionalValue(pattern) + const tagValue = getValue(tag, pos) return tagValue === value || tagValue === TAG_WILDCARD } @@ -144,8 +144,8 @@ export function findRowsMatchingPattern( pattern: MoveOutPattern, index: TagIndex, ): Set { - const { position, value } = getPositionalValue(pattern) - const positionIndex = index[position] + const { pos, value } = getPositionalValue(pattern) + const positionIndex = index[pos] const rowSet = positionIndex?.get(value) return rowSet ?? new Set() } diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index 823f76b1a..01dfa2773 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -637,7 +637,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event with pattern matching hash1 at position 0 const pattern: MoveOutPattern = { - position: 0, + pos: 0, value: `hash1`, } @@ -646,7 +646,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [pattern], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -709,7 +709,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event matching sharedTag1 (hash1 at position 0) // This should remove sharedTag1 from both row 1 and row 2 const pattern: MoveOutPattern = { - position: 0, + pos: 0, value: `hash1`, } @@ -718,7 +718,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [pattern], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -738,7 +738,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event matching sharedTag2 (hash4 at position 0) // This should remove sharedTag2 from both row 2 and row 3 const pattern2: MoveOutPattern = { - position: 0, + pos: 0, value: `hash4`, } @@ -747,7 +747,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [pattern2], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -765,7 +765,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event matching uniqueTag1 (hash7 at position 0) // This should remove uniqueTag1 from row 1 const pattern3: MoveOutPattern = { - position: 0, + pos: 0, value: `hash7`, } @@ -774,7 +774,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [pattern3], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -816,7 +816,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Since the tag is not indexed at position 1, it won't be found in the index // and the tag should remain const patternNonIndexed: MoveOutPattern = { - position: 1, + pos: 1, value: `b`, } @@ -825,7 +825,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [patternNonIndexed], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -840,7 +840,7 @@ describe(`Electric Tag Tracking and GC`, () => { // Position 2 is indexed (has value 'c'), so it will be found in the index // The pattern matching position 2 with value 'c' matches the tag a|_|c, so the tag is removed const patternIndexed: MoveOutPattern = { - position: 2, + pos: 2, value: `c`, } @@ -849,7 +849,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [patternIndexed], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -902,11 +902,11 @@ describe(`Electric Tag Tracking and GC`, () => { // Send move-out event with multiple patterns const pattern1: MoveOutPattern = { - position: 0, + pos: 0, value: `hash1`, } const pattern2: MoveOutPattern = { - position: 0, + pos: 0, value: `hash4`, } @@ -915,7 +915,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { event: `move-out`, patterns: [pattern1, pattern2], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -987,7 +987,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { operation: `insert`, tags: [tag2], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -1009,7 +1009,7 @@ describe(`Electric Tag Tracking and GC`, () => { headers: { operation: `delete`, removed_tags: [tag2], - } as any, // TODO: remove this when pushing to CI + }, }, { headers: { control: `up-to-date` }, @@ -1089,13 +1089,13 @@ describe(`Electric Tag Tracking and GC`, () => { // Move out that matches the tag const pattern: MoveOutPattern = { - position: 1, + pos: 1, value: `hash2`, } subscriber([ { - headers: { event: `move-out`, patterns: [pattern] } as any, // TODO: remove this when pushing to CI + headers: { event: `move-out`, patterns: [pattern] }, }, { headers: { control: `up-to-date` }, From 9f5a4d8b954bf7007ffced684b45f19907637cc8 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 09:31:17 +0000 Subject: [PATCH 18/26] ci: apply automated fixes --- packages/db-collection-e2e/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db-collection-e2e/docker/docker-compose.yml b/packages/db-collection-e2e/docker/docker-compose.yml index a756912c2..441aa5e48 100644 --- a/packages/db-collection-e2e/docker/docker-compose.yml +++ b/packages/db-collection-e2e/docker/docker-compose.yml @@ -29,7 +29,7 @@ services: environment: DATABASE_URL: postgresql://postgres:password@postgres:5432/e2e_test?sslmode=disable ELECTRIC_INSECURE: true - ELECTRIC_FEATURE_FLAGS: "allow_subqueries,tagged_subqueries" + ELECTRIC_FEATURE_FLAGS: 'allow_subqueries,tagged_subqueries' ports: - '3000:3000' depends_on: From 66f12e2153183b2e808d19c035b8016994f66ee0 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 13:19:54 +0100 Subject: [PATCH 19/26] Track state of rows in TX to be able to delete rows on a move out --- .../electric-db-collection/src/electric.ts | 110 ++++++++++++++---- 1 file changed, 89 insertions(+), 21 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index ca835bc76..9ac715ca0 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1089,6 +1089,7 @@ function createElectricSync>( begin: () => void, write: (message: Omit, `key`>) => void, transactionStarted: boolean, + transactionState: Map, ): boolean => { if (tagLength === undefined) { debug( @@ -1112,12 +1113,15 @@ function createElectricSync>( txStarted = true } - const rowValue = collection.get(rowId) + // Get row value from transaction state (uncommitted) or collection + const rowValue = transactionState.get(rowId) ?? collection.get(rowId) if (rowValue !== undefined) { write({ type: `delete`, value: rowValue, }) + // Remove from transaction state since we're deleting it + transactionState.delete(rowId) } } } @@ -1238,6 +1242,10 @@ function createElectricSync>( syncMode === `progressive` && !hasReceivedUpToDate const bufferedMessages: Array> = [] // Buffer change messages during initial sync + // Track row state during the current transaction to access uncommitted row values + // This allows us to handle partial updates correctly by merging with existing state + const transactionState = new Map() + /** * Process a change message: handle tags and write the mutation */ @@ -1252,20 +1260,62 @@ function createElectricSync>( const hasTags = tags || removedTags const rowId = collection.getKeyFromItem(changeMessage.value) + const operation = changeMessage.headers.operation + + if (operation === `insert`) { + // For insert, store the full row in transaction state + transactionState.set(rowId, changeMessage.value) + + if (hasTags) { + processTagsForChangeMessage(tags, removedTags, rowId) + } - if (changeMessage.headers.operation === `delete`) { + write({ + type: `insert`, + value: changeMessage.value, + metadata: { + ...changeMessage.headers, + }, + }) + } else if (operation === `update`) { + // For update, merge with existing state (from transaction state or collection) + const existingValue = + transactionState.get(rowId) ?? collection.get(rowId) + + // Merge the update with existing value (handles partial updates) + const updatedValue = + existingValue !== undefined + ? Object.assign({}, existingValue, changeMessage.value) + : changeMessage.value + + // Store the merged result in transaction state + transactionState.set(rowId, updatedValue) + + if (hasTags) { + processTagsForChangeMessage(tags, removedTags, rowId) + } + + write({ + type: `update`, + value: updatedValue, + metadata: { + ...changeMessage.headers, + }, + }) + } else { + // Operation is delete clearTagsForRow(rowId) - } else if (hasTags) { - processTagsForChangeMessage(tags, removedTags, rowId) + // Remove from transaction state + transactionState.delete(rowId) + + write({ + type: `delete`, + value: changeMessage.value, + metadata: { + ...changeMessage.headers, + }, + }) } - - write({ - type: changeMessage.headers.operation, - value: changeMessage.value, - metadata: { - ...changeMessage.headers, - }, - }) } // Create deduplicated loadSubset wrapper for non-eager modes @@ -1293,7 +1343,7 @@ function createElectricSync>( for (const message of messages) { // Add message to current batch buffer (for race condition handling) - if (isChangeMessage(message)) { + if (isChangeMessage(message) || isMoveOutMessage(message)) { currentBatchMessages.setState((currentBuffer) => { const newBuffer = [...currentBuffer, message] // Limit buffer size for safety @@ -1367,14 +1417,20 @@ function createElectricSync>( commitPoint = `subset-end` } } else if (isMoveOutMessage(message)) { - // Handle move-out event: remove matching tags from rows - transactionStarted = processMoveOutEvent( - message.headers.patterns, - collection, - begin, - write, - transactionStarted, - ) + // Handle move-out event: buffer if buffering, otherwise process immediately + if (isBufferingInitialSync()) { + bufferedMessages.push(message) + } else { + // Normal processing: process move-out immediately + transactionStarted = processMoveOutEvent( + message.headers.patterns, + collection, + begin, + write, + transactionStarted, + transactionState, + ) + } } else if (isMustRefetchMessage(message)) { debug( `${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate`, @@ -1432,11 +1488,22 @@ function createElectricSync>( } else if (isSnapshotEndMessage(bufferedMsg)) { // Extract snapshots from buffered messages (will be committed to store after transaction) newSnapshots.push(parseSnapshotMessage(bufferedMsg)) + } else if (isMoveOutMessage(bufferedMsg)) { + // Process buffered move-out messages during atomic swap + processMoveOutEvent( + bufferedMsg.headers.patterns, + collection, + begin, + write, + transactionStarted, + transactionState, + ) } } // Commit the atomic swap commit() + transactionState.clear() // Exit buffering phase by marking that we've received up-to-date // isBufferingInitialSync() will now return false @@ -1451,6 +1518,7 @@ function createElectricSync>( if (transactionStarted) { commit() transactionStarted = false + transactionState.clear() } } wrappedMarkReady(isBufferingInitialSync()) From 5ba8d77845ffcfbd06581183245bdf7337235bf3 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 15:49:06 +0100 Subject: [PATCH 20/26] Do not rename test suite on import --- packages/db-collection-e2e/src/index.ts | 2 +- packages/electric-db-collection/e2e/electric.e2e.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/db-collection-e2e/src/index.ts b/packages/db-collection-e2e/src/index.ts index 0e6bc0dcc..bccff24e2 100644 --- a/packages/db-collection-e2e/src/index.ts +++ b/packages/db-collection-e2e/src/index.ts @@ -26,4 +26,4 @@ export { createCollationTestSuite } from './suites/collation.suite' export { createMutationsTestSuite } from './suites/mutations.suite' export { createLiveUpdatesTestSuite } from './suites/live-updates.suite' export { createProgressiveTestSuite } from './suites/progressive.suite' -export { createMovesTestSuite as createTagsTestSuite } from './suites/moves.suite' +export { createMovesTestSuite } from './suites/moves.suite' diff --git a/packages/electric-db-collection/e2e/electric.e2e.test.ts b/packages/electric-db-collection/e2e/electric.e2e.test.ts index 395a52163..cb41e26da 100644 --- a/packages/electric-db-collection/e2e/electric.e2e.test.ts +++ b/packages/electric-db-collection/e2e/electric.e2e.test.ts @@ -13,11 +13,11 @@ import { createDeduplicationTestSuite, createJoinsTestSuite, createLiveUpdatesTestSuite, + createMovesTestSuite, createMutationsTestSuite, createPaginationTestSuite, createPredicatesTestSuite, createProgressiveTestSuite, - createTagsTestSuite, generateSeedData, } from '../../db-collection-e2e/src/index' import { waitFor } from '../../db-collection-e2e/src/utils/helpers' @@ -594,5 +594,5 @@ describe(`Electric Collection E2E Tests`, () => { createMutationsTestSuite(getConfig) createLiveUpdatesTestSuite(getConfig) createProgressiveTestSuite(getConfig) - createTagsTestSuite(getConfig as any) + createMovesTestSuite(getConfig as any) }) From 89eec5be5d5e9477e34b4ba6d7d813770d8f902c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 16:04:23 +0100 Subject: [PATCH 21/26] Run e2e test suite for moves on all sync modes --- .../src/suites/moves.suite.ts | 1164 +++++++++-------- 1 file changed, 601 insertions(+), 563 deletions(-) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 348592a0d..340828984 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -25,6 +25,8 @@ interface TagsTestConfig extends E2ETestConfig { } } +type SyncMode = 'eager' | 'on-demand' | 'progressive' + export function createMovesTestSuite(getConfig: () => Promise) { describe(`Moves Suite`, () => { let usersTable: string @@ -48,14 +50,17 @@ export function createMovesTestSuite(getConfig: () => Promise) { // This creates a shape: posts WHERE userId IN (SELECT id FROM users WHERE isActive = true) // When a user's isActive changes, posts will move in/out of this shape function createPostsByActiveUsersCollection( - id: string = `tags-posts-active-users-${Date.now()}`, + syncMode: SyncMode, + id?: string, ): Collection { // Remove quotes from table names for the WHERE clause SQL const usersTableUnquoted = usersTable.replace(/"/g, ``) + const collectionId = + id || `tags-posts-active-users-${syncMode}-${Date.now()}` return createCollection( electricCollectionOptions({ - id, + id: collectionId, shapeOptions: { url: `${baseUrl}/v1/shape`, params: { @@ -66,9 +71,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { where: `"userId" IN (SELECT id FROM ${testSchema}.${usersTableUnquoted} WHERE "isActive" = true)`, }, }, - syncMode: `eager`, + syncMode, getKey: (item: any) => item.id, - startSync: true, + startSync: syncMode !== 'progressive', }), ) as any } @@ -76,7 +81,12 @@ export function createMovesTestSuite(getConfig: () => Promise) { // Helper to wait for collection to be ready async function waitForReady( collection: Collection, + syncMode: SyncMode, ) { + if (syncMode === 'progressive') { + // For progressive mode, start sync explicitly + collection.startSyncImmediate() + } await collection.preload() await waitFor(() => collection.status === `ready`, { timeout: 30000, @@ -108,566 +118,594 @@ export function createMovesTestSuite(getConfig: () => Promise) { }) } - it(`Initial snapshot contains only posts from active users`, async () => { - // Create collection on posts with WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) - const collection = createPostsByActiveUsersCollection() - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert 2 active users and 1 inactive user - const userId1 = randomUUID() - const userId2 = randomUUID() - const userId3 = randomUUID() - - await config.mutations.insertUser({ - id: userId1, - name: `Active User 1`, - email: `user1@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - await config.mutations.insertUser({ - id: userId2, - name: `Active User 2`, - email: `user2@test.com`, - age: 30, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - await config.mutations.insertUser({ - id: userId3, - name: `Inactive User`, - email: `user3@test.com`, - age: 42, - isActive: false, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert posts for these users - const postId1 = randomUUID() - const postId2 = randomUUID() - const postId3 = randomUUID() - - await config.mutations.insertPost({ - id: postId1, - userId: userId1, - title: `Post 1`, - content: `Content 1`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await config.mutations.insertPost({ - id: postId2, - userId: userId2, - title: `Post 2`, - content: `Content 2`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await config.mutations.insertPost({ - id: postId3, - userId: userId3, - title: `Post 3`, - content: `Content 3`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - // Wait for collection to sync - await waitForReady(collection) - - // Wait for both posts to appear (users are active, so posts match the subquery) - await waitForItem(collection, postId1) - await waitForItem(collection, postId2) - - // Verify only posts 1 and 2 are in the collection - expect(collection.has(postId1)).toBe(true) - expect(collection.has(postId2)).toBe(true) - expect(collection.has(postId3)).toBe(false) - - // Wait a bit to make sure post 3 is not coming in later - await new Promise((resolve) => setTimeout(resolve, 50)) - expect(collection.has(postId3)).toBe(false) - - // Note: Tags are internal to Electric and may not be directly accessible - // The test verifies that posts with matching conditions appear in snapshot - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable}`) - await dbClient.query(`DELETE FROM ${usersTable}`) - await collection.cleanup() - }) - - it(`Move-in: row becomes eligible for subquery`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = false - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Inactive User`, - email: `inactive@test.com`, - age: 25, - isActive: false, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Inactive User Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - // Wait a bit to ensure post doesn't appear (user is inactive, so post doesn't match subquery) - await new Promise((resolve) => setTimeout(resolve, 500)) - expect(collection.has(postId)).toBe(false) - - // Update user to isActive = true (move-in for the post) - await config.mutations.updateUser(userId, { isActive: true }) - - // Wait for post to appear (move-in) - await waitForItem(collection, postId, 1000) - expect(collection.has(postId)).toBe(true) - expect(collection.get(postId)?.title).toBe(`Inactive User Post`) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`Move-out: row becomes ineligible for subquery`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Active User`, - email: `active@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Active User Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - // Wait for post to appear (user is active, so post matches subquery) - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Update user to isActive = false (move-out for the post) - await config.mutations.updateUser(userId, { isActive: false }) - - // Wait for post to be removed (move-out) - await waitForItemRemoved(collection, postId) - expect(collection.has(postId)).toBe(false) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`Move-out → move-in cycle ("flapping row")`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Flapping User`, - email: `flapping@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Flapping Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Move-out: isActive = false - await config.mutations.updateUser(userId, { isActive: false }) - await waitForItemRemoved(collection, postId, 15000) - expect(collection.has(postId)).toBe(false) - - // Move-in: isActive = true - await config.mutations.updateUser(userId, { isActive: true }) - await waitForItem(collection, postId, 15000) - expect(collection.has(postId)).toBe(true) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`Tags-only update (row stays within subquery)`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Active User`, - email: `active@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Tagged Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Update post title (tags might change but post stays in subquery since user is still active) - await dbClient.query( - `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, - [`Updated Tagged Post`, postId], - ) - - // Wait a bit and verify post still exists - await new Promise((resolve) => setTimeout(resolve, 500)) - expect(collection.has(postId)).toBe(true) - expect(collection.get(postId)?.title).toBe(`Updated Tagged Post`) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`Database DELETE leads to row being removed from collection`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Active User`, - email: `active@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `To Be Deleted`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection, postId) - expect(collection.has(postId)).toBe(true) - - // Delete post in Postgres - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - - // Wait for post to be removed - await waitForItemRemoved(collection, postId) - expect(collection.has(postId)).toBe(false) - - // Clean up - await config.mutations.deleteUser(userId) - await collection.cleanup() - }) - - it(`Snapshot after move-out should not re-include removed rows`, async () => { - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Create first collection - const collection1 = createPostsByActiveUsersCollection() - await waitForReady(collection1) - - // Insert user with isActive = true - const userId = randomUUID() - await config.mutations.insertUser({ - id: userId, - name: `Snapshot Test User`, - email: `snapshot@test.com`, - age: 25, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert post for this user - const postId = randomUUID() - await config.mutations.insertPost({ - id: postId, - userId, - title: `Snapshot Test Post`, - content: `Content`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await waitForItem(collection1, postId) - expect(collection1.has(postId)).toBe(true) - - // Update user → post moves out - await config.mutations.updateUser(userId, { isActive: false }) - - await waitForItemRemoved(collection1, postId) - expect(collection1.has(postId)).toBe(false) - - // Clean up first collection - await collection1.cleanup() - - // Create fresh collection (new subscription) - const collection2 = createPostsByActiveUsersCollection() - await waitForReady(collection2) - - // Wait a bit to ensure snapshot is complete - await new Promise((resolve) => setTimeout(resolve, 1000)) - - // Snapshot should NOT include the removed post (user is inactive) - expect(collection2.has(postId)).toBe(false) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId]) - await config.mutations.deleteUser(userId) - await collection2.cleanup() - }) - - it(`Multi-row transaction: some rows move in, some move out`, async () => { - const collection = createPostsByActiveUsersCollection() - await waitForReady(collection) - - if (!config.mutations) { - throw new Error(`Mutations not configured`) - } - - // Insert 3 users all with isActive = true - const userId1 = randomUUID() - const userId2 = randomUUID() - const userId3 = randomUUID() - - await config.mutations.insertUser({ - id: userId1, - name: `User 1`, - email: `user1@test.com`, - age: 25, - isActive: false, - createdAt: new Date(), - metadata: null, - deletedAt: null, + // Helper function to run all tests for a given sync mode + function runTestsForSyncMode(syncMode: SyncMode) { + describe(`${syncMode} mode`, () => { + it(`Initial snapshot contains only posts from active users`, async () => { + // Create collection on posts with WHERE clause: userId IN (SELECT id FROM users WHERE isActive = true) + const collection = createPostsByActiveUsersCollection(syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert 2 active users and 1 inactive user + const userId1 = randomUUID() + const userId2 = randomUUID() + const userId3 = randomUUID() + + await config.mutations.insertUser({ + id: userId1, + name: `Active User 1`, + email: `user1@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId2, + name: `Active User 2`, + email: `user2@test.com`, + age: 30, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId3, + name: `Inactive User`, + email: `user3@test.com`, + age: 42, + isActive: false, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert posts for these users + const postId1 = randomUUID() + const postId2 = randomUUID() + const postId3 = randomUUID() + + await config.mutations.insertPost({ + id: postId1, + userId: userId1, + title: `Post 1`, + content: `Content 1`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId2, + userId: userId2, + title: `Post 2`, + content: `Content 2`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId3, + userId: userId3, + title: `Post 3`, + content: `Content 3`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for collection to sync + await waitForReady(collection, syncMode) + + // Wait for both posts to appear (users are active, so posts match the subquery) + await waitForItem(collection, postId1) + await waitForItem(collection, postId2) + + // Verify only posts 1 and 2 are in the collection + expect(collection.has(postId1)).toBe(true) + expect(collection.has(postId2)).toBe(true) + expect(collection.has(postId3)).toBe(false) + + // Wait a bit to make sure post 3 is not coming in later + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(collection.has(postId3)).toBe(false) + + // Note: Tags are internal to Electric and may not be directly accessible + // The test verifies that posts with matching conditions appear in snapshot + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable}`) + await dbClient.query(`DELETE FROM ${usersTable}`) + await collection.cleanup() + }) + + it(`Move-in: row becomes eligible for subquery`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = false + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Inactive User`, + email: `inactive@test.com`, + age: 25, + isActive: false, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Inactive User Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait a bit to ensure post doesn't appear (user is inactive, so post doesn't match subquery) + await new Promise((resolve) => setTimeout(resolve, 500)) + expect(collection.has(postId)).toBe(false) + + // Update user to isActive = true (move-in for the post) + await config.mutations.updateUser(userId, { isActive: true }) + + // Wait for post to appear (move-in) + await waitForItem(collection, postId, 1000) + expect(collection.has(postId)).toBe(true) + expect(collection.get(postId)?.title).toBe(`Inactive User Post`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`Move-out: row becomes ineligible for subquery`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Active User Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for post to appear (user is active, so post matches subquery) + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update user to isActive = false (move-out for the post) + await config.mutations.updateUser(userId, { isActive: false }) + + // Wait for post to be removed (move-out) + await waitForItemRemoved(collection, postId) + expect(collection.has(postId)).toBe(false) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`Move-out → move-in cycle`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Flapping User`, + email: `flapping@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Flapping Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Move-out: isActive = false + await config.mutations.updateUser(userId, { isActive: false }) + await waitForItemRemoved(collection, postId, 15000) + expect(collection.has(postId)).toBe(false) + + // Move-in: isActive = true + await config.mutations.updateUser(userId, { isActive: true }) + await waitForItem(collection, postId, 15000) + expect(collection.has(postId)).toBe(true) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`Tags-only update (row stays within subquery)`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Tagged Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Update post title (tags might change but post stays in subquery since user is still active) + await dbClient.query( + `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, + [`Updated Tagged Post`, postId], + ) + + // Wait a bit and verify post still exists + await new Promise((resolve) => setTimeout(resolve, 500)) + expect(collection.has(postId)).toBe(true) + expect(collection.get(postId)?.title).toBe(`Updated Tagged Post`) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`Database DELETE leads to row being removed from collection`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Active User`, + email: `active@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `To Be Deleted`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection, postId) + expect(collection.has(postId)).toBe(true) + + // Delete post in Postgres + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + + // Wait for post to be removed + await waitForItemRemoved(collection, postId) + expect(collection.has(postId)).toBe(false) + + // Clean up + await config.mutations.deleteUser(userId) + await collection.cleanup() + }) + + it(`Snapshot after move-out should not re-include removed rows`, async () => { + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Create first collection + const collection1 = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection1, syncMode) + + // Insert user with isActive = true + const userId = randomUUID() + await config.mutations.insertUser({ + id: userId, + name: `Snapshot Test User`, + email: `snapshot@test.com`, + age: 25, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert post for this user + const postId = randomUUID() + await config.mutations.insertPost({ + id: postId, + userId, + title: `Snapshot Test Post`, + content: `Content`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await waitForItem(collection1, postId) + expect(collection1.has(postId)).toBe(true) + + // Update user → post moves out + await config.mutations.updateUser(userId, { isActive: false }) + + await waitForItemRemoved(collection1, postId) + expect(collection1.has(postId)).toBe(false) + + // Clean up first collection + await collection1.cleanup() + + // Create fresh collection (new subscription) + const collection2 = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection2, syncMode) + + // Wait a bit to ensure snapshot is complete + await new Promise((resolve) => setTimeout(resolve, 1000)) + + // Snapshot should NOT include the removed post (user is inactive) + expect(collection2.has(postId)).toBe(false) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId, + ]) + await config.mutations.deleteUser(userId) + await collection2.cleanup() + }) + + it(`Multi-row transaction: some rows move in, some move out`, async () => { + const collection = createPostsByActiveUsersCollection(syncMode) + await waitForReady(collection, syncMode) + + if (!config.mutations) { + throw new Error(`Mutations not configured`) + } + + // Insert 3 users all with isActive = true + const userId1 = randomUUID() + const userId2 = randomUUID() + const userId3 = randomUUID() + + await config.mutations.insertUser({ + id: userId1, + name: `User 1`, + email: `user1@test.com`, + age: 25, + isActive: false, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId2, + name: `User 2`, + email: `user2@test.com`, + age: 30, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + await config.mutations.insertUser({ + id: userId3, + name: `User 3`, + email: `user3@test.com`, + age: 35, + isActive: true, + createdAt: new Date(), + metadata: null, + deletedAt: null, + }) + + // Insert posts for these users + const postId1 = randomUUID() + const postId2 = randomUUID() + const postId3 = randomUUID() + + await config.mutations.insertPost({ + id: postId1, + userId: userId1, + title: `Post 1`, + content: `Content 1`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId2, + userId: userId2, + title: `Post 2`, + content: `Content 2`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + await config.mutations.insertPost({ + id: postId3, + userId: userId3, + title: `Post 3`, + content: `Content 3`, + viewCount: 0, + largeViewCount: BigInt(0), + publishedAt: null, + deletedAt: null, + }) + + // Wait for posts 2 and 3 to appear + await waitForItem(collection, postId2) + await waitForItem(collection, postId3) + + expect(collection.has(postId1)).toBe(false) + + // In one SQL transaction: + // user1: isActive → true (post1 moves in) + // post2: title change (stays in since user2 is still active) + // user3: isActive → false (post3 moves out) + await dbClient.query(`BEGIN`) + try { + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [true, userId1], + ) + await dbClient.query( + `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, + [`Updated Post 2`, postId2], + ) + await dbClient.query( + `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, + [false, userId3], + ) + await dbClient.query(`COMMIT`) + } catch (error) { + await dbClient.query(`ROLLBACK`) + throw error + } + + // Wait for changes to propagate + await waitForItemRemoved(collection, postId3) + expect(collection.has(postId1)).toBe(true) // post1: moved in (user1 active) + expect(collection.has(postId2)).toBe(true) // post2: still in (user2 active) + expect(collection.get(postId2)?.title).toBe(`Updated Post 2`) + expect(collection.has(postId3)).toBe(false) // post3: moved out (user3 inactive) + + // Clean up + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId1, + ]) + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId2, + ]) + await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [ + postId3, + ]) + await config.mutations.deleteUser(userId1) + await config.mutations.deleteUser(userId2) + await config.mutations.deleteUser(userId3) + await collection.cleanup() + }) }) + } - await config.mutations.insertUser({ - id: userId2, - name: `User 2`, - email: `user2@test.com`, - age: 30, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - await config.mutations.insertUser({ - id: userId3, - name: `User 3`, - email: `user3@test.com`, - age: 35, - isActive: true, - createdAt: new Date(), - metadata: null, - deletedAt: null, - }) - - // Insert posts for these users - const postId1 = randomUUID() - const postId2 = randomUUID() - const postId3 = randomUUID() - - await config.mutations.insertPost({ - id: postId1, - userId: userId1, - title: `Post 1`, - content: `Content 1`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await config.mutations.insertPost({ - id: postId2, - userId: userId2, - title: `Post 2`, - content: `Content 2`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - await config.mutations.insertPost({ - id: postId3, - userId: userId3, - title: `Post 3`, - content: `Content 3`, - viewCount: 0, - largeViewCount: BigInt(0), - publishedAt: null, - deletedAt: null, - }) - - // Wait for posts 2 and 3 to appear - await waitForItem(collection, postId2) - await waitForItem(collection, postId3) - - expect(collection.has(postId1)).toBe(false) - - // In one SQL transaction: - // user1: isActive → true (post1 moves in) - // post2: title change (stays in since user2 is still active) - // user3: isActive → false (post3 moves out) - await dbClient.query(`BEGIN`) - try { - await dbClient.query( - `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [true, userId1], - ) - await dbClient.query( - `UPDATE ${postsTable} SET title = $1 WHERE id = $2`, - [`Updated Post 2`, postId2], - ) - await dbClient.query( - `UPDATE ${usersTable} SET "isActive" = $1 WHERE id = $2`, - [false, userId3], - ) - await dbClient.query(`COMMIT`) - } catch (error) { - await dbClient.query(`ROLLBACK`) - throw error - } - - // Wait for changes to propagate - await waitForItemRemoved(collection, postId3) - expect(collection.has(postId1)).toBe(true) // post1: moved in (user1 active) - expect(collection.has(postId2)).toBe(true) // post2: still in (user2 active) - expect(collection.get(postId2)?.title).toBe(`Updated Post 2`) - expect(collection.has(postId3)).toBe(false) // post3: moved out (user3 inactive) - - // Clean up - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId1]) - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId2]) - await dbClient.query(`DELETE FROM ${postsTable} WHERE id = $1`, [postId3]) - await config.mutations.deleteUser(userId1) - await config.mutations.deleteUser(userId2) - await config.mutations.deleteUser(userId3) - await collection.cleanup() - }) + // Run tests for each sync mode + runTestsForSyncMode('eager') + runTestsForSyncMode('on-demand') + runTestsForSyncMode('progressive') }) } From 3f8379c2a0edc93716fc0f350c8b1d7a5c94ce06 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 16:12:19 +0100 Subject: [PATCH 22/26] Add some sleep time to reduce flakiness --- packages/db-collection-e2e/src/suites/moves.suite.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 340828984..839b98a72 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -680,6 +680,7 @@ export function createMovesTestSuite(getConfig: () => Promise) { // Wait for changes to propagate await waitForItemRemoved(collection, postId3) + await new Promise((resolve) => setTimeout(resolve, 1000)) expect(collection.has(postId1)).toBe(true) // post1: moved in (user1 active) expect(collection.has(postId2)).toBe(true) // post2: still in (user2 active) expect(collection.get(postId2)?.title).toBe(`Updated Post 2`) From 094537a6c72704758e3323ffe341f3e6f0896093 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 16:26:52 +0100 Subject: [PATCH 23/26] Add some extra wait time to ensure users exist before creating posts --- packages/db-collection-e2e/src/suites/moves.suite.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 839b98a72..6ed6a70d0 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -610,6 +610,8 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + // Insert posts for these users const postId1 = randomUUID() const postId2 = randomUUID() From 6de102c7041230602153461b3fa145a67b24e198 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 8 Dec 2025 16:38:05 +0100 Subject: [PATCH 24/26] Await users before inserting posts --- .../src/suites/moves.suite.ts | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/packages/db-collection-e2e/src/suites/moves.suite.ts b/packages/db-collection-e2e/src/suites/moves.suite.ts index 6ed6a70d0..cfc7fda40 100644 --- a/packages/db-collection-e2e/src/suites/moves.suite.ts +++ b/packages/db-collection-e2e/src/suites/moves.suite.ts @@ -118,6 +118,24 @@ export function createMovesTestSuite(getConfig: () => Promise) { }) } + // Helper to wait for users to be synced to Electric/TanStack DB + async function waitForUsersSynced( + userIds: Array, + timeout: number = 10000, + ) { + // Use eager collection since it continuously syncs all data + const usersCollection = config.collections.eager.users + await waitFor( + () => { + return userIds.every((userId) => usersCollection.has(userId)) + }, + { + timeout, + message: `Users ${userIds.join(', ')} did not sync to collection`, + }, + ) + } + // Helper function to run all tests for a given sync mode function runTestsForSyncMode(syncMode: SyncMode) { describe(`${syncMode} mode`, () => { @@ -167,6 +185,10 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for all 3 users to be synced to Electric before inserting posts + // This ensures the subquery in the WHERE clause can properly evaluate + await waitForUsersSynced([userId1, userId2, userId3]) + // Insert posts for these users const postId1 = randomUUID() const postId2 = randomUUID() @@ -251,6 +273,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -305,6 +330,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -358,6 +386,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -413,6 +444,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -469,6 +503,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -521,6 +558,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) + // Wait for user to be synced to Electric before inserting post + await waitForUsersSynced([userId]) + // Insert post for this user const postId = randomUUID() await config.mutations.insertPost({ @@ -610,7 +650,9 @@ export function createMovesTestSuite(getConfig: () => Promise) { deletedAt: null, }) - await new Promise((resolve) => setTimeout(resolve, 1000)) + // Wait for all 3 users to be synced to Electric before inserting posts + // This ensures the subquery in the WHERE clause can properly evaluate + await waitForUsersSynced([userId1, userId2, userId3]) // Insert posts for these users const postId1 = randomUUID() From 85a4e4bd1464f9ff724894ab4ab9dadb6234948c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 11 Dec 2025 14:45:26 +0100 Subject: [PATCH 25/26] Delete a row by key on a move out such that we don't have to materialize the collection's state when processing a transaction. --- .../electric-db-collection/src/electric.ts | 95 ++++--------------- 1 file changed, 18 insertions(+), 77 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 9ac715ca0..477e37641 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -31,8 +31,7 @@ import type { } from './tag-index' import type { BaseCollectionConfig, - ChangeMessage, - Collection, + ChangeMessageOrDeleteKeyMessage, CollectionConfig, DeleteMutationFnParams, InsertMutationFnParams, @@ -1085,11 +1084,9 @@ function createElectricSync>( */ const processMoveOutEvent = ( patterns: Array, - collection: Collection, begin: () => void, - write: (message: Omit, `key`>) => void, + write: (message: ChangeMessageOrDeleteKeyMessage) => void, transactionStarted: boolean, - transactionState: Map, ): boolean => { if (tagLength === undefined) { debug( @@ -1113,16 +1110,10 @@ function createElectricSync>( txStarted = true } - // Get row value from transaction state (uncommitted) or collection - const rowValue = transactionState.get(rowId) ?? collection.get(rowId) - if (rowValue !== undefined) { - write({ - type: `delete`, - value: rowValue, - }) - // Remove from transaction state since we're deleting it - transactionState.delete(rowId) - } + write({ + type: `delete`, + key: rowId, + }) } } } @@ -1242,10 +1233,6 @@ function createElectricSync>( syncMode === `progressive` && !hasReceivedUpToDate const bufferedMessages: Array> = [] // Buffer change messages during initial sync - // Track row state during the current transaction to access uncommitted row values - // This allows us to handle partial updates correctly by merging with existing state - const transactionState = new Map() - /** * Process a change message: handle tags and write the mutation */ @@ -1262,60 +1249,20 @@ function createElectricSync>( const rowId = collection.getKeyFromItem(changeMessage.value) const operation = changeMessage.headers.operation - if (operation === `insert`) { - // For insert, store the full row in transaction state - transactionState.set(rowId, changeMessage.value) - - if (hasTags) { - processTagsForChangeMessage(tags, removedTags, rowId) - } - - write({ - type: `insert`, - value: changeMessage.value, - metadata: { - ...changeMessage.headers, - }, - }) - } else if (operation === `update`) { - // For update, merge with existing state (from transaction state or collection) - const existingValue = - transactionState.get(rowId) ?? collection.get(rowId) - - // Merge the update with existing value (handles partial updates) - const updatedValue = - existingValue !== undefined - ? Object.assign({}, existingValue, changeMessage.value) - : changeMessage.value - - // Store the merged result in transaction state - transactionState.set(rowId, updatedValue) - - if (hasTags) { - processTagsForChangeMessage(tags, removedTags, rowId) - } - - write({ - type: `update`, - value: updatedValue, - metadata: { - ...changeMessage.headers, - }, - }) - } else { - // Operation is delete + if (operation === `delete`) { clearTagsForRow(rowId) - // Remove from transaction state - transactionState.delete(rowId) - - write({ - type: `delete`, - value: changeMessage.value, - metadata: { - ...changeMessage.headers, - }, - }) + } else if (hasTags) { + processTagsForChangeMessage(tags, removedTags, rowId) } + + write({ + type: changeMessage.headers.operation, + value: changeMessage.value, + // Include the primary key and relation info in the metadata + metadata: { + ...changeMessage.headers, + }, + }) } // Create deduplicated loadSubset wrapper for non-eager modes @@ -1424,11 +1371,9 @@ function createElectricSync>( // Normal processing: process move-out immediately transactionStarted = processMoveOutEvent( message.headers.patterns, - collection, begin, write, transactionStarted, - transactionState, ) } } else if (isMustRefetchMessage(message)) { @@ -1492,18 +1437,15 @@ function createElectricSync>( // Process buffered move-out messages during atomic swap processMoveOutEvent( bufferedMsg.headers.patterns, - collection, begin, write, transactionStarted, - transactionState, ) } } // Commit the atomic swap commit() - transactionState.clear() // Exit buffering phase by marking that we've received up-to-date // isBufferingInitialSync() will now return false @@ -1518,7 +1460,6 @@ function createElectricSync>( if (transactionStarted) { commit() transactionStarted = false - transactionState.clear() } } wrappedMarkReady(isBufferingInitialSync()) From 83a8afc50529f63afe6b998c8a5629a859261ecf Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 17 Dec 2025 12:01:58 +0000 Subject: [PATCH 26/26] ci: apply automated fixes --- .changeset/witty-animals-agree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/witty-animals-agree.md b/.changeset/witty-animals-agree.md index be350c75e..846144348 100644 --- a/.changeset/witty-animals-agree.md +++ b/.changeset/witty-animals-agree.md @@ -1,5 +1,5 @@ --- -"@tanstack/electric-db-collection": patch +'@tanstack/electric-db-collection': patch --- Support tagged rows and move out events in Electric collection.