From 7c56a626a395766a1b4baa0f069f804b9619f2f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 11:29:17 +0000 Subject: [PATCH 1/8] Refactor hotspot complexity and move utilities to src/utils --- .github/workflows/npm-publish.yml | 2 +- package.json | 6 +- src/Consumer.js | 4 +- src/EventStore.js | 92 ++++++++++++------------ src/EventStream.js | 4 +- src/Index/ReadableIndex.js | 2 +- src/Index/WritableIndex.js | 6 +- src/IndexMatcher.js | 4 +- src/JoinEventStream.js | 2 +- src/Partition/ReadablePartition.js | 43 +++++++---- src/Partition/WritablePartition.js | 38 ++++++---- src/Storage/ReadableStorage.js | 6 +- src/Storage/WritableStorage.js | 68 ++++++++++++------ src/Watcher.js | 2 +- src/{ => utils}/fsUtil.js | 0 src/{ => utils}/metadataUtil.js | 112 +++++++++++++++++++---------- src/{ => utils}/util.js | 0 test/metadataUtil.spec.js | 3 +- test/util.spec.js | 4 +- 19 files changed, 242 insertions(+), 156 deletions(-) rename src/{ => utils}/fsUtil.js (100%) rename src/{ => utils}/metadataUtil.js (73%) rename src/{ => utils}/util.js (100%) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 9ef06378..234e7c92 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -62,7 +62,7 @@ jobs: "package/src/Storage.js" "package/src/Partition.js" "package/src/Index.js" - "package/src/metadataUtil.js" + "package/src/utils/metadataUtil.js" ) for f in "${REQUIRED[@]}"; do echo "$FILES" | grep -qF "$f" || { echo "ERROR: required file '$f' is missing from the package"; exit 1; } diff --git a/package.json b/package.json index f989889d..8cfaffdb 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,8 @@ "src/Partition/*.js", "src/Storage/*.js", "src/WatchesFile.js", - "src/util.js", - "src/fsUtil.js", - "src/metadataUtil.js", - "index.js" + "index.js", + "src/utils/*.js" ], "license": "MIT", "maintainers": [ diff --git a/src/Consumer.js b/src/Consumer.js index bb223a73..9b2e813d 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -1,8 +1,8 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; -import { assert } from './util.js'; -import { ensureDirectory } from './fsUtil.js'; +import { assert } from './utils/util.js'; +import { ensureDirectory } from './utils/fsUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; diff --git a/src/EventStore.js b/src/EventStore.js index 855f72e2..93d233ba 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -6,9 +6,9 @@ import events from 'events'; import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js'; import Index from './Index.js'; import Consumer from './Consumer.js'; -import { assert, getPropertyAtPath } from './util.js'; -import { ensureDirectory, scanForFiles } from './fsUtil.js'; -import { buildTypeMatcherFn } from './metadataUtil.js'; +import { assert, getPropertyAtPath } from './utils/util.js'; +import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; +import { buildTypeMatcherFn } from './utils/metadataUtil.js'; const ExpectedVersion = { Any: -1, @@ -20,6 +20,7 @@ const ExpectedVersion = { */ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type']; const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; +const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']); class OptimisticConcurrencyError extends Error {} @@ -240,11 +241,8 @@ class EventStore extends events.EventEmitter { * @returns {this} */ on(event, listener) { - if (event === 'preCommit' || event === 'preRead') { - if (event === 'preCommit') { - assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); - } - this.storage.on(event, listener); + if (this.isStorageHookEvent(event)) { + this.delegateStorageHookEvent('on', event, listener); return this; } return super.on(event, listener); @@ -265,11 +263,8 @@ class EventStore extends events.EventEmitter { * @returns {this} */ once(event, listener) { - if (event === 'preCommit' || event === 'preRead') { - if (event === 'preCommit') { - assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); - } - this.storage.once(event, listener); + if (this.isStorageHookEvent(event)) { + this.delegateStorageHookEvent('once', event, listener); return this; } return super.once(event, listener); @@ -284,13 +279,24 @@ class EventStore extends events.EventEmitter { * @returns {this} */ off(event, listener) { - if (event === 'preCommit' || event === 'preRead') { + if (this.isStorageHookEvent(event)) { this.storage.off(event, listener); return this; } return super.off(event, listener); } + isStorageHookEvent(event) { + return STORAGE_HOOK_EVENTS.has(event); + } + + delegateStorageHookEvent(method, event, listener) { + if (event === 'preCommit') { + assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); + } + this.storage[method](event, listener); + } + /** * @inheritDoc */ @@ -433,6 +439,20 @@ class EventStore extends events.EventEmitter { return type; } + getExistingQueryTypes(types) { + const queryTypes = []; + for (const type of types) { + if (type in this.streams) { + queryTypes.push(type); + continue; + } + if (!this.typeAccessor) { + throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`); + } + } + return queryTypes; + } + /** * Commit a list of events for the given stream name, which is expected to be at the given version. * Note that the events committed may still appear in other streams too - the given stream name is only @@ -546,22 +566,7 @@ class EventStore extends events.EventEmitter { */ query(types, matcher = null, minRevision = 1, raw = false) { assert(Array.isArray(types) && types.length > 0, 'Must specify a non-empty array of event types for query.'); - - const queryTypes = []; - for (const type of types) { - if (!(type in this.streams)) { - if (this.typeAccessor) { - // typeAccessor is configured: type streams are created on commit, so a missing - // stream simply means no event of this type has been committed yet — treat as empty. - continue; - } - // No typeAccessor: the stream was never created; we cannot know whether events of - // this type exist in the store, so throw to avoid an unintentional full-store scan. - throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`); - } - queryTypes.push(type); - } - + const queryTypes = this.getExistingQueryTypes(types); const condition = new CommitCondition(types, matcher, this.storage.length, raw); const stream = this.fromStreams('_query_' + types.join('_'), queryTypes, minRevision, -1, matcher, raw); return { stream, condition }; @@ -579,10 +584,7 @@ class EventStore extends events.EventEmitter { * @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist. */ getEventStream(streamName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); if (!(streamName in this.streams)) { return false; } @@ -601,10 +603,7 @@ class EventStore extends events.EventEmitter { * @returns {EventStream} The event stream. */ getAllEvents(minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); return this.getEventStream('_all', minRevision, maxRevision, predicate, raw); } @@ -621,10 +620,7 @@ class EventStore extends events.EventEmitter { * @throws {Error} if any of the streams doesn't exist. */ fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); assert(streamNames instanceof Array, 'Must specify an array of stream names.'); if (streamNames.length === 0) { @@ -663,10 +659,7 @@ class EventStore extends events.EventEmitter { * @throws {Error} If no stream for this category exists. */ getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); if (categoryName in this.streams) { return this.getEventStream(categoryName, minRevision, maxRevision, predicate, raw); } @@ -850,6 +843,13 @@ function parseStreamFromIndexName(indexName) { return indexName; } +function normalizePredicateRaw(predicate, raw) { + if (typeof predicate === 'boolean' && raw === false) { + return { predicate: null, raw: predicate }; + } + return { predicate, raw }; +} + EventStore.Storage = Storage; EventStore.Index = Index; diff --git a/src/EventStream.js b/src/EventStream.js index e12cce8d..69b8715b 100644 --- a/src/EventStream.js +++ b/src/EventStream.js @@ -1,6 +1,6 @@ import stream from 'stream'; -import { assert } from './util.js'; -import { buildRawBufferMatcher, matches } from './metadataUtil.js'; +import { assert } from './utils/util.js'; +import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js'; const NDJSON_NEWLINE = Buffer.from('\n'); diff --git a/src/Index/ReadableIndex.js b/src/Index/ReadableIndex.js index 979ec381..678ebbdd 100644 --- a/src/Index/ReadableIndex.js +++ b/src/Index/ReadableIndex.js @@ -2,7 +2,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; import Entry, { assertValidEntryClass } from '../IndexEntry.js'; -import { assert, wrapAndCheck, binarySearch } from '../util.js'; +import { assert, wrapAndCheck, binarySearch } from '../utils/util.js'; // node-event-store-index V01 const HEADER_MAGIC = "nesidx01"; diff --git a/src/Index/WritableIndex.js b/src/Index/WritableIndex.js index ed2e5659..283fecbd 100644 --- a/src/Index/WritableIndex.js +++ b/src/Index/WritableIndex.js @@ -1,8 +1,8 @@ import fs from 'fs'; import ReadableIndex, { Entry, CorruptedIndexError, HEADER_MAGIC } from './ReadableIndex.js'; -import { assertEqual } from '../util.js'; -import { buildMetadataHeader } from '../metadataUtil.js'; -import { ensureDirectory } from '../fsUtil.js'; +import { assertEqual } from '../utils/util.js'; +import { buildMetadataHeader } from '../utils/metadataUtil.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; /** * An index is a simple append-only file that stores an ordered list of entry elements pointing to the actual file position diff --git a/src/IndexMatcher.js b/src/IndexMatcher.js index d9402459..13ab3a4f 100644 --- a/src/IndexMatcher.js +++ b/src/IndexMatcher.js @@ -1,5 +1,5 @@ -import { getPropertyAtPath } from './util.js'; -import { matches } from './metadataUtil.js'; +import { getPropertyAtPath } from './utils/util.js'; +import { matches } from './utils/metadataUtil.js'; /** * @typedef {object|function(object):boolean} Matcher diff --git a/src/JoinEventStream.js b/src/JoinEventStream.js index fe590924..bdd35ac7 100644 --- a/src/JoinEventStream.js +++ b/src/JoinEventStream.js @@ -1,5 +1,5 @@ import EventStream from './EventStream.js'; -import { kWayMerge } from './util.js'; +import { kWayMerge } from './utils/util.js'; /** Reusable sentinel used for missing or empty per-stream iterators. */ const emptyIterator = Object.freeze({ next() { return { done: true }; } }); diff --git a/src/Partition/ReadablePartition.js b/src/Partition/ReadablePartition.js index caf53491..15a5711e 100644 --- a/src/Partition/ReadablePartition.js +++ b/src/Partition/ReadablePartition.js @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; -import { assert, alignTo, hash, binarySearch } from '../util.js'; +import { assert, alignTo, hash, binarySearch } from '../utils/util.js'; @@ -211,6 +211,30 @@ class ReadablePartition extends events.EventEmitter { return ({ dataSize, sequenceNumber, time64 }); } + resolveIterationPosition(position) { + return position < 0 ? this.size + position + 1 : position; + } + + selectReader(position, size, backwardsHint) { + if (size > 0 && backwardsHint) { + const bufferOffset = DOCUMENT_HEADER_SIZE + size; + const reader = this.prepareReadBufferBackwards(position + bufferOffset, bufferOffset); + return { reader, bufferOffset }; + } + return { reader: this.prepareReadBuffer(position), bufferOffset: 0 }; + } + + assignHeaderOutput(headerOut, header) { + if (headerOut === null) { + return; + } + headerOut.dataSize = header.dataSize; + headerOut.sequenceNumber = header.sequenceNumber; + // Denormalize time64 relative to this partition's epoch so callers can compare + // timestamps across partitions without needing to know the epoch value. + headerOut.time64 = this.metadata.epoch + header.time64; + } + /** * Read the data from the given position. * @@ -229,10 +253,7 @@ class ReadablePartition extends events.EventEmitter { assert(this.fd, 'Partition is not opened.'); assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position ${position}. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`); - const bufferOffset = size > 0 && backwardsHint ? DOCUMENT_HEADER_SIZE + size : 0; - const reader = size > 0 && backwardsHint - ? this.prepareReadBufferBackwards(position + bufferOffset, bufferOffset) - : this.prepareReadBuffer(position); + const { reader, bufferOffset } = this.selectReader(position, size, backwardsHint); if (reader.length < DOCUMENT_HEADER_SIZE) { return false; } @@ -244,13 +265,7 @@ class ReadablePartition extends events.EventEmitter { let dataPosition = reader.cursor + DOCUMENT_HEADER_SIZE; const header = this.readDocumentHeader(reader.buffer, reader.cursor, position, size); const dataSize = header.dataSize; - if (headerOut !== null) { - headerOut.dataSize = header.dataSize; - headerOut.sequenceNumber = header.sequenceNumber; - // Denormalize time64 relative to this partition's epoch so callers can compare - // timestamps across partitions without needing to know the epoch value. - headerOut.time64 = this.metadata.epoch + header.time64; - } + this.assignHeaderOutput(headerOut, header); const writeSize = this.documentWriteSize(dataSize); if (position + writeSize > this.size) { @@ -441,7 +456,7 @@ class ReadablePartition extends events.EventEmitter { * @returns {Generator} A generator that returns all documents in this partition. */ *readAll(after = 0, headerOut = null) { - let position = after < 0 ? this.size + after + 1 : after; + let position = this.resolveIterationPosition(after); const internalHeader = headerOut !== null ? headerOut : {}; let data; while ((data = this.readFrom(position, 0, internalHeader)) !== false) { @@ -461,7 +476,7 @@ class ReadablePartition extends events.EventEmitter { * @returns {Generator} A generator that returns all documents in this partition in reverse order. */ *readAllBackwards(before = -1, headerOut = null) { - let position = before < 0 ? this.size + before + 1 : before; + let position = this.resolveIterationPosition(before); const internalHeader = headerOut !== null ? headerOut : {}; while ((position = this.findDocumentPositionBefore(position)) !== false) { const data = this.readFrom(position, 0, internalHeader, true); diff --git a/src/Partition/WritablePartition.js b/src/Partition/WritablePartition.js index 30a87418..b8d4828d 100644 --- a/src/Partition/WritablePartition.js +++ b/src/Partition/WritablePartition.js @@ -1,8 +1,8 @@ import fs from 'fs'; import ReadablePartition, { CorruptFileError, HEADER_MAGIC, DOCUMENT_ALIGNMENT, DOCUMENT_SEPARATOR, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE } from './ReadablePartition.js'; -import { assert, alignTo } from '../util.js'; -import { buildMetadataHeader } from '../metadataUtil.js'; -import { ensureDirectory } from '../fsUtil.js'; +import { assert, alignTo } from '../utils/util.js'; +import { buildMetadataHeader } from '../utils/metadataUtil.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; import Clock from '../Clock.js'; const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024; @@ -177,12 +177,7 @@ class WritablePartition extends ReadablePartition { * @returns {number} The size of the document header */ writeDocumentHeader(buffer, offset, dataSize, sequenceNumber = null, time64 = null) { - if (sequenceNumber === null) { - sequenceNumber = 0; - } - if (time64 === null) { - time64 = this.clock.time(); - } + ({ sequenceNumber, time64 } = this.normalizeWriteMetadata(sequenceNumber, time64)); /* istanbul ignore if */ if (time64 < 0) { throw new Error('Time may not be negative!'); @@ -193,6 +188,24 @@ class WritablePartition extends ReadablePartition { return DOCUMENT_HEADER_SIZE; } + normalizeWriteMetadata(sequenceNumber, time64) { + return { + sequenceNumber: sequenceNumber === null ? 0 : sequenceNumber, + time64: time64 === null ? this.clock.time() : time64 + }; + } + + normalizeWriteArguments(sequenceNumber, callback) { + if (typeof sequenceNumber === 'function') { + return { sequenceNumber: null, callback: sequenceNumber }; + } + return { sequenceNumber, callback }; + } + + shouldWriteUnbuffered(dataSize) { + return dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5; + } + /** * Write the given data to the partition without buffering. * @private @@ -259,15 +272,12 @@ class WritablePartition extends ReadablePartition { */ write(data, sequenceNumber, callback) { assert(this.fd, 'Partition is not opened.'); - if (typeof sequenceNumber === 'function') { - callback = sequenceNumber; - sequenceNumber = null; - } + ({ sequenceNumber, callback } = this.normalizeWriteArguments(sequenceNumber, callback)); const dataSize = Buffer.byteLength(data, 'utf8'); assert(dataSize <= 64 * 1024 * 1024, 'Document is too large! Maximum is 64 MB'); const dataPosition = this.size; - if (dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5) { + if (this.shouldWriteUnbuffered(dataSize)) { this.size += this.writeUnbuffered(data, dataSize, sequenceNumber, callback); } else { this.size += this.writeBuffered(data, dataSize, sequenceNumber, callback); diff --git a/src/Storage/ReadableStorage.js b/src/Storage/ReadableStorage.js index 443fe42a..31dc3922 100644 --- a/src/Storage/ReadableStorage.js +++ b/src/Storage/ReadableStorage.js @@ -3,9 +3,9 @@ import path from 'path'; import events from 'events'; import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js'; import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js'; -import { assert, wrapAndCheck, iterate, kWayMerge } from '../util.js'; -import { scanForFiles } from '../fsUtil.js'; -import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js'; +import { assert, wrapAndCheck, iterate, kWayMerge } from '../utils/util.js'; +import { scanForFiles } from '../utils/fsUtil.js'; +import { createHmac, matches, buildMetadataForMatcher } from '../utils/metadataUtil.js'; import IndexMatcher from '../IndexMatcher.js'; import PartitionPool from '../PartitionPool.js'; diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index dbdd41be..9f642710 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -3,9 +3,9 @@ import path from 'path'; import WritablePartition from '../Partition/WritablePartition.js'; import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js'; import ReadableStorage from './ReadableStorage.js'; -import { assert } from '../util.js'; -import { ensureDirectory } from '../fsUtil.js'; -import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js'; +import { assert } from '../utils/util.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; +import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../utils/metadataUtil.js'; const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024; @@ -312,6 +312,36 @@ class WritableStorage extends ReadableStorage { this.on('preCommit', hook); } + getPartitionIdForName(partitionShortName, partitionName) { + const partitionId = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName); + this.partitionIds[partitionShortName] = partitionId; + return partitionId; + } + + buildPartitionConfig(partitionShortName) { + if (typeof this.partitionConfig.metadata !== 'function') { + return this.partitionConfig; + } + return { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }; + } + + ensurePartitionDirectory(partitionName) { + if (!partitionName.includes('/')) { + return; + } + ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName))); + } + + createNamedPartition(partitionId, partitionName, partitionShortName) { + if (this.partitions.has(partitionId)) { + return; + } + const partitionConfig = this.buildPartitionConfig(partitionShortName); + this.ensurePartitionDirectory(partitionName); + this.partitions.add(partitionId, this.createPartition(partitionName, partitionConfig)); + this.emit('partition-created', partitionId); + } + /** * Get a partition either by name or by id. * If a partition with the given name does not exist, a new one will be created. @@ -328,22 +358,22 @@ class WritableStorage extends ReadableStorage { if (typeof partitionIdentifier === 'string') { const partitionShortName = partitionIdentifier; const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : ''); - partitionIdentifier = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName); - this.partitionIds[partitionShortName] = partitionIdentifier; - if (!this.partitions.has(partitionIdentifier)) { - const partitionConfig = typeof this.partitionConfig.metadata === 'function' - ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) } - : this.partitionConfig; - if (partitionName.includes('/')) { - ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName))); - } - this.partitions.add(partitionIdentifier, this.createPartition(partitionName, partitionConfig)); - this.emit('partition-created', partitionIdentifier); - } + partitionIdentifier = this.getPartitionIdForName(partitionShortName, partitionName); + this.createNamedPartition(partitionIdentifier, partitionName, partitionShortName); } return super.getPartition(partitionIdentifier); } + addToSecondaryIndexes(indexEntry, document) { + this.forEachSecondaryIndex((index, name) => { + if (!index.isOpen()) { + index.open(); + } + index.add(indexEntry); + this.emit('index-add', name, index.length, document); + }, document); + } + /** * @api * @param {object} document The document to write to storage. @@ -363,13 +393,7 @@ class WritableStorage extends ReadableStorage { assert(position !== false, 'Error writing document.'); const indexEntry = this.addIndex(partition.id, position, dataSize, document); - this.forEachSecondaryIndex((index, name) => { - if (!index.isOpen()) { - index.open(); - } - index.add(indexEntry); - this.emit('index-add', name, index.length, document); - }, document); + this.addToSecondaryIndexes(indexEntry, document); return this.index.length; } diff --git a/src/Watcher.js b/src/Watcher.js index 320e577c..8bca521e 100644 --- a/src/Watcher.js +++ b/src/Watcher.js @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; -import { assert } from './util.js'; +import { assert } from './utils/util.js'; /** @type {Map} */ const directoryWatchers = new Map(); diff --git a/src/fsUtil.js b/src/utils/fsUtil.js similarity index 100% rename from src/fsUtil.js rename to src/utils/fsUtil.js diff --git a/src/metadataUtil.js b/src/utils/metadataUtil.js similarity index 73% rename from src/metadataUtil.js rename to src/utils/metadataUtil.js index 5d43aa3b..dcb57747 100644 --- a/src/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -1,6 +1,28 @@ import crypto from 'crypto'; import { assertEqual } from './util.js'; +const BYTE_QUOTE = 0x22; +const BYTE_ESCAPE = 0x5c; +const BYTE_OPEN_OBJECT = 0x7b; +const BYTE_CLOSE_OBJECT = 0x7d; +const BYTE_OPEN_ARRAY = 0x5b; +const BYTE_CLOSE_ARRAY = 0x5d; +const BYTE_COMMA = 0x2c; + +function isNonArrayObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function matchesProperty(documentValue, matcherValue) { + if (Array.isArray(matcherValue)) { + return matcherValue.includes(documentValue); + } + if (isNonArrayObject(matcherValue)) { + return matches(documentValue, matcherValue); + } + return typeof matcherValue === 'undefined' || documentValue === matcherValue; +} + /** * Build a buffer containing the file magic header and a JSON stringified metadata block, padded to be a multiple of 16 bytes long. * @@ -49,15 +71,7 @@ function matches(document, matcher) { if (typeof matcher === 'function') return matcher(document); for (let prop of Object.getOwnPropertyNames(matcher)) { - if (Array.isArray(matcher[prop])) { - if (!matcher[prop].includes(document[prop])) { - return false; - } - } else if (typeof matcher[prop] === 'object') { - if (!matches(document[prop], matcher[prop])) { - return false; - } - } else if (typeof matcher[prop] !== 'undefined' && document[prop] !== matcher[prop]) { + if (!matchesProperty(document[prop], matcher[prop])) { return false; } } @@ -151,8 +165,7 @@ function buildRawBufferMatcher(matcher = {}) { */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { - let i = 0; - if (child.valuePatterns && !child.valuePatterns.some(pattern => (child.valueMatches[i++] = buffer.indexOf(pattern, startOffset)) !== -1)) { + if (child.valuePatterns && !findPatternOffset(buffer, startOffset, child.valuePatterns, child.valueMatches)) { return false; } if (child.objectPattern) { @@ -175,25 +188,33 @@ function buildMatcherTree(matcher) { const node = { children: [] }; for (const [key, value] of Object.entries(matcher)) { - const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); - const child = { objectPattern: null, valuePatterns: null, node: null, objMatch: null, valueMatches: [] }; + node.children.push(buildMatcherTreeChild(key, value)); + } - if (Array.isArray(value)) { - if (value.some(item => item && typeof item === 'object')) { - throw new TypeError('Array matcher values must be scalars.'); - } - child.valuePatterns = value.map(item => Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(item), 'utf8')])); - } else if (value && typeof value === 'object') { - child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value); - } else { - child.valuePatterns = [Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')])]; - } + return node; +} - node.children.push(child); +function buildMatcherTreeChild(key, value) { + const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); + const child = { objectPattern: null, valuePatterns: null, node: null, objMatch: null, valueMatches: [] }; + if (Array.isArray(value)) { + if (value.some(item => item && typeof item === 'object')) { + throw new TypeError('Array matcher values must be scalars.'); + } + child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); + return child; } + if (isNonArrayObject(value)) { + child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); + child.node = buildMatcherTree(value); + return child; + } + child.valuePatterns = [buildValuePattern(keyPrefix, value)]; + return child; +} - return node; +function buildValuePattern(keyPrefix, value) { + return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); } /** @@ -203,8 +224,7 @@ function buildMatcherTree(matcher) { function matchesNode(buffer, startOffset, node) { for (const child of node.children) { if (child.valuePatterns) { - let i = 0; - if (!child.valuePatterns.some(pattern => indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i++]) !== -1)) { + if (!findSameLevelPatternOffset(buffer, startOffset, child.valuePatterns, child.valueMatches)) { return false; } } @@ -223,6 +243,26 @@ function matchesNode(buffer, startOffset, node) { return true; } +function findPatternOffset(buffer, startOffset, patterns, matchesOut) { + for (let i = 0; i < patterns.length; i++) { + const match = buffer.indexOf(patterns[i], startOffset); + matchesOut[i] = match; + if (match !== -1) { + return true; + } + } + return false; +} + +function findSameLevelPatternOffset(buffer, startOffset, patterns, preMatches) { + for (let i = 0; i < patterns.length; i++) { + if (indexOfSameLevel(buffer, patterns[i], startOffset, preMatches[i]) !== -1) { + return true; + } + } + return false; +} + /** * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting * from `startOffset`. It scans character-by-character tracking JSON nesting depth and string @@ -248,11 +288,11 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { while (i < buffer.length) { if (inString) { - if (buffer[i] === 0x5c) { // '\\' + if (buffer[i] === BYTE_ESCAPE) { // '\\' i += 2; continue; } - if (buffer[i] === 0x22) { // '"' + if (buffer[i] === BYTE_QUOTE) { // '"' inString = false; } i++; @@ -260,11 +300,11 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { } const ch = buffer[i]; - if (ch === 0x7b || ch === 0x5b) { // '{' or '[' + if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '[' depth++; i++; continue; - } else if (ch === 0x7d || ch === 0x5d) { // '}' or ']' + } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']' depth--; if (depth < 0) { @@ -273,17 +313,17 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { i++; continue; - } else if (ch === 0x22) { // '"' + } else if (ch === BYTE_QUOTE) { // '"' inString = true; } if (i >= matchPosition) { - if (i === matchPosition && ch === 0x22 && depth === 0) { // '"' + if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"' const end = i + pattern.length; - if (pattern[pattern.length - 1] === 0x7b) { // '{' + if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{' return i; } - if (buffer[end] === 0x2c || buffer[end] === 0x7d || buffer[end] === 0x5d) { // ',' or '}' or ']' + if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']' return i; } } diff --git a/src/util.js b/src/utils/util.js similarity index 100% rename from src/util.js rename to src/utils/util.js diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index a24ba679..9ba0a046 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -1,5 +1,5 @@ import expect from 'expect.js'; -import { buildRawBufferMatcher } from '../src/metadataUtil.js'; +import { buildRawBufferMatcher } from '../src/utils/metadataUtil.js'; describe('metadataUtil', function() { @@ -132,4 +132,3 @@ describe('metadataUtil', function() { }); - diff --git a/test/util.spec.js b/test/util.spec.js index 9f0a2327..b9adc01e 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -1,8 +1,8 @@ import expect from 'expect.js'; import fs from 'fs-extra'; import path from 'path'; -import { iterate } from '../src/util.js'; -import { scanForFiles } from '../src/fsUtil.js'; +import { iterate } from '../src/utils/util.js'; +import { scanForFiles } from '../src/utils/fsUtil.js'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); From ebd1bd5f747ce9069206b45b1728ea57155235f4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 11:31:37 +0000 Subject: [PATCH 2/8] Polish matcher helper extraction and finalize validation --- src/utils/metadataUtil.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index dcb57747..ca6cde13 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -9,15 +9,15 @@ const BYTE_OPEN_ARRAY = 0x5b; const BYTE_CLOSE_ARRAY = 0x5d; const BYTE_COMMA = 0x2c; -function isNonArrayObject(value) { +function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } -function matchesProperty(documentValue, matcherValue) { +function propertyMatchesValue(documentValue, matcherValue) { if (Array.isArray(matcherValue)) { return matcherValue.includes(documentValue); } - if (isNonArrayObject(matcherValue)) { + if (isPlainObject(matcherValue)) { return matches(documentValue, matcherValue); } return typeof matcherValue === 'undefined' || documentValue === matcherValue; @@ -71,7 +71,7 @@ function matches(document, matcher) { if (typeof matcher === 'function') return matcher(document); for (let prop of Object.getOwnPropertyNames(matcher)) { - if (!matchesProperty(document[prop], matcher[prop])) { + if (!propertyMatchesValue(document[prop], matcher[prop])) { return false; } } @@ -165,14 +165,16 @@ function buildRawBufferMatcher(matcher = {}) { */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns && !findPatternOffset(buffer, startOffset, child.valuePatterns, child.valueMatches)) { + if (child.valuePatterns && !populatePatternMatches(buffer, startOffset, child.valuePatterns, child.valueMatches)) { return false; } if (child.objectPattern) { - if ((child.objMatch = buffer.indexOf(child.objectPattern, startOffset)) === -1) { + const objectMatch = buffer.indexOf(child.objectPattern, startOffset); + if (objectMatch === -1) { return false; } - if (!preCheck(buffer, child.objMatch, child.node)) { + child.objMatch = objectMatch; + if (!preCheck(buffer, objectMatch, child.node)) { return false; } } @@ -204,7 +206,7 @@ function buildMatcherTreeChild(key, value) { child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); return child; } - if (isNonArrayObject(value)) { + if (isPlainObject(value)) { child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); child.node = buildMatcherTree(value); return child; @@ -243,7 +245,7 @@ function matchesNode(buffer, startOffset, node) { return true; } -function findPatternOffset(buffer, startOffset, patterns, matchesOut) { +function populatePatternMatches(buffer, startOffset, patterns, matchesOut) { for (let i = 0; i < patterns.length; i++) { const match = buffer.indexOf(patterns[i], startOffset); matchesOut[i] = match; From b06120653bd003955f820fccb7e010444b12125c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 12:20:36 +0000 Subject: [PATCH 3/8] Address PR feedback on matcher helpers and index reuse --- src/Storage/WritableStorage.js | 16 +++-- src/utils/jsonUtil.js | 87 ++++++++++++++++++++++++ src/utils/metadataUtil.js | 121 +++------------------------------ 3 files changed, 106 insertions(+), 118 deletions(-) create mode 100644 src/utils/jsonUtil.js diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index 9f642710..19eb7fb3 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -366,14 +366,18 @@ class WritableStorage extends ReadableStorage { addToSecondaryIndexes(indexEntry, document) { this.forEachSecondaryIndex((index, name) => { - if (!index.isOpen()) { - index.open(); - } - index.add(indexEntry); - this.emit('index-add', name, index.length, document); + this.addToSecondaryIndex(index, name, indexEntry, document); }, document); } + addToSecondaryIndex(index, name, indexEntry, document) { + if (!index.isOpen()) { + index.open(); + } + index.add(indexEntry); + this.emit('index-add', name, index.length, document); + } + /** * @api * @param {object} document The document to write to storage. @@ -430,7 +434,7 @@ class WritableStorage extends ReadableStorage { try { this.forEachDocument((document, indexEntry) => { if (matches(document, matcher)) { - index.add(indexEntry); + this.addToSecondaryIndex(index, name, indexEntry, document); } }); } catch (e) { diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js new file mode 100644 index 00000000..a68455b4 --- /dev/null +++ b/src/utils/jsonUtil.js @@ -0,0 +1,87 @@ +const BYTE_QUOTE = 0x22; +const BYTE_ESCAPE = 0x5c; +const BYTE_OPEN_OBJECT = 0x7b; +const BYTE_CLOSE_OBJECT = 0x7d; +const BYTE_OPEN_ARRAY = 0x5b; +const BYTE_CLOSE_ARRAY = 0x5d; +const BYTE_COMMA = 0x2c; + +/** + * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting + * from `startOffset`. It scans character-by-character tracking JSON nesting depth and string + * quoting. If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested + * object/array, so the scan continues searching for the next candidate at depth 0. Returns -1 + * when no such position exists before the end of the buffer or when a closing brace reduces depth + * below zero (the top-level object has ended). + */ +function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { + /* c8 ignore start */ + // Defensive fallback: public call path precomputes an initial candidate in preCheck. + if (matchPosition === undefined) { + matchPosition = buffer.indexOf(pattern, startOffset); + } + if (matchPosition === -1) { + return -1; + } + /* c8 ignore stop */ + + let depth = 0; + let inString = false; + let i = startOffset; + + while (i < buffer.length) { + if (inString) { + if (buffer[i] === BYTE_ESCAPE) { // '\\' + i += 2; + continue; + } + if (buffer[i] === BYTE_QUOTE) { // '"' + inString = false; + } + i++; + continue; + } + + const ch = buffer[i]; + if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '[' + depth++; + i++; + continue; + } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']' + depth--; + + if (depth < 0) { + return -1; + } + + i++; + continue; + } else if (ch === BYTE_QUOTE) { // '"' + inString = true; + } + + if (i >= matchPosition) { + if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"' + const end = i + pattern.length; + if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{' + return i; + } + if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']' + return i; + } + } + + matchPosition = buffer.indexOf(pattern, matchPosition + 1); + if (matchPosition < 0) { + return -1; + } + } + + i++; + } + + /* c8 ignore next */ + return -1; +} + +export { BYTE_OPEN_OBJECT, indexOfSameLevel }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index ca6cde13..8e14014f 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -1,13 +1,6 @@ import crypto from 'crypto'; import { assertEqual } from './util.js'; - -const BYTE_QUOTE = 0x22; -const BYTE_ESCAPE = 0x5c; -const BYTE_OPEN_OBJECT = 0x7b; -const BYTE_CLOSE_OBJECT = 0x7d; -const BYTE_OPEN_ARRAY = 0x5b; -const BYTE_CLOSE_ARRAY = 0x5d; -const BYTE_COMMA = 0x2c; +import { BYTE_OPEN_OBJECT, indexOfSameLevel } from './jsonUtil.js'; function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); @@ -149,7 +142,7 @@ function buildRawBufferMatcher(matcher = {}) { } return function matchesRawBuffer(buffer) { - if (buffer[0] !== 0x7b) { // '{' + if (buffer[0] !== BYTE_OPEN_OBJECT) { return false; } if (!preCheck(buffer, 1, root)) { @@ -165,7 +158,11 @@ function buildRawBufferMatcher(matcher = {}) { */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns && !populatePatternMatches(buffer, startOffset, child.valuePatterns, child.valueMatches)) { + if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { + const match = buffer.indexOf(pattern, startOffset); + child.valueMatches[i] = match; + return match !== -1; + })) { return false; } if (child.objectPattern) { @@ -225,10 +222,8 @@ function buildValuePattern(keyPrefix, value) { */ function matchesNode(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns) { - if (!findSameLevelPatternOffset(buffer, startOffset, child.valuePatterns, child.valueMatches)) { - return false; - } + if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1)) { + return false; } if (child.node) { @@ -245,104 +240,6 @@ function matchesNode(buffer, startOffset, node) { return true; } -function populatePatternMatches(buffer, startOffset, patterns, matchesOut) { - for (let i = 0; i < patterns.length; i++) { - const match = buffer.indexOf(patterns[i], startOffset); - matchesOut[i] = match; - if (match !== -1) { - return true; - } - } - return false; -} - -function findSameLevelPatternOffset(buffer, startOffset, patterns, preMatches) { - for (let i = 0; i < patterns.length; i++) { - if (indexOfSameLevel(buffer, patterns[i], startOffset, preMatches[i]) !== -1) { - return true; - } - } - return false; -} - -/** - * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting - * from `startOffset`. It scans character-by-character tracking JSON nesting depth and string - * quoting. If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested - * object/array, so the scan continues searching for the next candidate at depth 0. Returns -1 - * when no such position exists before the end of the buffer or when a closing brace reduces depth - * below zero (the top-level object has ended). - */ -function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { - /* c8 ignore start */ - // Defensive fallback: public call path precomputes an initial candidate in preCheck. - if (matchPosition === undefined) { - matchPosition = buffer.indexOf(pattern, startOffset); - } - if (matchPosition === -1) { - return -1; - } - /* c8 ignore stop */ - - let depth = 0; - let inString = false; - let i = startOffset; - - while (i < buffer.length) { - if (inString) { - if (buffer[i] === BYTE_ESCAPE) { // '\\' - i += 2; - continue; - } - if (buffer[i] === BYTE_QUOTE) { // '"' - inString = false; - } - i++; - continue; - } - - const ch = buffer[i]; - if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '[' - depth++; - i++; - continue; - } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']' - depth--; - - if (depth < 0) { - return -1; - } - - i++; - continue; - } else if (ch === BYTE_QUOTE) { // '"' - inString = true; - } - - if (i >= matchPosition) { - if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"' - const end = i + pattern.length; - if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{' - return i; - } - if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']' - return i; - } - } - - matchPosition = buffer.indexOf(pattern, matchPosition + 1); - if (matchPosition < 0) { - return -1; - } - } - - i++; - } - - /* c8 ignore next */ - return -1; -} - export { createHmac, matches, From f840ca47a78bfaf0b3c77fada1251881e4f73163 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 12:23:11 +0000 Subject: [PATCH 4/8] Tidy matcher predicate readability --- src/utils/metadataUtil.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 8e14014f..245e5b1c 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -222,7 +222,9 @@ function buildValuePattern(keyPrefix, value) { */ function matchesNode(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1)) { + if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { + return indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1; + })) { return false; } From f3c0c033418e13540cb804abf39074480829aee9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 13:15:31 +0000 Subject: [PATCH 5/8] Adjust secondary-index reuse and matcher object branch --- src/Storage/WritableStorage.js | 20 ++++++++++---------- src/utils/metadataUtil.js | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index 19eb7fb3..ad8dc49b 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -366,18 +366,14 @@ class WritableStorage extends ReadableStorage { addToSecondaryIndexes(indexEntry, document) { this.forEachSecondaryIndex((index, name) => { - this.addToSecondaryIndex(index, name, indexEntry, document); + if (!index.isOpen()) { + index.open(); + } + index.add(indexEntry); + this.emit('index-add', name, index.length, document); }, document); } - addToSecondaryIndex(index, name, indexEntry, document) { - if (!index.isOpen()) { - index.open(); - } - index.add(indexEntry); - this.emit('index-add', name, index.length, document); - } - /** * @api * @param {object} document The document to write to storage. @@ -434,7 +430,11 @@ class WritableStorage extends ReadableStorage { try { this.forEachDocument((document, indexEntry) => { if (matches(document, matcher)) { - this.addToSecondaryIndex(index, name, indexEntry, document); + if (!index.isOpen()) { + index.open(); + } + index.add(indexEntry); + this.emit('index-add', name, index.length, document); } }); } catch (e) { diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 245e5b1c..776778c6 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -202,8 +202,7 @@ function buildMatcherTreeChild(key, value) { } child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); return child; - } - if (isPlainObject(value)) { + } else if (value && typeof value === 'object') { child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); child.node = buildMatcherTree(value); return child; From a9498385f1c129e980763dcb161aa3ee740d0652 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 15:28:48 +0000 Subject: [PATCH 6/8] Avoid index-open/event emission during ensureIndex reindex --- src/Storage/WritableStorage.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index ad8dc49b..9f642710 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -430,11 +430,7 @@ class WritableStorage extends ReadableStorage { try { this.forEachDocument((document, indexEntry) => { if (matches(document, matcher)) { - if (!index.isOpen()) { - index.open(); - } index.add(indexEntry); - this.emit('index-add', name, index.length, document); } }); } catch (e) { From 58f64f902b512141b35307ec8e5e57fef857a31c Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Wed, 27 May 2026 19:23:47 +0200 Subject: [PATCH 7/8] Simplify code Co-authored-by: Alexander Berl --- src/utils/metadataUtil.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 279eaeb8..fa089fca 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -9,8 +9,7 @@ function isPlainObject(value) { function propertyMatchesValue(documentValue, matcherValue) { if (Array.isArray(matcherValue)) { return matcherValue.includes(documentValue); - } - if (isPlainObject(matcherValue)) { + } else if (matcherValue && typeof matcherValue === 'object') { return matches(documentValue, matcherValue); } return typeof matcherValue === 'undefined' || documentValue === matcherValue; @@ -156,9 +155,8 @@ function buildRawBufferMatcher(matcher = {}) { function preCheck(buffer, startOffset, node) { for (const child of node.children) { if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { - const match = buffer.indexOf(pattern, startOffset); - child.valueMatches[i] = match; - return match !== -1; + child.valueMatches[i] = buffer.indexOf(pattern, startOffset); + return child.valueMatches[i] !== -1; })) { return false; } From e796281d748d60d0e1807ba744caedd1de84bfda Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 17:29:11 +0000 Subject: [PATCH 8/8] Inline secondary index updates in write --- src/Storage/WritableStorage.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index 91c97e91..ed3d7845 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -363,16 +363,6 @@ class WritableStorage extends ReadableStorage { return super.getPartition(partitionIdentifier); } - addToSecondaryIndexes(indexEntry, document) { - this.forEachSecondaryIndex((index, name) => { - if (!index.isOpen()) { - index.open(); - } - index.add(indexEntry); - this.emit('index-add', name, index.length, document); - }, document); - } - /** * @api * @param {object} document The document to write to storage. @@ -392,7 +382,11 @@ class WritableStorage extends ReadableStorage { assert(position !== false, 'Error writing document.'); const indexEntry = this.addIndex(partition.id, position, dataSize, document); - this.addToSecondaryIndexes(indexEntry, document); + this.forEachSecondaryIndex((index, name) => { + index.open(); + index.add(indexEntry); + this.emit('index-add', name, index.length, document); + }, document); return this.index.length; }