diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 9ef0637..234e7c9 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -62,7 +62,7 @@ jobs: "package/src/Storage.js" "package/src/Partition.js" "package/src/Index.js" - "package/src/metadataUtil.js" + "package/src/utils/metadataUtil.js" ) for f in "${REQUIRED[@]}"; do echo "$FILES" | grep -qF "$f" || { echo "ERROR: required file '$f' is missing from the package"; exit 1; } diff --git a/package.json b/package.json index f989889..8cfaffd 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,8 @@ "src/Partition/*.js", "src/Storage/*.js", "src/WatchesFile.js", - "src/util.js", - "src/fsUtil.js", - "src/metadataUtil.js", - "index.js" + "index.js", + "src/utils/*.js" ], "license": "MIT", "maintainers": [ diff --git a/src/Consumer.js b/src/Consumer.js index bb223a7..9b2e813 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -1,8 +1,8 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; -import { assert } from './util.js'; -import { ensureDirectory } from './fsUtil.js'; +import { assert } from './utils/util.js'; +import { ensureDirectory } from './utils/fsUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; diff --git a/src/EventStore.js b/src/EventStore.js index 53852eb..80ae74e 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -6,9 +6,9 @@ import events from 'events'; import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js'; import Index from './Index.js'; import Consumer from './Consumer.js'; -import { assert, getPropertyAtPath } from './util.js'; -import { ensureDirectory, scanForFiles } from './fsUtil.js'; -import { buildTypeMatcherFn } from './metadataUtil.js'; +import { assert, getPropertyAtPath } from './utils/util.js'; +import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; +import { buildTypeMatcherFn } from './utils/metadataUtil.js'; const ExpectedVersion = { Any: -1, @@ -20,6 +20,7 @@ const ExpectedVersion = { */ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type']; const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; +const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']); class OptimisticConcurrencyError extends Error {} @@ -240,11 +241,8 @@ class EventStore extends events.EventEmitter { * @returns {this} */ on(event, listener) { - if (event === 'preCommit' || event === 'preRead') { - if (event === 'preCommit') { - assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); - } - this.storage.on(event, listener); + if (this.isStorageHookEvent(event)) { + this.delegateStorageHookEvent('on', event, listener); return this; } return super.on(event, listener); @@ -265,11 +263,8 @@ class EventStore extends events.EventEmitter { * @returns {this} */ once(event, listener) { - if (event === 'preCommit' || event === 'preRead') { - if (event === 'preCommit') { - assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); - } - this.storage.once(event, listener); + if (this.isStorageHookEvent(event)) { + this.delegateStorageHookEvent('once', event, listener); return this; } return super.once(event, listener); @@ -284,13 +279,24 @@ class EventStore extends events.EventEmitter { * @returns {this} */ off(event, listener) { - if (event === 'preCommit' || event === 'preRead') { + if (this.isStorageHookEvent(event)) { this.storage.off(event, listener); return this; } return super.off(event, listener); } + isStorageHookEvent(event) { + return STORAGE_HOOK_EVENTS.has(event); + } + + delegateStorageHookEvent(method, event, listener) { + if (event === 'preCommit') { + assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.'); + } + this.storage[method](event, listener); + } + /** * @inheritDoc */ @@ -429,6 +435,20 @@ class EventStore extends events.EventEmitter { return type; } + getExistingQueryTypes(types) { + const queryTypes = []; + for (const type of types) { + if (type in this.streams) { + queryTypes.push(type); + continue; + } + if (!this.typeAccessor) { + throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`); + } + } + return queryTypes; + } + /** * Commit a list of events for the given stream name, which is expected to be at the given version. * Note that the events committed may still appear in other streams too - the given stream name is only @@ -543,20 +563,7 @@ class EventStore extends events.EventEmitter { */ query(types, matcher = null, minRevision = 1, raw = false) { assert(Array.isArray(types) && types.length > 0, 'Must specify a non-empty array of event types for query.'); - - const queryTypes = []; - for (const type of types) { - if (!(type in this.streams)) { - // No typeAccessor: the stream was never created; we cannot know whether events of - // this type exist in the store, so throw to avoid an unintentional full-store scan. - assert(!!this.typeAccessor, `Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`); - // typeAccessor is configured: type streams are created on commit, so a missing - // stream simply means no event of this type has been committed yet — treat as empty. - continue; - } - queryTypes.push(type); - } - + const queryTypes = this.getExistingQueryTypes(types); const condition = new CommitCondition(types, matcher, this.storage.length, raw); const stream = this.fromStreams('_query_' + types.join('_'), queryTypes, minRevision, -1, matcher, raw); return { stream, condition }; @@ -574,10 +581,7 @@ class EventStore extends events.EventEmitter { * @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist. */ getEventStream(streamName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); if (!(streamName in this.streams)) { return false; } @@ -596,10 +600,7 @@ class EventStore extends events.EventEmitter { * @returns {EventStream} The event stream. */ getAllEvents(minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); return this.getEventStream('_all', minRevision, maxRevision, predicate, raw); } @@ -616,10 +617,7 @@ class EventStore extends events.EventEmitter { * @throws {Error} if any of the streams doesn't exist. */ fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); assert(streamNames instanceof Array, 'Must specify an array of stream names.'); if (streamNames.length === 0) { @@ -658,10 +656,7 @@ class EventStore extends events.EventEmitter { * @throws {Error} If no stream for this category exists. */ getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) { - if (typeof predicate === 'boolean' && raw === false) { - raw = predicate; - predicate = null; - } + ({ predicate, raw } = normalizePredicateRaw(predicate, raw)); if (categoryName in this.streams) { return this.getEventStream(categoryName, minRevision, maxRevision, predicate, raw); } @@ -844,6 +839,13 @@ function parseStreamFromIndexName(indexName) { return indexName; } +function normalizePredicateRaw(predicate, raw) { + if (typeof predicate === 'boolean' && raw === false) { + return { predicate: null, raw: predicate }; + } + return { predicate, raw }; +} + EventStore.Storage = Storage; EventStore.Index = Index; diff --git a/src/EventStream.js b/src/EventStream.js index e12cce8..69b8715 100644 --- a/src/EventStream.js +++ b/src/EventStream.js @@ -1,6 +1,6 @@ import stream from 'stream'; -import { assert } from './util.js'; -import { buildRawBufferMatcher, matches } from './metadataUtil.js'; +import { assert } from './utils/util.js'; +import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js'; const NDJSON_NEWLINE = Buffer.from('\n'); diff --git a/src/Index/ReadableIndex.js b/src/Index/ReadableIndex.js index 538b29a..dea919e 100644 --- a/src/Index/ReadableIndex.js +++ b/src/Index/ReadableIndex.js @@ -2,7 +2,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; import Entry, { assertValidEntryClass } from '../IndexEntry.js'; -import { assert, wrapAndCheck, binarySearch } from '../util.js'; +import { assert, wrapAndCheck, binarySearch } from '../utils/util.js'; // node-event-store-index V01 const HEADER_MAGIC = "nesidx01"; diff --git a/src/Index/WritableIndex.js b/src/Index/WritableIndex.js index 6eb8447..6cc08e6 100644 --- a/src/Index/WritableIndex.js +++ b/src/Index/WritableIndex.js @@ -1,8 +1,8 @@ import fs from 'fs'; import ReadableIndex, { Entry, CorruptedIndexError, HEADER_MAGIC } from './ReadableIndex.js'; -import { assert, assertEqual } from '../util.js'; -import { buildMetadataHeader } from '../metadataUtil.js'; -import { ensureDirectory } from '../fsUtil.js'; +import { assert, assertEqual } from '../utils/util.js'; +import { buildMetadataHeader } from '../utils/metadataUtil.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; /** * An index is a simple append-only file that stores an ordered list of entry elements pointing to the actual file position diff --git a/src/IndexMatcher.js b/src/IndexMatcher.js index d940245..13ab3a4 100644 --- a/src/IndexMatcher.js +++ b/src/IndexMatcher.js @@ -1,5 +1,5 @@ -import { getPropertyAtPath } from './util.js'; -import { matches } from './metadataUtil.js'; +import { getPropertyAtPath } from './utils/util.js'; +import { matches } from './utils/metadataUtil.js'; /** * @typedef {object|function(object):boolean} Matcher diff --git a/src/JoinEventStream.js b/src/JoinEventStream.js index 2f569c6..0b61fc9 100644 --- a/src/JoinEventStream.js +++ b/src/JoinEventStream.js @@ -1,5 +1,5 @@ import EventStream from './EventStream.js'; -import {assert, kWayMerge} from './util.js'; +import { assert, kWayMerge } from './utils/util.js'; /** Reusable sentinel used for missing or empty per-stream iterators. */ const emptyIterator = Object.freeze({ next() { return { done: true }; } }); diff --git a/src/Partition/ReadablePartition.js b/src/Partition/ReadablePartition.js index 4c7dff1..b2ee3be 100644 --- a/src/Partition/ReadablePartition.js +++ b/src/Partition/ReadablePartition.js @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; -import { assert, alignTo, hash, binarySearch } from '../util.js'; +import { assert, alignTo, hash, binarySearch } from '../utils/util.js'; @@ -209,6 +209,30 @@ class ReadablePartition extends events.EventEmitter { return ({ dataSize, sequenceNumber, time64 }); } + resolveIterationPosition(position) { + return position < 0 ? this.size + position + 1 : position; + } + + selectReader(position, size, backwardsHint) { + if (size > 0 && backwardsHint) { + const bufferOffset = DOCUMENT_HEADER_SIZE + size; + const reader = this.prepareReadBufferBackwards(position + bufferOffset, bufferOffset); + return { reader, bufferOffset }; + } + return { reader: this.prepareReadBuffer(position), bufferOffset: 0 }; + } + + assignHeaderOutput(headerOut, header) { + if (headerOut === null) { + return; + } + headerOut.dataSize = header.dataSize; + headerOut.sequenceNumber = header.sequenceNumber; + // Denormalize time64 relative to this partition's epoch so callers can compare + // timestamps across partitions without needing to know the epoch value. + headerOut.time64 = this.metadata.epoch + header.time64; + } + /** * Read the data from the given position. * @@ -227,10 +251,7 @@ class ReadablePartition extends events.EventEmitter { assert(this.fd, 'Partition is not opened.'); assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position ${position}. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`); - const bufferOffset = size > 0 && backwardsHint ? DOCUMENT_HEADER_SIZE + size : 0; - const reader = size > 0 && backwardsHint - ? this.prepareReadBufferBackwards(position + bufferOffset, bufferOffset) - : this.prepareReadBuffer(position); + const { reader, bufferOffset } = this.selectReader(position, size, backwardsHint); if (reader.length < DOCUMENT_HEADER_SIZE) { return false; } @@ -242,13 +263,7 @@ class ReadablePartition extends events.EventEmitter { let dataPosition = reader.cursor + DOCUMENT_HEADER_SIZE; const header = this.readDocumentHeader(reader.buffer, reader.cursor, position, size); const dataSize = header.dataSize; - if (headerOut !== null) { - headerOut.dataSize = header.dataSize; - headerOut.sequenceNumber = header.sequenceNumber; - // Denormalize time64 relative to this partition's epoch so callers can compare - // timestamps across partitions without needing to know the epoch value. - headerOut.time64 = this.metadata.epoch + header.time64; - } + this.assignHeaderOutput(headerOut, header); const writeSize = this.documentWriteSize(dataSize); assert(position + writeSize <= this.size, `Invalid document at position ${position}. This may be caused by an unfinished write.`, CorruptFileError); @@ -437,7 +452,7 @@ class ReadablePartition extends events.EventEmitter { * @returns {Generator} A generator that returns all documents in this partition. */ *readAll(after = 0, headerOut = null) { - let position = after < 0 ? this.size + after + 1 : after; + let position = this.resolveIterationPosition(after); const internalHeader = headerOut !== null ? headerOut : {}; let data; while ((data = this.readFrom(position, 0, internalHeader)) !== false) { @@ -457,7 +472,7 @@ class ReadablePartition extends events.EventEmitter { * @returns {Generator} A generator that returns all documents in this partition in reverse order. */ *readAllBackwards(before = -1, headerOut = null) { - let position = before < 0 ? this.size + before + 1 : before; + let position = this.resolveIterationPosition(before); const internalHeader = headerOut !== null ? headerOut : {}; while ((position = this.findDocumentPositionBefore(position)) !== false) { const data = this.readFrom(position, 0, internalHeader, true); diff --git a/src/Partition/WritablePartition.js b/src/Partition/WritablePartition.js index 503b9bf..92a8b2e 100644 --- a/src/Partition/WritablePartition.js +++ b/src/Partition/WritablePartition.js @@ -1,8 +1,8 @@ import fs from 'fs'; import ReadablePartition, { CorruptFileError, HEADER_MAGIC, DOCUMENT_ALIGNMENT, DOCUMENT_SEPARATOR, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE } from './ReadablePartition.js'; -import { assert, alignTo } from '../util.js'; -import { buildMetadataHeader } from '../metadataUtil.js'; -import { ensureDirectory } from '../fsUtil.js'; +import { assert, alignTo } from '../utils/util.js'; +import { buildMetadataHeader } from '../utils/metadataUtil.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; import Clock from '../Clock.js'; const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024; @@ -177,12 +177,7 @@ class WritablePartition extends ReadablePartition { * @returns {number} The size of the document header */ writeDocumentHeader(buffer, offset, dataSize, sequenceNumber = null, time64 = null) { - if (sequenceNumber === null) { - sequenceNumber = 0; - } - if (time64 === null) { - time64 = this.clock.time(); - } + ({ sequenceNumber, time64 } = this.normalizeWriteMetadata(sequenceNumber, time64)); /* istanbul ignore if */ assert(time64 >= 0, 'Time may not be negative!'); @@ -192,6 +187,24 @@ class WritablePartition extends ReadablePartition { return DOCUMENT_HEADER_SIZE; } + normalizeWriteMetadata(sequenceNumber, time64) { + return { + sequenceNumber: sequenceNumber === null ? 0 : sequenceNumber, + time64: time64 === null ? this.clock.time() : time64 + }; + } + + normalizeWriteArguments(sequenceNumber, callback) { + if (typeof sequenceNumber === 'function') { + return { sequenceNumber: null, callback: sequenceNumber }; + } + return { sequenceNumber, callback }; + } + + shouldWriteUnbuffered(dataSize) { + return dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5; + } + /** * Write the given data to the partition without buffering. * @private @@ -258,15 +271,12 @@ class WritablePartition extends ReadablePartition { */ write(data, sequenceNumber, callback) { assert(this.fd, 'Partition is not opened.'); - if (typeof sequenceNumber === 'function') { - callback = sequenceNumber; - sequenceNumber = null; - } + ({ sequenceNumber, callback } = this.normalizeWriteArguments(sequenceNumber, callback)); const dataSize = Buffer.byteLength(data, 'utf8'); assert(dataSize <= 64 * 1024 * 1024, 'Document is too large! Maximum is 64 MB'); const dataPosition = this.size; - if (dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5) { + if (this.shouldWriteUnbuffered(dataSize)) { this.size += this.writeUnbuffered(data, dataSize, sequenceNumber, callback); } else { this.size += this.writeBuffered(data, dataSize, sequenceNumber, callback); diff --git a/src/Storage/ReadableStorage.js b/src/Storage/ReadableStorage.js index 443fe42..31dc392 100644 --- a/src/Storage/ReadableStorage.js +++ b/src/Storage/ReadableStorage.js @@ -3,9 +3,9 @@ import path from 'path'; import events from 'events'; import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js'; import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js'; -import { assert, wrapAndCheck, iterate, kWayMerge } from '../util.js'; -import { scanForFiles } from '../fsUtil.js'; -import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js'; +import { assert, wrapAndCheck, iterate, kWayMerge } from '../utils/util.js'; +import { scanForFiles } from '../utils/fsUtil.js'; +import { createHmac, matches, buildMetadataForMatcher } from '../utils/metadataUtil.js'; import IndexMatcher from '../IndexMatcher.js'; import PartitionPool from '../PartitionPool.js'; diff --git a/src/Storage/WritableStorage.js b/src/Storage/WritableStorage.js index 384c2fa..ed3d784 100644 --- a/src/Storage/WritableStorage.js +++ b/src/Storage/WritableStorage.js @@ -3,9 +3,9 @@ import path from 'path'; import WritablePartition from '../Partition/WritablePartition.js'; import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js'; import ReadableStorage from './ReadableStorage.js'; -import { assert } from '../util.js'; -import { ensureDirectory } from '../fsUtil.js'; -import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js'; +import { assert } from '../utils/util.js'; +import { ensureDirectory } from '../utils/fsUtil.js'; +import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../utils/metadataUtil.js'; const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024; @@ -311,6 +311,36 @@ class WritableStorage extends ReadableStorage { this.on('preCommit', hook); } + getPartitionIdForName(partitionShortName, partitionName) { + const partitionId = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName); + this.partitionIds[partitionShortName] = partitionId; + return partitionId; + } + + buildPartitionConfig(partitionShortName) { + if (typeof this.partitionConfig.metadata !== 'function') { + return this.partitionConfig; + } + return { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }; + } + + ensurePartitionDirectory(partitionName) { + if (!partitionName.includes('/')) { + return; + } + ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName))); + } + + createNamedPartition(partitionId, partitionName, partitionShortName) { + if (this.partitions.has(partitionId)) { + return; + } + const partitionConfig = this.buildPartitionConfig(partitionShortName); + this.ensurePartitionDirectory(partitionName); + this.partitions.add(partitionId, this.createPartition(partitionName, partitionConfig)); + this.emit('partition-created', partitionId); + } + /** * Get a partition either by name or by id. * If a partition with the given name does not exist, a new one will be created. @@ -327,18 +357,8 @@ class WritableStorage extends ReadableStorage { if (typeof partitionIdentifier === 'string') { const partitionShortName = partitionIdentifier; const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : ''); - partitionIdentifier = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName); - this.partitionIds[partitionShortName] = partitionIdentifier; - if (!this.partitions.has(partitionIdentifier)) { - const partitionConfig = typeof this.partitionConfig.metadata === 'function' - ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) } - : this.partitionConfig; - if (partitionName.includes('/')) { - ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName))); - } - this.partitions.add(partitionIdentifier, this.createPartition(partitionName, partitionConfig)); - this.emit('partition-created', partitionIdentifier); - } + partitionIdentifier = this.getPartitionIdForName(partitionShortName, partitionName); + this.createNamedPartition(partitionIdentifier, partitionName, partitionShortName); } return super.getPartition(partitionIdentifier); } @@ -363,9 +383,7 @@ class WritableStorage extends ReadableStorage { const indexEntry = this.addIndex(partition.id, position, dataSize, document); this.forEachSecondaryIndex((index, name) => { - if (!index.isOpen()) { - index.open(); - } + index.open(); index.add(indexEntry); this.emit('index-add', name, index.length, document); }, document); diff --git a/src/Watcher.js b/src/Watcher.js index 320e577..8bca521 100644 --- a/src/Watcher.js +++ b/src/Watcher.js @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import events from 'events'; -import { assert } from './util.js'; +import { assert } from './utils/util.js'; /** @type {Map} */ const directoryWatchers = new Map(); diff --git a/src/fsUtil.js b/src/utils/fsUtil.js similarity index 100% rename from src/fsUtil.js rename to src/utils/fsUtil.js diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js new file mode 100644 index 0000000..a68455b --- /dev/null +++ b/src/utils/jsonUtil.js @@ -0,0 +1,87 @@ +const BYTE_QUOTE = 0x22; +const BYTE_ESCAPE = 0x5c; +const BYTE_OPEN_OBJECT = 0x7b; +const BYTE_CLOSE_OBJECT = 0x7d; +const BYTE_OPEN_ARRAY = 0x5b; +const BYTE_CLOSE_ARRAY = 0x5d; +const BYTE_COMMA = 0x2c; + +/** + * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting + * from `startOffset`. It scans character-by-character tracking JSON nesting depth and string + * quoting. If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested + * object/array, so the scan continues searching for the next candidate at depth 0. Returns -1 + * when no such position exists before the end of the buffer or when a closing brace reduces depth + * below zero (the top-level object has ended). + */ +function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { + /* c8 ignore start */ + // Defensive fallback: public call path precomputes an initial candidate in preCheck. + if (matchPosition === undefined) { + matchPosition = buffer.indexOf(pattern, startOffset); + } + if (matchPosition === -1) { + return -1; + } + /* c8 ignore stop */ + + let depth = 0; + let inString = false; + let i = startOffset; + + while (i < buffer.length) { + if (inString) { + if (buffer[i] === BYTE_ESCAPE) { // '\\' + i += 2; + continue; + } + if (buffer[i] === BYTE_QUOTE) { // '"' + inString = false; + } + i++; + continue; + } + + const ch = buffer[i]; + if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '[' + depth++; + i++; + continue; + } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']' + depth--; + + if (depth < 0) { + return -1; + } + + i++; + continue; + } else if (ch === BYTE_QUOTE) { // '"' + inString = true; + } + + if (i >= matchPosition) { + if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"' + const end = i + pattern.length; + if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{' + return i; + } + if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']' + return i; + } + } + + matchPosition = buffer.indexOf(pattern, matchPosition + 1); + if (matchPosition < 0) { + return -1; + } + } + + i++; + } + + /* c8 ignore next */ + return -1; +} + +export { BYTE_OPEN_OBJECT, indexOfSameLevel }; diff --git a/src/metadataUtil.js b/src/utils/metadataUtil.js similarity index 59% rename from src/metadataUtil.js rename to src/utils/metadataUtil.js index f73f62e..fa089fc 100644 --- a/src/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -1,5 +1,19 @@ import crypto from 'crypto'; -import {assert, assertEqual} from './util.js'; +import { assert, assertEqual } from './util.js'; +import { BYTE_OPEN_OBJECT, indexOfSameLevel } from './jsonUtil.js'; + +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function propertyMatchesValue(documentValue, matcherValue) { + if (Array.isArray(matcherValue)) { + return matcherValue.includes(documentValue); + } else if (matcherValue && typeof matcherValue === 'object') { + return matches(documentValue, matcherValue); + } + return typeof matcherValue === 'undefined' || documentValue === matcherValue; +} /** * Build a buffer containing the file magic header and a JSON stringified metadata block, padded to be a multiple of 16 bytes long. @@ -49,15 +63,7 @@ function matches(document, matcher) { if (typeof matcher === 'function') return matcher(document); for (let prop of Object.getOwnPropertyNames(matcher)) { - if (Array.isArray(matcher[prop])) { - if (!matcher[prop].includes(document[prop])) { - return false; - } - } else if (typeof matcher[prop] === 'object') { - if (!matches(document[prop], matcher[prop])) { - return false; - } - } else if (typeof matcher[prop] !== 'undefined' && document[prop] !== matcher[prop]) { + if (!propertyMatchesValue(document[prop], matcher[prop])) { return false; } } @@ -132,7 +138,7 @@ function buildRawBufferMatcher(matcher = {}) { } return function matchesRawBuffer(buffer) { - if (buffer[0] !== 0x7b) { // '{' + if (buffer[0] !== BYTE_OPEN_OBJECT) { return false; } if (!preCheck(buffer, 1, root)) { @@ -148,15 +154,19 @@ function buildRawBufferMatcher(matcher = {}) { */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { - let i = 0; - if (child.valuePatterns && !child.valuePatterns.some(pattern => (child.valueMatches[i++] = buffer.indexOf(pattern, startOffset)) !== -1)) { + if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { + child.valueMatches[i] = buffer.indexOf(pattern, startOffset); + return child.valueMatches[i] !== -1; + })) { return false; } if (child.objectPattern) { - if ((child.objMatch = buffer.indexOf(child.objectPattern, startOffset)) === -1) { + const objectMatch = buffer.indexOf(child.objectPattern, startOffset); + if (objectMatch === -1) { return false; } - if (!preCheck(buffer, child.objMatch, child.node)) { + child.objMatch = objectMatch; + if (!preCheck(buffer, objectMatch, child.node)) { return false; } } @@ -172,24 +182,32 @@ function buildMatcherTree(matcher) { const node = { children: [] }; for (const [key, value] of Object.entries(matcher)) { - const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); - const child = { objectPattern: null, valuePatterns: null, node: null, objMatch: null, valueMatches: [] }; + node.children.push(buildMatcherTreeChild(key, value)); + } - if (Array.isArray(value)) { - assert(!value.some(item => item && typeof item === 'object'), 'Array matcher values must be scalars.', TypeError); + return node; +} - child.valuePatterns = value.map(item => Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(item), 'utf8')])); - } else if (value && typeof value === 'object') { - child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value); - } else { - child.valuePatterns = [Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')])]; +function buildMatcherTreeChild(key, value) { + const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); + const child = { objectPattern: null, valuePatterns: null, node: null, objMatch: null, valueMatches: [] }; + if (Array.isArray(value)) { + if (value.some(item => item && typeof item === 'object')) { + throw new TypeError('Array matcher values must be scalars.'); } - - node.children.push(child); + child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); + return child; + } else if (value && typeof value === 'object') { + child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); + child.node = buildMatcherTree(value); + return child; } + child.valuePatterns = [buildValuePattern(keyPrefix, value)]; + return child; +} - return node; +function buildValuePattern(keyPrefix, value) { + return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); } /** @@ -198,11 +216,10 @@ function buildMatcherTree(matcher) { */ function matchesNode(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns) { - let i = 0; - if (!child.valuePatterns.some(pattern => indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i++]) !== -1)) { - return false; - } + if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { + return indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1; + })) { + return false; } if (child.node) { @@ -219,84 +236,6 @@ function matchesNode(buffer, startOffset, node) { return true; } -/** - * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting - * from `startOffset`. It scans character-by-character tracking JSON nesting depth and string - * quoting. If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested - * object/array, so the scan continues searching for the next candidate at depth 0. Returns -1 - * when no such position exists before the end of the buffer or when a closing brace reduces depth - * below zero (the top-level object has ended). - */ -function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) { - /* c8 ignore start */ - // Defensive fallback: public call path precomputes an initial candidate in preCheck. - if (matchPosition === undefined) { - matchPosition = buffer.indexOf(pattern, startOffset); - } - if (matchPosition === -1) { - return -1; - } - /* c8 ignore stop */ - - let depth = 0; - let inString = false; - let i = startOffset; - - while (i < buffer.length) { - if (inString) { - if (buffer[i] === 0x5c) { // '\\' - i += 2; - continue; - } - if (buffer[i] === 0x22) { // '"' - inString = false; - } - i++; - continue; - } - - const ch = buffer[i]; - if (ch === 0x7b || ch === 0x5b) { // '{' or '[' - depth++; - i++; - continue; - } else if (ch === 0x7d || ch === 0x5d) { // '}' or ']' - depth--; - - if (depth < 0) { - return -1; - } - - i++; - continue; - } else if (ch === 0x22) { // '"' - inString = true; - } - - if (i >= matchPosition) { - if (i === matchPosition && ch === 0x22 && depth === 0) { // '"' - const end = i + pattern.length; - if (pattern[pattern.length - 1] === 0x7b) { // '{' - return i; - } - if (buffer[end] === 0x2c || buffer[end] === 0x7d || buffer[end] === 0x5d) { // ',' or '}' or ']' - return i; - } - } - - matchPosition = buffer.indexOf(pattern, matchPosition + 1); - if (matchPosition < 0) { - return -1; - } - } - - i++; - } - - /* c8 ignore next */ - return -1; -} - export { createHmac, matches, diff --git a/src/util.js b/src/utils/util.js similarity index 100% rename from src/util.js rename to src/utils/util.js diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index a24ba67..9ba0a04 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -1,5 +1,5 @@ import expect from 'expect.js'; -import { buildRawBufferMatcher } from '../src/metadataUtil.js'; +import { buildRawBufferMatcher } from '../src/utils/metadataUtil.js'; describe('metadataUtil', function() { @@ -132,4 +132,3 @@ describe('metadataUtil', function() { }); - diff --git a/test/util.spec.js b/test/util.spec.js index 9f0a232..b9adc01 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -1,8 +1,8 @@ import expect from 'expect.js'; import fs from 'fs-extra'; import path from 'path'; -import { iterate } from '../src/util.js'; -import { scanForFiles } from '../src/fsUtil.js'; +import { iterate } from '../src/utils/util.js'; +import { scanForFiles } from '../src/utils/fsUtil.js'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url));