diff --git a/doc/gatekeeper-api.json b/doc/gatekeeper-api.json index eb231757..57b0d393 100644 --- a/doc/gatekeeper-api.json +++ b/doc/gatekeeper-api.json @@ -1217,6 +1217,13 @@ "type": "integer", "description": "Number of events that failed validation (bad signature, size limit, etc.)." }, + "rejectedIndices": { + "type": "array", + "description": "Zero-based indexes of rejected events in the original submitted batch order (for importDIDs this is `dids.flat()` order).", + "items": { + "type": "integer" + } + }, "total": { "type": "integer", "description": "Total number of events in the queue after this import." @@ -1439,6 +1446,13 @@ "type": "integer", "description": "Number of events that failed validation." }, + "rejectedIndices": { + "type": "array", + "description": "Zero-based indexes of rejected events in the original submitted batch order.", + "items": { + "type": "integer" + } + }, "total": { "type": "integer", "description": "The total event queue size after this import." @@ -1786,6 +1800,13 @@ "pending": { "type": "integer", "description": "Number of events still left in the queue after processing." + }, + "acceptedHashes": { + "type": "array", + "description": "Lower-case signature hashes of events accepted during this processing run (added or merged).", + "items": { + "type": "string" + } } } } diff --git a/docker-compose.yml b/docker-compose.yml index 89e7e434..633eb646 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,16 @@ services: - KC_NODE_NAME=${KC_NODE_NAME} - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} + - KC_HYPR_NEGENTROPY_ENABLE=${KC_HYPR_NEGENTROPY_ENABLE} + - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} + - KC_HYPR_NEGENTROPY_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_WINDOW_DAYS} + - KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=${KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW} + - KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=${KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION} + - KC_HYPR_NEGENTROPY_INTERVAL=${KC_HYPR_NEGENTROPY_INTERVAL} + - KC_HYPR_LEGACY_SYNC_ENABLE=${KC_HYPR_LEGACY_SYNC_ENABLE} - KC_LOG_LEVEL=${KC_LOG_LEVEL} + volumes: + - ./data:/app/hyperswarm/data user: "${KC_UID}:${KC_GID}" depends_on: - gatekeeper diff --git a/jest.config.js b/jest.config.js index 8ba28592..bfb605bf 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,6 +32,8 @@ const config = { '^pino$': '/tests/common/pino.mock.ts', '^\\.\\/typeGuards\\.js$': '/packages/keymaster/src/db/typeGuards.ts', '^\\.\\/db\\/typeGuards\\.js$': '/packages/keymaster/src/db/typeGuards.ts', + '^\\.\\/sync-mapping\\.js$': '/services/mediators/hyperswarm/src/sync-mapping.ts', + '^\\.\\/sync-persistence\\.js$': '/services/mediators/hyperswarm/src/sync-persistence.ts', '^\\.\\/abstract-json\\.js$': '/packages/gatekeeper/src/db/abstract-json.ts', '^\\.\\/cipher-base\\.js$': '/packages/cipher/src/cipher-base.ts', '^\\.\\/encryption\\.js$': '/packages/keymaster/src/encryption.ts', @@ -40,6 +42,10 @@ const config = { "/node_modules/", "/kc-app/", "/client/" + ], + coveragePathIgnorePatterns: [ + "/node_modules/", + "/services/mediators/hyperswarm/src/negentropy/Negentropy\\.cjs$", ] }; diff --git a/packages/gatekeeper/src/gatekeeper.ts b/packages/gatekeeper/src/gatekeeper.ts index 55965561..9094b9e5 100644 --- a/packages/gatekeeper/src/gatekeeper.ts +++ b/packages/gatekeeper/src/gatekeeper.ts @@ -1026,6 +1026,7 @@ export default class Gatekeeper implements GatekeeperInterface { let added = 0; let merged = 0; let rejected = 0; + const acceptedHashes: string[] = []; this.eventsQueue = []; @@ -1036,10 +1037,16 @@ export default class Gatekeeper implements GatekeeperInterface { if (status === ImportStatus.ADDED) { added += 1; + if (event.operation.signature?.hash) { + acceptedHashes.push(event.operation.signature.hash.toLowerCase()); + } this.log.debug(`import ${i}/${total}: added event for ${event.did}`); } else if (status === ImportStatus.MERGED) { merged += 1; + if (event.operation.signature?.hash) { + acceptedHashes.push(event.operation.signature.hash.toLowerCase()); + } this.log.debug(`import ${i}/${total}: merged event for ${event.did}`); } else if (status === ImportStatus.REJECTED) { @@ -1054,7 +1061,7 @@ export default class Gatekeeper implements GatekeeperInterface { event = tempQueue.shift(); } - return { added, merged, rejected }; + return { added, merged, rejected, acceptedHashes }; } async processEvents(): Promise { @@ -1066,6 +1073,7 @@ export default class Gatekeeper implements GatekeeperInterface { let merged = 0; let rejected = 0; let done = false; + const acceptedHashes = new Set(); try { this.isProcessingEvents = true; @@ -1076,6 +1084,9 @@ export default class Gatekeeper implements GatekeeperInterface { added += response.added; merged += response.merged; rejected += response.rejected; + for (const hash of response.acceptedHashes) { + acceptedHashes.add(hash); + } done = (response.added === 0 && response.merged === 0); } @@ -1089,7 +1100,13 @@ export default class Gatekeeper implements GatekeeperInterface { } const pending = this.eventsQueue.length; - const response = { added, merged, rejected, pending }; + const response = { + added, + merged, + rejected, + pending, + acceptedHashes: Array.from(acceptedHashes), + }; this.log.debug(`processEvents: ${JSON.stringify(response)}`); @@ -1181,8 +1198,8 @@ export default class Gatekeeper implements GatekeeperInterface { } let queued = 0; - let rejected = 0; let processed = 0; + const rejectedIndices: number[] = []; for (let i = 0; i < batch.length; i++) { const event = batch[i]; @@ -1200,15 +1217,16 @@ export default class Gatekeeper implements GatekeeperInterface { } } else { - rejected += 1; + rejectedIndices.push(i); } } return { queued, processed, - rejected, - total: this.eventsQueue.length + rejected: rejectedIndices.length, + total: this.eventsQueue.length, + rejectedIndices, }; } diff --git a/packages/gatekeeper/src/types.ts b/packages/gatekeeper/src/types.ts index f0f1b659..e7cd1f4e 100644 --- a/packages/gatekeeper/src/types.ts +++ b/packages/gatekeeper/src/types.ts @@ -13,6 +13,7 @@ export interface ImportBatchResult { processed: number; rejected: number; total: number; + rejectedIndices: number[]; } export interface ProcessEventsResult { @@ -21,6 +22,7 @@ export interface ProcessEventsResult { merged?: number; rejected?: number; pending?: number; + acceptedHashes?: string[]; } export interface VerifyDbResult { @@ -109,6 +111,7 @@ export interface ImportEventsResult { added: number; merged: number; rejected: number; + acceptedHashes: string[]; } export interface GatekeeperClientOptions { diff --git a/sample.env b/sample.env index c180ef21..c68ce785 100644 --- a/sample.env +++ b/sample.env @@ -35,8 +35,15 @@ KC_KEYMASTER_URL=http://localhost:4226 KC_SEARCH_URL=http://localhost:4002 # Hyperswarm -KC_HYPR_EXPORT_INTERVAL=2 +KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public +KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync for peers without negentropy. true|false. +KC_HYPR_NEGENTROPY_ENABLE=true # Enables negentropy sync protocol. true|false. +KC_HYPR_NEGENTROPY_INTERVAL=300 # Seconds between retry attempts for peers not yet fully synced. integer >= 1. +KC_HYPR_NEGENTROPY_WINDOW_DAYS=30 # Reconciliation window size in days for full-sync chunking. integer >= 1. +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=128 # Negentropy frame limit in KB. valid values are 0 (unlimited) or >= 4. +KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window. integer >= 1. +KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session. integer >= 1. # Bitcoin mediator KC_BTC_HOST=localhost diff --git a/services/gatekeeper/server/src/gatekeeper-api.ts b/services/gatekeeper/server/src/gatekeeper-api.ts index 2c81c839..77d6ced2 100644 --- a/services/gatekeeper/server/src/gatekeeper-api.ts +++ b/services/gatekeeper/server/src/gatekeeper-api.ts @@ -1114,6 +1114,11 @@ v1router.post('/dids/export', async (req, res) => { * rejected: * type: integer * description: Number of events that failed validation (bad signature, size limit, etc.). + * rejectedIndices: + * type: array + * description: Zero-based indexes of rejected events in the original submitted batch order (for importDIDs this is `dids.flat()` order). + * items: + * type: integer * total: * type: integer * description: Total number of events in the queue after this import. @@ -1290,6 +1295,11 @@ v1router.post('/batch/export', async (req, res) => { * rejected: * type: integer * description: Number of events that failed validation. + * rejectedIndices: + * type: array + * description: Zero-based indexes of rejected events in the original submitted batch order. + * items: + * type: integer * total: * type: integer * description: The total event queue size after this import. @@ -1608,6 +1618,11 @@ v1router.get('/db/verify', async (req, res) => { * pending: * type: integer * description: Number of events still left in the queue after processing. + * acceptedHashes: + * type: array + * description: Lower-case signature hashes of events accepted during this processing run (added or merged). + * items: + * type: string * 500: * description: Internal Server Error. * content: diff --git a/services/mediators/hyperswarm/README.md b/services/mediators/hyperswarm/README.md index 648abdd9..0e187c10 100644 --- a/services/mediators/hyperswarm/README.md +++ b/services/mediators/hyperswarm/README.md @@ -2,28 +2,71 @@ The Hyperswarm mediator is responsible for distributing unconfirmed MDIP operations to the network and for organizing an IPFS peer network for file-sharing. -When a node gets a new connection, it sends the connection a `sync` message and the connection replies with a series of `batch` messages containing all the operations in the connection's DID database. The nodes imports these operations into its Gatekeeper. The Gatekeeper will add any new operations it hasn't seen before, merge any operations it has already seen, and reject invalid operations. +The mediator supports two synchronization modes: -While running the mediator will poll the Gatekeeper's hyperswarm queue for new operations, and relay them to all of its connections with a `queue` message. -When a node receives a `queue` message it will import the operations like during a `batch` but also relay the message to all of its connections, distributing the new operations with a "gossip protocol". +- `negentropy` mode (preferred): full-history windowed sync on connect, with periodic retry only until the peer reaches a completed sync, using `neg_open`/`neg_msg`/`ops_req`/`ops_push`/`neg_close`. +- `legacy` mode (compatibility): classic `sync` -> full-history `batch` transfer (`shareDb`). + +Realtime propagation is always handled by the Gatekeeper queue gossip path: +- mediator polls `gatekeeper.getQueue('hyperswarm')` +- relays queue operations with a `queue` message +- peers import and further relay `queue` messages + +This keeps low latency for new operations while negentropy handles catch-up. + +## Sync mode behavior + +| peer mode | connect-time behavior | periodic behavior | queue gossip | +| --- | --- | --- | --- | +| `negentropy` | negotiate + run full-history windowed session | periodic retry until sync completes, then stop | enabled | +| `legacy` | `sync` + `shareDb` full-history export | n/a | enabled | + +`shareDb` is intentionally retained for backward compatibility and can be disabled once compatibility validation is complete. + +## Observability + +The mediator emits periodic structured sync metrics in `connectionLoop` including: +- session mode selection counts (`legacy` vs `negentropy`) and fallback rate +- negentropy rounds and have/need totals +- ops requested/pushed sent and received +- gatekeeper apply/reject totals +- bytes sent/received +- session duration aggregates +- queue delay aggregates (from operation `signature.signed` to relay/import time) ## Environment variables -| variable | default | description | -| ------------------------- | ---------------------- | ----------------------------- | -| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | -| `KC_KEYMASTER_URL` | http://localhost:4226 | MDIP keymaster service URL | -| `KC_IPFS_URL` | http://localhost:5001/api/v0 | IPFS RPC URL | -| `KC_IPFS_ENABLE` | true | Enable IPFS + Keymaster peering integration | -| `KC_NODE_ID ` | (no default) | Keymaster node agent name | -| `KC_NODE_NAME` | anon | Human-readable name for the node | -| `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | -| `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | -| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | +| variable | default | description | +| ------------------------- |------------------------------| ----------------------------- | +| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | +| `KC_KEYMASTER_URL` | http://localhost:4226 | MDIP keymaster service URL | +| `KC_IPFS_URL` | http://localhost:5001/api/v0 | IPFS RPC URL | +| `KC_IPFS_ENABLE` | true | Enable IPFS + Keymaster peering integration | +| `KC_NODE_ID ` | (no default) | Keymaster node agent name | +| `KC_NODE_NAME` | anon | Human-readable name for the node | +| `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | +| `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | +| `KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT` | 0 | Negentropy frame-size limit in KB (0 or >= 4) | +| `KC_HYPR_NEGENTROPY_WINDOW_DAYS` | 30 | Reconciliation window size in days for full-sync chunking | +| `KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW` | 25000 | Maximum operations loaded into a single window adapter | +| `KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION` | 64 | Maximum negentropy rounds per window session | +| `KC_HYPR_NEGENTROPY_INTERVAL` | 300 | Seconds between retry attempts for peers not yet fully synced | +| `KC_HYPR_LEGACY_SYNC_ENABLE` | true | Allow legacy `sync`/`shareDb` compatibility path | +| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | + +Negentropy session concurrency is currently fixed at one active session per node. ## IPFS disabled mode Set `KC_IPFS_ENABLE=false` to run the mediator without IPFS or Keymaster integration. In this mode: -- operations still sync and relay over Hyperswarm (batch/queue/sync/ping) +- operations still sync and relay over Hyperswarm (queue + negentropy; legacy sync if enabled) - IPFS peering is disabled and node IPFS info is not published - `KC_NODE_ID` is not required because Keymaster is not used + +## Sync Store Scaffolding + +The mediator now includes a sync-store abstraction in `src/db/` with: +- `SqliteOperationSyncStore` for persistent ordered storage +- `InMemoryOperationSyncStore` for tests + +The SQLite implementation uses a fixed data path under `data/hyperswarm` (relative to the mediator working directory), with an index on `(ts, id)` to use SQLite's native B-tree ordering for range queries. diff --git a/services/mediators/hyperswarm/package-lock.json b/services/mediators/hyperswarm/package-lock.json index 3736aac3..3120fe2c 100644 --- a/services/mediators/hyperswarm/package-lock.json +++ b/services/mediators/hyperswarm/package-lock.json @@ -1,12 +1,12 @@ { "name": "hyperswarm-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hyperswarm-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "license": "MIT", "dependencies": { "@noble/hashes": "^1.3.3", @@ -16,13 +16,21 @@ "express": "^4.21.0", "graceful-goodbye": "^1.3.0", "hyperswarm": "^4.7.14", - "morgan": "^1.10.0" + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7" }, "devDependencies": { "@types/async": "^3.2.24", "@types/b4a": "^1.6.5" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "license": "MIT", + "optional": true + }, "node_modules/@hyperswarm/secret-stream": { "version": "6.6.3", "resolved": "https://registry.npmjs.org/@hyperswarm/secret-stream/-/secret-stream-6.6.3.tgz", @@ -49,6 +57,42 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "license": "MIT", + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/@types/async": { "version": "3.2.24", "resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.24.tgz", @@ -73,6 +117,13 @@ "undici-types": "~6.21.0" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC", + "optional": true + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -85,6 +136,103 @@ "node": ">= 0.6" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/agent-base/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/agent-base/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "optional": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/aproba": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", + "integrity": "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==", + "license": "ISC", + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -100,27 +248,47 @@ "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT", + "optional": true + }, "node_modules/bare-events": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==" }, - "node_modules/basic-auth": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", - "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.8" + "file-uri-to-path": "1.0.0" } }, - "node_modules/basic-auth/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/bits-to-bytes": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bits-to-bytes/-/bits-to-bytes-1.3.0.tgz", @@ -129,6 +297,17 @@ "b4a": "^1.5.0" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/blind-relay": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/blind-relay/-/blind-relay-1.3.3.tgz", @@ -177,6 +356,41 @@ "compact-encoding-net": "^1.2.0" } }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -185,6 +399,36 @@ "node": ">= 0.8" } }, + "node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -203,6 +447,35 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/compact-encoding": { "version": "2.15.0", "resolved": "https://registry.npmjs.org/compact-encoding/-/compact-encoding-2.15.0.tgz", @@ -227,6 +500,20 @@ "compact-encoding": "^2.4.1" } }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT", + "optional": true + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "license": "ISC", + "optional": true + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -276,6 +563,30 @@ "streamx": "^2.12.4" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -292,6 +603,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT", + "optional": true + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -309,6 +627,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/dht-rpc": { "version": "6.14.1", "resolved": "https://registry.npmjs.org/dht-rpc/-/dht-rpc-6.14.1.tgz", @@ -343,6 +670,13 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT", + "optional": true + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -351,6 +685,55 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "license": "MIT", + "optional": true + }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", @@ -383,6 +766,15 @@ "node": ">= 0.6" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.21.2", "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", @@ -434,6 +826,12 @@ "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -467,6 +865,31 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC", + "optional": true + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -475,6 +898,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -493,6 +937,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "optional": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -504,6 +976,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC", + "optional": true + }, "node_modules/graceful-goodbye": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/graceful-goodbye/-/graceful-goodbye-1.3.0.tgz", @@ -545,6 +1024,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "license": "ISC", + "optional": true + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -556,6 +1042,13 @@ "node": ">= 0.4" } }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause", + "optional": true + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -571,40 +1064,129 @@ "node": ">= 0.8" } }, - "node_modules/hypercore-crypto": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/hypercore-crypto/-/hypercore-crypto-3.4.2.tgz", - "integrity": "sha512-16ii4M6T1dFfRa41Szv3IR0wXfImJMYJ8ysZEGwHEDH7sMeWVEBck6tg1GCNutYl39E+H7wMY2p3ndCRfj+XdQ==", + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "license": "MIT", + "optional": true, "dependencies": { - "b4a": "^1.6.6", - "compact-encoding": "^2.15.0", - "sodium-universal": "^4.0.1" + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" } }, - "node_modules/hypercore-id-encoding": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/hypercore-id-encoding/-/hypercore-id-encoding-1.3.0.tgz", - "integrity": "sha512-W6sHdGo5h7LXEsoWfKf/KfuROZmZRQDlGqJF2EPHW+noCK66Vvr0+zE6cL0vqQi18s0kQPeN7Sq3QyR0Ytc2VQ==", + "node_modules/http-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, "dependencies": { - "b4a": "^1.5.3", - "z32": "^1.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/hyperdht": { - "version": "6.17.2", - "resolved": "https://registry.npmjs.org/hyperdht/-/hyperdht-6.17.2.tgz", - "integrity": "sha512-Oxxmj0khgroLR8MizK8BNdOQBRmJp7H+vOSpPv6IDACAN6oaJiqi5m9ValCph7Gc85LhqloExcxep02hOvMamA==", + "node_modules/http-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "optional": true, "dependencies": { - "@hyperswarm/secret-stream": "^6.6.2", - "b4a": "^1.3.1", - "bare-events": "^2.2.0", - "blind-relay": "^1.3.0", - "bogon": "^1.0.0", - "compact-encoding": "^2.4.1", - "compact-encoding-net": "^1.0.1", - "debugging-stream": "^2.0.0", - "dht-rpc": "^6.14.0", - "hypercore-crypto": "^3.3.0", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/https-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/hypercore-crypto": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/hypercore-crypto/-/hypercore-crypto-3.4.2.tgz", + "integrity": "sha512-16ii4M6T1dFfRa41Szv3IR0wXfImJMYJ8ysZEGwHEDH7sMeWVEBck6tg1GCNutYl39E+H7wMY2p3ndCRfj+XdQ==", + "dependencies": { + "b4a": "^1.6.6", + "compact-encoding": "^2.15.0", + "sodium-universal": "^4.0.1" + } + }, + "node_modules/hypercore-id-encoding": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/hypercore-id-encoding/-/hypercore-id-encoding-1.3.0.tgz", + "integrity": "sha512-W6sHdGo5h7LXEsoWfKf/KfuROZmZRQDlGqJF2EPHW+noCK66Vvr0+zE6cL0vqQi18s0kQPeN7Sq3QyR0Ytc2VQ==", + "dependencies": { + "b4a": "^1.5.3", + "z32": "^1.0.0" + } + }, + "node_modules/hyperdht": { + "version": "6.17.2", + "resolved": "https://registry.npmjs.org/hyperdht/-/hyperdht-6.17.2.tgz", + "integrity": "sha512-Oxxmj0khgroLR8MizK8BNdOQBRmJp7H+vOSpPv6IDACAN6oaJiqi5m9ValCph7Gc85LhqloExcxep02hOvMamA==", + "dependencies": { + "@hyperswarm/secret-stream": "^6.6.2", + "b4a": "^1.3.1", + "bare-events": "^2.2.0", + "blind-relay": "^1.3.0", + "bogon": "^1.0.0", + "compact-encoding": "^2.4.1", + "compact-encoding-net": "^1.0.1", + "debugging-stream": "^2.0.0", + "dht-rpc": "^6.14.0", + "hypercore-crypto": "^3.3.0", "hypercore-id-encoding": "^1.2.0", "hypertrace": "^1.3.0", "noise-curve-ed": "^2.0.0", @@ -650,11 +1232,86 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "license": "ISC", + "optional": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "optional": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -663,6 +1320,30 @@ "node": ">= 0.10" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "license": "MIT", + "optional": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, "node_modules/kademlia-routing-table": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/kademlia-routing-table/-/kademlia-routing-table-1.0.3.tgz", @@ -671,6 +1352,47 @@ "bare-events": "^2.2.0" } }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "license": "ISC", + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -725,33 +1447,153 @@ "node": ">= 0.6" } }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "optional": true, "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" + "brace-expansion": "^1.1.7" }, "engines": { - "node": ">= 0.8.0" + "node": "*" } }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", "dependencies": { - "ee-first": "1.1.1" + "yallist": "^4.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "license": "MIT", + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" } }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -762,6 +1604,12 @@ "resolved": "https://registry.npmjs.org/nanoassert/-/nanoassert-2.0.0.tgz", "integrity": "sha512-7vO7n28+aYO4J+8w96AzhmU8G+Y/xpPDJz/se19ICsqj/momRbb9mh9ZUtkoJ5X3nTnPdhEJyc0qnM6yAsHBaA==" }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -780,6 +1628,49 @@ "node": ">= 0.6" } }, + "node_modules/node-abi": { + "version": "3.87.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.87.0.tgz", + "integrity": "sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "license": "MIT", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, "node_modules/node-gyp-build": { "version": "4.8.2", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.2.tgz", @@ -810,6 +1701,39 @@ "sodium-universal": "^4.0.0" } }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/object-inspect": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", @@ -832,13 +1756,29 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", - "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", "license": "MIT", + "optional": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, "engines": { - "node": ">= 0.8" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/parseurl": { @@ -849,12 +1789,69 @@ "node": ">= 0.8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-to-regexp": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "license": "ISC", + "optional": true + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", + "optional": true, + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/protomux": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/protomux/-/protomux-3.9.2.tgz", @@ -880,6 +1877,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -921,6 +1928,35 @@ "node": ">= 0.8" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/record-cache": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/record-cache/-/record-cache-1.2.0.tgz", @@ -929,6 +1965,33 @@ "b4a": "^1.3.1" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -958,6 +2021,18 @@ "resolved": "https://registry.npmjs.org/safety-catch/-/safety-catch-1.0.2.tgz", "integrity": "sha512-C1UYVZ4dtbBxEtvOcpjBaaD27nP8MlvyAQEp2fOTOEe6pfUpk1cDUxij6BR1jZup6rSyUTaBBplK7LanskrULA==" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -1008,6 +2083,13 @@ "node": ">= 0.8.0" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "license": "ISC", + "optional": true + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -1054,11 +2136,129 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC", + "optional": true + }, "node_modules/signal-promise": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/signal-promise/-/signal-promise-1.0.3.tgz", "integrity": "sha512-WBgv0UnIq2C+Aeh0/n+IRpP6967eIx9WpynTUoiW3isPpfe1zu2LJzyfXdo9Tgef8yR/sGjcMvoUXD7EYdiz+g==" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/socks-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socks-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, "node_modules/sodium-native": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sodium-native/-/sodium-native-4.2.0.tgz", @@ -1093,6 +2293,49 @@ } } }, + "node_modules/sqlite": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-5.1.1.tgz", + "integrity": "sha512-oBkezXa2hnkfuJwUo44Hl9hS3er+YFtueifoajrgidvqsJRQFpc5fKoAkAor1O5ZnLoa28GBScfHXs8j0K358Q==", + "license": "MIT" + }, + "node_modules/sqlite3": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.7.tgz", + "integrity": "sha512-GGIyOiFaG+TUra3JIfkI/zGP8yZYLPQ0pl1bH+ODjiX57sPhrLU5sQJn1y9bDKZUFYkX1crlrPfSYt0BKKdkog==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, + "node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1114,6 +2357,113 @@ "bare-events": "^2.2.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, "node_modules/text-decoder": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.0.tgz", @@ -1140,6 +2490,18 @@ "node": ">=0.6" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1171,6 +2533,26 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "license": "ISC", + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, "node_modules/unordered-set": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unordered-set/-/unordered-set-2.0.1.tgz", @@ -1192,6 +2574,12 @@ "b4a": "^1.6.6" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1208,11 +2596,49 @@ "node": ">= 0.8" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/xache": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/xache/-/xache-1.2.1.tgz", "integrity": "sha512-igRS6jPreJ54ABdzhh4mCDXcz+XMaWO2q1ABRV2yWYuk29jlp8VT7UBdCqNkX7rpYBbXsebVVKkwIuYZjyZNqA==" }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, "node_modules/z32": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/z32/-/z32-1.1.0.tgz", diff --git a/services/mediators/hyperswarm/package.json b/services/mediators/hyperswarm/package.json index a94d1132..60719709 100644 --- a/services/mediators/hyperswarm/package.json +++ b/services/mediators/hyperswarm/package.json @@ -5,7 +5,7 @@ "description": "MDIP hyperswarm network mediator", "main": "src/index.js", "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && mkdir -p dist/negentropy && cp src/negentropy/Negentropy.cjs dist/negentropy/Negentropy.cjs", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -18,7 +18,9 @@ "dotenv": "^16.4.5", "express": "^4.21.0", "graceful-goodbye": "^1.3.0", - "hyperswarm": "^4.7.14" + "hyperswarm": "^4.7.14", + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7" }, "devDependencies": { "@types/async": "^3.2.24", diff --git a/services/mediators/hyperswarm/src/bootstrap.ts b/services/mediators/hyperswarm/src/bootstrap.ts new file mode 100644 index 00000000..8a610205 --- /dev/null +++ b/services/mediators/hyperswarm/src/bootstrap.ts @@ -0,0 +1,215 @@ +import type { GatekeeperEvent, GatekeeperInterface, MdipDocument } from '@mdip/gatekeeper/types'; +import type { OperationSyncStore } from './db/types.js'; +import { mapAcceptedOperationsToSyncRecords } from './sync-persistence.js'; + +const DEFAULT_DRIFT_THRESHOLD_PCT = 0.01; // 1% +const BOOTSTRAP_DID_BATCH_SIZE = 500; + +export type BootstrapGatekeeper = Pick; + +export interface BootstrapResult { + skipped: boolean; + reason?: 'store_within_drift_tolerance'; + countBefore: number; + countAfter: number; + exported: number; + mapped: number; + invalid: number; + inserted: number; + driftPct: number; + driftThresholdPct: number; + durationMs: number; +} + +export interface BootstrapOptions { + driftThresholdPct?: number; +} + +function assertValidDriftThreshold(value: number): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error('Invalid driftThresholdPct; expected a number between 0 and 1'); + } +} + +interface BootstrapBatchTotals { + exported: number; + mapped: number; + invalid: number; + canonicalCount: number; +} + +interface BootstrapImportTotals extends BootstrapBatchTotals { + inserted: number; +} + +function toOperations(events: GatekeeperEvent[]): NonNullable[] { + return events + .map(event => event.operation) + .filter((operation): operation is NonNullable => !!operation); +} + +function normalizeDids(input: string[] | MdipDocument[]): string[] { + const dids = input + .map(item => { + if (typeof item === 'string') { + return item; + } + return item?.didDocument?.id; + }) + .filter((did): did is string => typeof did === 'string' && did !== ''); + + return Array.from(new Set(dids)); +} + +async function exportBatchForDids( + gatekeeper: BootstrapGatekeeper, + dids: string[], + batchIndex: number, + batchCount: number, +): Promise[]> { + try { + const events = await gatekeeper.exportBatch(dids); + return toOperations(events); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `bootstrap exportBatch failed for DID batch ${batchIndex + 1}/${batchCount} (${dids.length} dids): ${message}` + ); + } +} + +async function scanGatekeeperBatches( + gatekeeper: BootstrapGatekeeper, + dids: string[], +): Promise { + let exported = 0; + let mapped = 0; + let invalid = 0; + const batchCount = dids.length === 0 ? 0 : Math.ceil(dids.length / BOOTSTRAP_DID_BATCH_SIZE); + + for (let offset = 0, batchIndex = 0; offset < dids.length; offset += BOOTSTRAP_DID_BATCH_SIZE, batchIndex += 1) { + const didBatch = dids.slice(offset, offset + BOOTSTRAP_DID_BATCH_SIZE); + const operations = await exportBatchForDids(gatekeeper, didBatch, batchIndex, batchCount); + exported += operations.length; + + const { records, invalid: invalidBatch } = mapAcceptedOperationsToSyncRecords(operations); + mapped += records.length; + invalid += invalidBatch; + } + + return { + exported, + mapped, + invalid, + canonicalCount: mapped, + }; +} + +async function importGatekeeperBatches( + gatekeeper: BootstrapGatekeeper, + syncStore: OperationSyncStore, + dids: string[], +): Promise { + let exported = 0; + let mapped = 0; + let invalid = 0; + let inserted = 0; + const batchCount = dids.length === 0 ? 0 : Math.ceil(dids.length / BOOTSTRAP_DID_BATCH_SIZE); + + for (let offset = 0, batchIndex = 0; offset < dids.length; offset += BOOTSTRAP_DID_BATCH_SIZE, batchIndex += 1) { + const didBatch = dids.slice(offset, offset + BOOTSTRAP_DID_BATCH_SIZE); + const operations = await exportBatchForDids(gatekeeper, didBatch, batchIndex, batchCount); + exported += operations.length; + + const { records, invalid: invalidBatch } = mapAcceptedOperationsToSyncRecords(operations); + mapped += records.length; + invalid += invalidBatch; + + if (records.length > 0) { + inserted += await syncStore.upsertMany(records); + } + } + + return { + exported, + mapped, + invalid, + inserted, + canonicalCount: mapped, + }; +} + +export async function bootstrapSyncStoreIfEmpty( + syncStore: OperationSyncStore, + gatekeeper: BootstrapGatekeeper, + options: BootstrapOptions = {}, +): Promise { + const driftThresholdPct = options.driftThresholdPct ?? DEFAULT_DRIFT_THRESHOLD_PCT; + assertValidDriftThreshold(driftThresholdPct); + + const startedAt = Date.now(); + const countBefore = await syncStore.count(); + const dids = normalizeDids(await gatekeeper.getDIDs()); + + let exported = 0; + let mapped = 0; + let invalid = 0; + let inserted = 0; + let canonicalCount = 0; + + if (countBefore > 0) { + const scanned = await scanGatekeeperBatches(gatekeeper, dids); + exported = scanned.exported; + mapped = scanned.mapped; + invalid = scanned.invalid; + canonicalCount = scanned.canonicalCount; + } else { + const imported = await importGatekeeperBatches(gatekeeper, syncStore, dids); + exported = imported.exported; + mapped = imported.mapped; + invalid = imported.invalid; + inserted = imported.inserted; + canonicalCount = imported.canonicalCount; + } + + const driftPct = Math.abs(countBefore - canonicalCount) / Math.max(canonicalCount, 1); + + if (countBefore > 0 && driftPct < driftThresholdPct) { + return { + skipped: true, + reason: 'store_within_drift_tolerance', + countBefore, + countAfter: countBefore, + exported, + mapped, + invalid, + inserted: 0, + driftPct, + driftThresholdPct, + durationMs: Date.now() - startedAt, + }; + } + + if (countBefore > 0) { + await syncStore.reset(); + const imported = await importGatekeeperBatches(gatekeeper, syncStore, dids); + exported = imported.exported; + mapped = imported.mapped; + invalid = imported.invalid; + inserted = imported.inserted; + } + const countAfter = await syncStore.count(); + + return { + skipped: false, + countBefore, + countAfter, + exported, + mapped, + invalid, + inserted, + driftPct, + driftThresholdPct, + durationMs: Date.now() - startedAt, + }; +} diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index 3cb8d107..f7659a22 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -2,6 +2,50 @@ import dotenv from 'dotenv'; dotenv.config(); +function parsePositiveIntEnv(varName, defaultValue, options = {}) { + const allowZero = options.allowZero === true; + const raw = process.env[varName]; + if (raw == null || raw === '') { + return defaultValue; + } + + const value = Number.parseInt(raw, 10); + if (!Number.isInteger(value) || value < 0 || (!allowZero && value === 0)) { + const expected = allowZero ? 'a non-negative integer' : 'a positive integer'; + throw new Error(`Invalid ${varName}; expected ${expected}`); + } + + return value; +} + +function parseFrameSizeLimit() { + const valueKb = parsePositiveIntEnv('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT', 0, { allowZero: true }); + + if (valueKb > 0 && valueKb < 4) { + throw new Error('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4 (KB)'); + } + + return valueKb * 1024; +} + +function parseBooleanEnv(varName, defaultValue) { + const raw = process.env[varName]; + if (raw == null || raw === '') { + return defaultValue; + } + + const normalized = raw.trim().toLowerCase(); + if (normalized === 'true') { + return true; + } + + if (normalized === 'false') { + return false; + } + + throw new Error(`Invalid ${varName}; expected true or false`); +} + const config = { debug: process.env.KC_DEBUG ? process.env.KC_DEBUG === 'true' : false, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', @@ -11,7 +55,20 @@ const config = { nodeID: process.env.KC_NODE_ID || '', nodeName: process.env.KC_NODE_NAME || 'anon', protocol: process.env.KC_MDIP_PROTOCOL || '/MDIP/v1.0-public', - exportInterval: process.env.KC_HYPR_EXPORT_INTERVAL ? parseInt(process.env.KC_HYPR_EXPORT_INTERVAL) : 2, + exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), + negentropyEnabled: parseBooleanEnv('KC_HYPR_NEGENTROPY_ENABLE', true), + negentropyFrameSizeLimit: parseFrameSizeLimit(), + negentropyWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_WINDOW_DAYS', 30), + negentropyMaxRecordsPerWindow: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW', 25000), + negentropyMaxRoundsPerSession: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION', 64), + negentropyIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_INTERVAL', 300), + legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), }; +if (!config.negentropyEnabled && !config.legacySyncEnabled) { + throw new Error( + 'Invalid sync configuration; at least one of KC_HYPR_NEGENTROPY_ENABLE or KC_HYPR_LEGACY_SYNC_ENABLE must be true' + ); +} + export default config; diff --git a/services/mediators/hyperswarm/src/db/memory.ts b/services/mediators/hyperswarm/src/db/memory.ts new file mode 100644 index 00000000..88358b8e --- /dev/null +++ b/services/mediators/hyperswarm/src/db/memory.ts @@ -0,0 +1,94 @@ +import { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from './types.js'; + +export default class InMemoryOperationSyncStore implements OperationSyncStore { + private readonly records = new Map(); + + async start(): Promise { + // no-op + } + + async stop(): Promise { + // no-op + } + + async reset(): Promise { + this.records.clear(); + } + + async upsertMany(records: Array | SyncOperationRecord>): Promise { + if (!Array.isArray(records) || records.length === 0) { + return 0; + } + + let inserted = 0; + const now = Date.now(); + + for (const record of records) { + if (this.records.has(record.id)) { + continue; + } + + inserted += 1; + const insertedAt = 'insertedAt' in record ? record.insertedAt : now; + this.records.set(record.id, { + id: record.id, + ts: record.ts, + operation: record.operation, + insertedAt, + }); + } + + return inserted; + } + + async getByIds(ids: string[]): Promise { + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const out: SyncOperationRecord[] = []; + for (const id of ids) { + const row = this.records.get(id); + if (row) { + out.push(row); + } + } + return out; + } + + async iterateSorted(options: SyncStoreListOptions = {}): Promise { + const limit = options.limit ?? 1000; + const after = options.after; + const fromTs = options.fromTs; + const toTs = options.toTs; + + const sorted = Array.from(this.records.values()).sort((a, b) => { + if (a.ts !== b.ts) { + return a.ts - b.ts; + } + return a.id.localeCompare(b.id); + }); + + const filtered = sorted.filter(item => { + if (after && !(item.ts > after.ts || (item.ts === after.ts && item.id > after.id))) { + return false; + } + + if (typeof fromTs === 'number' && item.ts < fromTs) { + return false; + } + + return !(typeof toTs === 'number' && item.ts > toTs); + }); + + return filtered.slice(0, limit); + } + + async has(id: string): Promise { + return this.records.has(id); + } + + async count(): Promise { + return this.records.size; + } +} diff --git a/services/mediators/hyperswarm/src/db/sqlite.ts b/services/mediators/hyperswarm/src/db/sqlite.ts new file mode 100644 index 00000000..6838a3a0 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/sqlite.ts @@ -0,0 +1,231 @@ +import fs from 'fs/promises'; +import path from 'path'; +import * as sqlite from 'sqlite'; +import sqlite3 from 'sqlite3'; +import { childLogger } from '@mdip/common/logger'; +import { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from './types.js'; + +interface SyncRow { + id: string; + ts: number; + operation_json: string; + inserted_at: number; +} + +const SQLITE_NOT_STARTED_ERROR = 'Sync SQLite DB not open. Call start() first.'; +const log = childLogger({ service: 'hyperswarm-sync-db', module: 'sqlite' }); + +export default class SqliteOperationSyncStore implements OperationSyncStore { + private readonly dbName: string; + private readonly dataFolder: string; + private db: sqlite.Database | null; + private _lock: Promise = Promise.resolve(); + + constructor(dbName: string = 'operations.db', dataFolder: string = 'data/hyperswarm') { + this.dbName = dbName; + this.dataFolder = dataFolder; + this.db = null; + } + + private get dbPath(): string { + return path.join(this.dataFolder, this.dbName); + } + + private runExclusive(fn: () => Promise | T): Promise { + const run = async () => await fn(); + const chained = this._lock.then(run, run); + this._lock = chained.then(() => undefined, () => undefined); + return chained; + } + + private async withTx(fn: () => Promise): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + await this.db.exec('BEGIN IMMEDIATE'); + try { + const result = await fn(); + await this.db.exec('COMMIT'); + return result; + } catch (error) { + try { + await this.db.exec('ROLLBACK'); + } catch {} + throw error; + } + } + + async start(): Promise { + if (this.db) { + return; + } + + await fs.mkdir(this.dataFolder, { recursive: true }); + + this.db = await sqlite.open({ + filename: this.dbPath, + driver: sqlite3.Database + }); + + await this.db.exec(` + CREATE TABLE IF NOT EXISTS operations ( + id TEXT PRIMARY KEY, + ts INTEGER NOT NULL, + operation_json TEXT NOT NULL, + inserted_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_operations_ts_id ON operations (ts, id); + `); + } + + async stop(): Promise { + if (this.db) { + await this.db.close(); + this.db = null; + } + } + + async reset(): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + await this.runExclusive(async () => { + await this.withTx(async () => { + await this.db!.run('DELETE FROM operations'); + }); + }); + } + + async upsertMany(records: Array | SyncOperationRecord>): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + if (!Array.isArray(records) || records.length === 0) { + return 0; + } + + return this.runExclusive(async () => { + try { + return await this.withTx(async () => { + let inserted = 0; + const now = Date.now(); + + for (const record of records) { + const insertedAt = 'insertedAt' in record ? record.insertedAt : now; + const res = await this.db!.run( + `INSERT OR IGNORE INTO operations(id, ts, operation_json, inserted_at) VALUES(?, ?, ?, ?)`, + record.id, + record.ts, + JSON.stringify(record.operation), + insertedAt, + ); + + inserted += res.changes ?? 0; + } + + return inserted; + }); + } catch (error) { + log.error({ error }, 'upsertMany failed'); + throw error; + } + }); + } + + async getByIds(ids: string[]): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const placeholders = ids.map(() => '?').join(', '); + const rows = await this.db.all( + `SELECT id, ts, operation_json, inserted_at FROM operations WHERE id IN (${placeholders})`, + ...ids + ); + + const byId = new Map(rows.map(row => [row.id, this.mapRow(row)])); + return ids + .map(id => byId.get(id)) + .filter((item): item is SyncOperationRecord => !!item); + } + + async iterateSorted(options: SyncStoreListOptions = {}): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const limit = options.limit ?? 1000; + const after = options.after; + const fromTs = options.fromTs; + const toTs = options.toTs; + const params: Array = []; + const predicates: string[] = []; + + if (after) { + predicates.push('(ts > ? OR (ts = ? AND id > ?))'); + params.push(after.ts, after.ts, after.id); + } + + if (typeof fromTs === 'number') { + predicates.push('ts >= ?'); + params.push(fromTs); + } + + if (typeof toTs === 'number') { + predicates.push('ts <= ?'); + params.push(toTs); + } + + const where = predicates.length > 0 + ? `WHERE ${predicates.join(' AND ')}` + : ''; + + params.push(limit); + + const rows = await this.db.all( + `SELECT id, ts, operation_json, inserted_at + FROM operations + ${where} + ORDER BY ts ASC, id ASC + LIMIT ?`, + ...params + ); + + return rows.map(row => this.mapRow(row)); + } + + async has(id: string): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const row = await this.db.get<{ id: string }>('SELECT id FROM operations WHERE id = ? LIMIT 1', id); + return !!row; + } + + async count(): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const row = await this.db.get<{ count: number }>('SELECT COUNT(*) AS count FROM operations'); + return row?.count ?? 0; + } + + private mapRow(row: SyncRow): SyncOperationRecord { + return { + id: row.id, + ts: row.ts, + operation: JSON.parse(row.operation_json), + insertedAt: row.inserted_at, + }; + } +} diff --git a/services/mediators/hyperswarm/src/db/types.ts b/services/mediators/hyperswarm/src/db/types.ts new file mode 100644 index 00000000..ed578646 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/types.ts @@ -0,0 +1,31 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface SyncOperationRecord { + id: string; + ts: number; + operation: Operation; + insertedAt: number; +} + +export interface SyncStoreCursor { + ts: number; + id: string; +} + +export interface SyncStoreListOptions { + after?: SyncStoreCursor; + limit?: number; + fromTs?: number; + toTs?: number; +} + +export interface OperationSyncStore { + start(): Promise; + stop(): Promise; + reset(): Promise; + upsertMany(records: Array | SyncOperationRecord>): Promise; + getByIds(ids: string[]): Promise; + iterateSorted(options?: SyncStoreListOptions): Promise; + has(id: string): Promise; + count(): Promise; +} diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index b1fe49c9..2c40974b 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -12,31 +12,143 @@ import { Operation } from '@mdip/gatekeeper/types'; import CipherNode from '@mdip/cipher/node'; import { childLogger } from '@mdip/common/logger'; import config from './config.js'; +import type { OperationSyncStore } from './db/types.js'; +import SqliteOperationSyncStore from './db/sqlite.js'; +import NegentropyAdapter, { + type NegentropyWindowStats, + type ReconciliationWindow, +} from './negentropy/adapter.js'; +import { + NEG_SYNC_ID_RE, + chooseConnectSyncMode, + decodeNegentropyFrame, + encodeNegentropyFrame, + extractOperationHashes, + normalizeNegentropyIds, + normalizePeerCapabilities, + type ConnectSyncModeReason, + type NegentropyFrame, + type NegotiatedPeerCapabilities, + type PeerCapabilities, + type SyncMode, +} from './negentropy/protocol.js'; +import { + shouldAcceptLegacySync, + shouldSchedulePeriodicRepair, + shouldStartConnectTimeNegentropy, +} from './negentropy/policy.js'; +import { + addAggregateSample, + averageAggregate, + collectQueueDelaySamples, + createAggregateMetric, + messageBytes, + safeRate, + type AggregateMetric, +} from './negentropy/observability.js'; +import { + chunkIds, + chunkOperationsForPush, +} from './negentropy/transfer.js'; +import { bootstrapSyncStoreIfEmpty } from './bootstrap.js'; +import { + filterOperationsByAcceptedHashes, + filterIndexRejectedOperations, + mapAcceptedOperationsToSyncRecords, +} from './sync-persistence.js'; import { exit } from 'process'; +import path from 'path'; +import { pathToFileURL } from 'url'; const log = childLogger({ service: 'hyperswarm-mediator' }); -interface HyperMessage { - type: 'batch' | 'queue' | 'sync' | 'ping'; +interface HyperMessageBase { + type: string; time: string; node: string; relays: string[]; } -interface PingMessage extends HyperMessage { +interface PingMessage extends HyperMessageBase { + type: 'ping'; peers: string[]; + capabilities?: PeerCapabilities; } -interface BatchMessage extends HyperMessage { +interface BatchMessage extends HyperMessageBase { + type: 'batch' | 'queue'; data: Operation[]; } +interface SyncMessage extends HyperMessageBase { + type: 'sync'; +} + +interface NegOpenMessage extends HyperMessageBase { + type: 'neg_open'; + sessionId: string; + windowId: string; + window: { + name: string; + fromTs: number; + toTs: number; + maxRecords: number; + order: number; + }; + round: number; + frame: NegentropyFrame; +} + +interface NegMsgMessage extends HyperMessageBase { + type: 'neg_msg'; + sessionId: string; + windowId: string; + round: number; + frame: NegentropyFrame; +} + +interface NegCloseMessage extends HyperMessageBase { + type: 'neg_close'; + sessionId: string; + windowId: string; + round: number; + reason?: string; +} + +interface OpsReqMessage extends HyperMessageBase { + type: 'ops_req'; + sessionId: string; + windowId: string; + round: number; + ids: string[]; +} + +interface OpsPushMessage extends HyperMessageBase { + type: 'ops_push'; + sessionId: string; + windowId: string; + round: number; + data: Operation[]; +} + +type HyperMessage = + | BatchMessage + | SyncMessage + | PingMessage + | NegOpenMessage + | NegMsgMessage + | NegCloseMessage + | OpsReqMessage + | OpsPushMessage; + interface ImportQueueTask { name: string; msg: BatchMessage; } -interface ExportQueueTask extends ImportQueueTask { +interface ExportQueueTask { + name: string; + msg: SyncMessage; conn: HyperswarmConnection; } @@ -52,31 +164,148 @@ interface ConnectionInfo { nodeName: string; did: string; lastSeen: number; + capabilities: NegotiatedPeerCapabilities; + syncMode: SyncMode | 'unknown'; + syncStarted: boolean; + lastNegentropyAttemptAt: number; + negentropySynced: boolean; +} + +interface PeerSyncSession { + sessionId: string; + peerKey: string; + mode: SyncMode; + initiator: boolean; + windows: ReconciliationWindow[]; + windowIndex: number; + windowId: string | null; + currentWindowStats: NegentropyWindowStats | null; + completedWindows: NegentropyWindowStats[]; + startedAt: number; + lastActivity: number; + pendingHaveIds: Set; + pendingNeedIds: Set; + rounds: number; + maxRounds: number; + reconciliationComplete: boolean; + localClosed: boolean; +} + +interface MediatorSyncStats { + modeSelectionsTotal: number; + modeSelectionsLegacy: number; + modeSelectionsNegentropy: number; + modeSelectionsLegacyMissingCapabilities: number; + modeSelectionsLegacyNegentropyDisabled: number; + modeSelectionsLegacyVersionMismatch: number; + modeSelectionsNoModeLegacyDisabled: number; + queueOpsRelayed: number; + queueOpsImported: number; + queueDelayMs: AggregateMetric; + negentropySessionsStarted: number; + negentropySessionsClosed: number; + negentropySessionsCompleted: number; + negentropySessionsFailed: number; + negentropyRounds: number; + negentropyHaveIds: number; + negentropyNeedIds: number; + negentropyOpsReqSent: number; + negentropyOpsReqReceived: number; + negentropyOpsPushSent: number; + negentropyOpsPushReceived: number; + opsApplied: number; + opsRejected: number; + bytesSent: number; + bytesReceived: number; + syncDurationMs: AggregateMetric; +} + +export interface MediatorMainOptions { + syncStore?: OperationSyncStore; + startLoops?: boolean; } const gatekeeper = new GatekeeperClient(); const keymaster = new KeymasterClient(); const ipfs = new KuboClient(); const cipher = new CipherNode(); +let syncStore: OperationSyncStore = new SqliteOperationSyncStore(); +let negentropyAdapter: NegentropyAdapter | null = null; +let adapterChangeSeq = 0; +let adapterBuiltSeq = -1; +let adapterBuiltAt = 0; +let adapterBuiltWindowId: string | null = null; +let adapterBuiltWindowStats: NegentropyWindowStats | null = null; +let rebuildPromise: Promise | null = null; +let backgroundPrebuildQueued = false; EventEmitter.defaultMaxListeners = 100; const REGISTRY = 'hyperswarm'; const BATCH_SIZE = 100; +const NEGENTROPY_VERSION = 1; +const NEG_SESSION_IDLE_TIMEOUT_MS = 2 * 60 * 1000; +const NEG_MAX_IDS_PER_OPS_REQ = 1_000; +const NEG_MAX_IDS_PER_LOOKUP = 1_000; +const NEG_MAX_OPS_PER_PUSH = 256; +const NEG_MAX_BYTES_PER_PUSH = 512 * 1024; +const NEG_REPAIR_INTERVAL_MS = config.negentropyIntervalSeconds * 1000; +const NEG_ADAPTER_MAX_AGE_MS = 60 * 1000; const connectionInfo: Record = {}; const knownNodes: Record = {}; const knownPeers: Record = {}; const addedPeers: Record = {}; const badPeers: Record = {}; +const peerSessions = new Map(); +const syncStats: MediatorSyncStats = { + modeSelectionsTotal: 0, + modeSelectionsLegacy: 0, + modeSelectionsNegentropy: 0, + modeSelectionsLegacyMissingCapabilities: 0, + modeSelectionsLegacyNegentropyDisabled: 0, + modeSelectionsLegacyVersionMismatch: 0, + modeSelectionsNoModeLegacyDisabled: 0, + queueOpsRelayed: 0, + queueOpsImported: 0, + queueDelayMs: createAggregateMetric(), + negentropySessionsStarted: 0, + negentropySessionsClosed: 0, + negentropySessionsCompleted: 0, + negentropySessionsFailed: 0, + negentropyRounds: 0, + negentropyHaveIds: 0, + negentropyNeedIds: 0, + negentropyOpsReqSent: 0, + negentropyOpsReqReceived: 0, + negentropyOpsPushSent: 0, + negentropyOpsPushReceived: 0, + opsApplied: 0, + opsRejected: 0, + bytesSent: 0, + bytesReceived: 0, + syncDurationMs: createAggregateMetric(), +}; let swarm: Hyperswarm | null = null; let nodeKey = ''; let nodeInfo: NodeInfo; -goodbye(() => { +goodbye(async () => { if (swarm) { - swarm.destroy(); + try { + await Promise.resolve(swarm.destroy()); + } catch (error) { + log.error({ error }, 'swarm destroy error'); + } finally { + swarm = null; + } + } + + try { + await syncStore.stop(); + } catch (error) { + log.error({ error }, 'syncStore stop error'); } }); @@ -114,6 +343,7 @@ let syncQueue = asyncLib.queue( }; const json = JSON.stringify(msg); + syncStats.bytesSent += messageBytes(json); conn.write(json); } catch (error) { @@ -131,9 +361,6 @@ function addConnection(conn: HyperswarmConnection): void { log.info(`received connection from: ${peerName}`); - // Push the connection to the syncQueue instead of writing directly - syncQueue.push(conn); - connectionInfo[peerKey] = { connection: conn, key: peerKey, @@ -141,23 +368,963 @@ function addConnection(conn: HyperswarmConnection): void { nodeName: 'anon', did: '', lastSeen: new Date().getTime(), + capabilities: { + advertised: false, + negentropy: false, + version: null, + }, + syncMode: 'unknown', + syncStarted: false, + lastNegentropyAttemptAt: 0, + negentropySynced: false, }; const peerNames = Object.values(connectionInfo).map(info => info.peerName); log.debug(`--- ${peerNames.length} nodes connected, detected nodes: ${peerNames.join(', ')}`); + + void sendPingToPeer(peerKey); } function closeConnection(peerKey: string): void { const conn = connectionInfo[peerKey]; + if (!conn) { + return; + } log.info(`* connection closed with: ${conn.peerName} (${conn.nodeName}) *`); delete connectionInfo[peerKey]; + closePeerSession(peerKey, 'connection_closed'); } function shortName(peerKey: string): string { return peerKey.slice(0, 4) + '-' + peerKey.slice(-4); } +function createBaseMessage(type: T): Omit & { type: T } { + return { + type, + time: new Date().toISOString(), + node: nodeInfo?.name || config.nodeName, + relays: [], + }; +} + +function buildPingMessage(): PingMessage { + const capabilities = config.negentropyEnabled + ? { + negentropy: true, + negentropyVersion: NEGENTROPY_VERSION, + } + : { + negentropy: false, + }; + + return { + ...createBaseMessage('ping'), + peers: Object.keys(knownNodes), + capabilities, + }; +} + +function sendToPeer(peerKey: string, msg: HyperMessage): boolean { + const conn = connectionInfo[peerKey]; + if (!conn) { + return false; + } + + const json = JSON.stringify(msg); + syncStats.bytesSent += messageBytes(json); + conn.connection.write(json); + return true; +} + +async function sendPingToPeer(peerKey: string): Promise { + const ping = buildPingMessage(); + if (sendToPeer(peerKey, ping)) { + log.debug(`* sent ping to: ${shortName(peerKey)}`); + } +} + +function createSessionId(peerKey: string): string { + const nonce = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36); + return `${Date.now().toString(36)}-${shortName(nodeKey)}-${shortName(peerKey)}-${nonce}`; +} + +function createPeerSession(peerKey: string, mode: SyncMode, initiator: boolean, sessionId?: string): PeerSyncSession { + const now = Date.now(); + const session: PeerSyncSession = { + sessionId: sessionId ?? createSessionId(peerKey), + peerKey, + mode, + initiator, + windows: [], + windowIndex: 0, + windowId: null, + currentWindowStats: null, + completedWindows: [], + startedAt: now, + lastActivity: now, + pendingHaveIds: new Set(), + pendingNeedIds: new Set(), + rounds: 0, + maxRounds: config.negentropyMaxRoundsPerSession, + reconciliationComplete: false, + localClosed: false, + }; + peerSessions.set(peerKey, session); + connectionInfo[peerKey].syncMode = mode; + connectionInfo[peerKey].syncStarted = true; + if (mode === 'negentropy') { + syncStats.negentropySessionsStarted += 1; + connectionInfo[peerKey].lastNegentropyAttemptAt = now; + } + return session; +} + +function touchPeerSession(peerKey: string): void { + const session = peerSessions.get(peerKey); + if (!session) { + return; + } + session.lastActivity = Date.now(); +} + +function closePeerSession(peerKey: string, reason: string): void { + const session = peerSessions.get(peerKey); + if (!session) { + return; + } + + peerSessions.delete(peerKey); + addAggregateSample(syncStats.syncDurationMs, Date.now() - session.startedAt); + const conn = connectionInfo[peerKey]; + if (conn && session.mode === 'negentropy') { + conn.lastNegentropyAttemptAt = Date.now(); + syncStats.negentropySessionsClosed += 1; + if (reason === 'complete') { + conn.negentropySynced = true; + syncStats.negentropySessionsCompleted += 1; + } else { + conn.negentropySynced = false; + syncStats.negentropySessionsFailed += 1; + } + } + + maybeStartBackgroundPrebuild('session_closed'); + + log.debug({ + peer: shortName(peerKey), + mode: session.mode, + rounds: session.rounds, + pendingHave: session.pendingHaveIds.size, + pendingNeed: session.pendingNeedIds.size, + reason, + }, 'peer sync session closed'); +} + +function expireIdlePeerSessions(): void { + const now = Date.now(); + for (const [peerKey, session] of peerSessions.entries()) { + if (now - session.lastActivity > NEG_SESSION_IDLE_TIMEOUT_MS) { + sendNegClose(peerKey, session, 'idle_timeout'); + closePeerSession(peerKey, 'idle_timeout'); + } + } +} + +function choosePeerSyncMode(peerKey: string): { mode: SyncMode | null; reason: ConnectSyncModeReason } | null { + const conn = connectionInfo[peerKey]; + if (!conn) { + return null; + } + + return chooseConnectSyncMode( + conn.capabilities, + NEGENTROPY_VERSION, + config.legacySyncEnabled, + config.negentropyEnabled, + ); +} + +function getActiveNegentropySessions(): number { + let count = 0; + for (const session of peerSessions.values()) { + if (session.mode === 'negentropy') { + count += 1; + } + } + return count; +} + +async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic' = 'connect'): Promise { + const conn = connectionInfo[peerKey]; + if (!conn) { + return; + } + + if (source === 'connect' && conn.syncStarted) { + return; + } + + let mode: SyncMode | 'unknown' | null; + let modeReason: ConnectSyncModeReason | null = null; + + if (source === 'connect') { + const decision = choosePeerSyncMode(peerKey); + if (!decision) { + return; + } + mode = decision.mode; + modeReason = decision.reason; + } else { + mode = conn.syncMode; + } + + if (!mode || mode === 'unknown') { + if (source === 'connect' && modeReason === 'legacy_disabled') { + syncStats.modeSelectionsNoModeLegacyDisabled += 1; + } + return; + } + + if (source === 'connect' && mode === 'negentropy' && importQueue.length() > 0) { + setTimeout(() => void maybeStartPeerSync(peerKey, source), 1_000); + return; + } + + if (source === 'connect') { + syncStats.modeSelectionsTotal += 1; + if (mode === 'legacy') { + syncStats.modeSelectionsLegacy += 1; + if (modeReason === 'missing_capabilities') { + syncStats.modeSelectionsLegacyMissingCapabilities += 1; + } + if (modeReason === 'negentropy_disabled') { + syncStats.modeSelectionsLegacyNegentropyDisabled += 1; + } + if (modeReason === 'version_mismatch') { + syncStats.modeSelectionsLegacyVersionMismatch += 1; + } + } else { + syncStats.modeSelectionsNegentropy += 1; + } + } + + if (mode === 'legacy') { + if (!config.legacySyncEnabled) { + return; + } + + if (source !== 'connect') { + return; + } + + createPeerSession(peerKey, 'legacy', true, `legacy-${Date.now().toString(36)}`); + syncQueue.push(conn.connection); + log.info({ peer: shortName(peerKey), mode, modeReason }, 'peer sync mode selected'); + return; + } + + const initiator = nodeKey.localeCompare(peerKey) < 0; + const hasActiveSession = peerSessions.has(peerKey); + const activeNegentropySessions = getActiveNegentropySessions(); + + conn.syncMode = 'negentropy'; + conn.syncStarted = true; + + if (conn.negentropySynced) { + return; + } + + const shouldStart = source === 'connect' + ? shouldStartConnectTimeNegentropy(mode, hasActiveSession, initiator) + : shouldSchedulePeriodicRepair({ + syncMode: mode, + hasActiveSession, + importQueueLength: importQueue.length(), + activeNegentropySessions, + lastAttemptAtMs: conn.lastNegentropyAttemptAt, + nowMs: Date.now(), + repairIntervalMs: NEG_REPAIR_INTERVAL_MS, + isInitiator: initiator, + syncCompleted: conn.negentropySynced, + }); + + if (!shouldStart) { + return; + } + + if (activeNegentropySessions > 0) { + return; + } + + const session = createPeerSession(peerKey, 'negentropy', initiator); + session.windows = await planRuntimeWindows(); + session.windowIndex = 0; + session.completedWindows = []; + log.info( + { + peer: shortName(peerKey), + mode, + modeReason, + initiator, + sessionId: session.sessionId, + source, + plannedWindows: session.windows.length, + }, + 'peer sync mode selected' + ); + await startNextNegentropyWindow(peerKey, session); +} + +async function runPeriodicNegentropyRepair(): Promise { + for (const peerKey in connectionInfo) { + try { + await maybeStartPeerSync(peerKey, 'periodic'); + } catch (error) { + log.error({ error, peer: shortName(peerKey) }, 'periodic negentropy repair error'); + } + } +} + +function buildSyncStatsSnapshot(): object { + return { + modeSelections: { + total: syncStats.modeSelectionsTotal, + legacy: syncStats.modeSelectionsLegacy, + negentropy: syncStats.modeSelectionsNegentropy, + fallbackCount: syncStats.modeSelectionsLegacy, + fallbackRate: safeRate(syncStats.modeSelectionsLegacy, syncStats.modeSelectionsTotal), + legacyReasons: { + missingCapabilities: syncStats.modeSelectionsLegacyMissingCapabilities, + negentropyDisabled: syncStats.modeSelectionsLegacyNegentropyDisabled, + versionMismatch: syncStats.modeSelectionsLegacyVersionMismatch, + }, + noMode: { + legacyDisabled: syncStats.modeSelectionsNoModeLegacyDisabled, + }, + }, + queue: { + relayed: syncStats.queueOpsRelayed, + imported: syncStats.queueOpsImported, + delayMs: { + avg: averageAggregate(syncStats.queueDelayMs), + max: syncStats.queueDelayMs.max, + samples: syncStats.queueDelayMs.count, + }, + }, + negentropy: { + sessionsStarted: syncStats.negentropySessionsStarted, + sessionsClosed: syncStats.negentropySessionsClosed, + sessionsCompleted: syncStats.negentropySessionsCompleted, + sessionsFailed: syncStats.negentropySessionsFailed, + rounds: syncStats.negentropyRounds, + haveIds: syncStats.negentropyHaveIds, + needIds: syncStats.negentropyNeedIds, + opsRequested: syncStats.negentropyOpsReqSent, + opsRequestedReceived: syncStats.negentropyOpsReqReceived, + opsPushed: syncStats.negentropyOpsPushSent, + opsPushedReceived: syncStats.negentropyOpsPushReceived, + }, + gatekeeper: { + opsApplied: syncStats.opsApplied, + opsRejected: syncStats.opsRejected, + }, + transport: { + bytesSent: syncStats.bytesSent, + bytesReceived: syncStats.bytesReceived, + }, + syncDurationMs: { + avg: averageAggregate(syncStats.syncDurationMs), + max: syncStats.syncDurationMs.max, + sessions: syncStats.syncDurationMs.count, + }, + }; +} + +function isNegentropyAdapterDirty(): boolean { + return adapterBuiltSeq < adapterChangeSeq; +} + +function markNegentropyAdapterDirty(): void { + adapterChangeSeq += 1; +} + +function cloneWindowStats(stats: NegentropyWindowStats | null): NegentropyWindowStats | null { + return stats + ? { + ...stats, + } + : null; +} + +function buildFullHistoryWindow(): ReconciliationWindow { + return { + name: 'full_history', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: Number.MAX_SAFE_INTEGER, + order: 0, + }; +} + +function currentSyncTimestampSec(): number { + return Math.floor(Date.now() / 1000); +} + +function makeWindowId(window: ReconciliationWindow): string { + return `${window.order}:${window.name}:${window.fromTs}:${window.toTs}:${window.maxRecords}`; +} + +function windowLabel(window: ReconciliationWindow): string { + return `${window.name}[${window.fromTs},${window.toTs}]`; +} + +function getSessionWindow(session: PeerSyncSession): ReconciliationWindow | null { + if (session.windowIndex < 0 || session.windowIndex >= session.windows.length) { + return null; + } + return session.windows[session.windowIndex]; +} + +function parseRemoteWindow(raw: NegOpenMessage['window']): ReconciliationWindow | null { + if (!raw || typeof raw !== 'object') { + return null; + } + + const fromTs = Number(raw.fromTs); + const toTs = Number(raw.toTs); + const order = Number(raw.order); + const remoteMaxRecords = Number(raw.maxRecords); + const maxRecords = Number.isInteger(remoteMaxRecords) && remoteMaxRecords > 0 + ? Math.min(remoteMaxRecords, config.negentropyMaxRecordsPerWindow) + : config.negentropyMaxRecordsPerWindow; + + if (!Number.isFinite(fromTs) || !Number.isFinite(toTs) || fromTs > toTs) { + return null; + } + + if (!Number.isInteger(order) || order < 0) { + return null; + } + + return { + name: String(raw.name || `window_${order}`), + fromTs, + toTs, + order, + maxRecords, + }; +} + +function initializeSessionWindowState( + session: PeerSyncSession, + window: ReconciliationWindow, + windowId: string, + windowStats: NegentropyWindowStats, +): void { + session.windowId = windowId; + session.pendingHaveIds = new Set(); + session.pendingNeedIds = new Set(); + session.reconciliationComplete = false; + session.currentWindowStats = { + ...windowStats, + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + rounds: 0, + completed: false, + cappedByRounds: false, + }; +} + +function finalizeCurrentWindowStats( + session: PeerSyncSession, + options: { completed?: boolean; cappedByRounds?: boolean } = {}, +): NegentropyWindowStats | null { + if (!session.currentWindowStats) { + return null; + } + + const finished: NegentropyWindowStats = { + ...session.currentWindowStats, + completed: options.completed ?? true, + cappedByRounds: options.cappedByRounds ?? false, + }; + if (session.currentWindowStats.completed) { + return session.currentWindowStats; + } + session.currentWindowStats = finished; + session.completedWindows.push({ ...finished }); + return finished; +} + +function shouldAdvanceToOlderWindow(session: PeerSyncSession): boolean { + return session.windowIndex + 1 < session.windows.length; +} + +async function planRuntimeWindows(): Promise { + if (!negentropyAdapter) { + // eslint-disable-next-line sonarjs/no-duplicate-string + throw new Error('negentropy adapter unavailable'); + } + + const windows = await negentropyAdapter.planWindows(currentSyncTimestampSec()); + if (windows.length > 0) { + return windows; + } + + return [buildFullHistoryWindow()]; +} + +function maybeStartBackgroundPrebuild(reason: string): void { + if (!negentropyAdapter) { + return; + } + + if (!isNegentropyAdapterDirty()) { + return; + } + + if (getActiveNegentropySessions() > 0) { + return; + } + + if (rebuildPromise) { + backgroundPrebuildQueued = true; + return; + } + + backgroundPrebuildQueued = false; + void (async () => { + const windows = await planRuntimeWindows(); + const recentWindow = windows[0]; + await ensureWindowAdapterFresh(recentWindow, `background_${reason}`); + })() + .catch(error => { + log.error({ error, reason }, 'background negentropy prebuild failed'); + }) + .finally(() => { + if (!backgroundPrebuildQueued) { + return; + } + + backgroundPrebuildQueued = false; + if (isNegentropyAdapterDirty() && getActiveNegentropySessions() === 0) { + maybeStartBackgroundPrebuild('queued_followup'); + } + }); +} + +async function ensureWindowAdapterFresh(window: ReconciliationWindow, reason: string): Promise { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + const targetWindowId = makeWindowId(window); + const now = Date.now(); + const recentlyBuilt = adapterBuiltAt > 0 && (now - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; + const sameWindow = adapterBuiltWindowId === targetWindowId; + + if (!isNegentropyAdapterDirty() && recentlyBuilt && sameWindow) { + const cached = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (cached) { + return cached; + } + } + + if (rebuildPromise) { + await rebuildPromise; + const recentAfterWait = adapterBuiltAt > 0 && (Date.now() - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; + const sameWindowAfterWait = adapterBuiltWindowId === targetWindowId; + if (!isNegentropyAdapterDirty() && recentAfterWait && sameWindowAfterWait) { + const cached = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (cached) { + return cached; + } + } + } + + const rebuildStartSeq = adapterChangeSeq; + const rebuildStartedAt = Date.now(); + const currentRebuildPromise = (async () => { + const windowStats = await negentropyAdapter!.rebuildForWindow(window); + adapterBuiltSeq = rebuildStartSeq; + adapterBuiltAt = Date.now(); + adapterBuiltWindowId = targetWindowId; + adapterBuiltWindowStats = cloneWindowStats(windowStats); + log.debug( + { + reason, + durationMs: adapterBuiltAt - rebuildStartedAt, + adapterBuiltAt, + windowId: targetWindowId, + window: windowLabel(window), + dirtyAfterRebuild: isNegentropyAdapterDirty(), + }, + 'negentropy adapter rebuilt from sync-store' + ); + })(); + + rebuildPromise = currentRebuildPromise; + try { + await currentRebuildPromise; + } + finally { + if (rebuildPromise === currentRebuildPromise) { + rebuildPromise = null; + } + } + + const refreshed = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (!refreshed) { + throw new Error(`negentropy window stats unavailable after rebuild (${targetWindowId})`); + } + return refreshed; +} + +async function startNextNegentropyWindow(peerKey: string, session: PeerSyncSession): Promise { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + const window = getSessionWindow(session); + if (!window) { + throw new Error(`missing reconciliation window at index ${session.windowIndex}`); + } + + const windowId = makeWindowId(window); + const windowStats = await ensureWindowAdapterFresh(window, 'session_open_initiator'); + initializeSessionWindowState(session, window, windowId, windowStats); + const firstFrame = await negentropyAdapter.initiate(); + const msg: NegOpenMessage = { + ...createBaseMessage('neg_open'), + sessionId: session.sessionId, + windowId, + window: { + name: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + maxRecords: window.maxRecords, + order: window.order, + }, + round: session.rounds, + frame: encodeNegentropyFrame(firstFrame), + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_neg_open_failed'); + return; + } + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId, + window: windowLabel(window), + }, + 'negentropy window open sent' + ); +} + +async function maybeAdvanceToOlderWindow(peerKey: string, session: PeerSyncSession): Promise { + if (!shouldAdvanceToOlderWindow(session)) { + return false; + } + + session.windowIndex += 1; + await startNextNegentropyWindow(peerKey, session); + return true; +} + +function getExpectedWindowId(session: PeerSyncSession): string { + if (!session.windowId) { + throw new Error(`session ${session.sessionId} has no active window`); + } + return session.windowId; +} + +function isCurrentSessionWindow(peerKey: string, session: PeerSyncSession, windowId: string, msgType: string): boolean { + if (!session.windowId || windowId !== session.windowId) { + log.warn( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + msgType, + expectedWindowId: session.windowId, + receivedWindowId: windowId, + }, + 'ignoring negentropy message for non-current window' + ); + return false; + } + return true; +} + +async function sendOpsReq(peerKey: string, session: PeerSyncSession, ids: string[]): Promise { + const normalized = Array.from(new Set(ids.map(id => id.toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))); + const batches = chunkIds(normalized, NEG_MAX_IDS_PER_OPS_REQ); + + for (const batch of batches) { + const msg: OpsReqMessage = { + ...createBaseMessage('ops_req'), + sessionId: session.sessionId, + windowId: getExpectedWindowId(session), + round: session.rounds, + ids: batch, + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_ops_req_failed'); + return; + } + syncStats.negentropyOpsReqSent += batch.length; + } +} + +async function sendOpsPushForIds(peerKey: string, session: PeerSyncSession, ids: string[]): Promise { + const normalized = Array.from(new Set(ids.map(id => id.toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))); + const idLookupBatches = chunkIds(normalized, NEG_MAX_IDS_PER_LOOKUP); + + for (const idBatch of idLookupBatches) { + const rows = await syncStore.getByIds(idBatch); + const operations = rows.map(row => row.operation); + if (operations.length === 0) { + continue; + } + + const opBatches = chunkOperationsForPush(operations, { + maxOpsPerPush: NEG_MAX_OPS_PER_PUSH, + maxBytesPerPush: NEG_MAX_BYTES_PER_PUSH, + }); + + for (const opBatch of opBatches) { + const msg: OpsPushMessage = { + ...createBaseMessage('ops_push'), + sessionId: session.sessionId, + windowId: getExpectedWindowId(session), + round: session.rounds, + data: opBatch, + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_ops_push_failed'); + return; + } + syncStats.negentropyOpsPushSent += opBatch.length; + } + } +} + +function sendNegMsg(peerKey: string, session: PeerSyncSession, frame: string | Uint8Array): boolean { + const msg: NegMsgMessage = { + ...createBaseMessage('neg_msg'), + sessionId: session.sessionId, + windowId: getExpectedWindowId(session), + round: session.rounds, + frame: encodeNegentropyFrame(frame), + }; + + return sendToPeer(peerKey, msg); +} + +function sendNegClose(peerKey: string, session: PeerSyncSession, reason: string): boolean { + session.localClosed = true; + const windowId = session.windowId ?? 'none'; + const closeMsg: NegCloseMessage = { + ...createBaseMessage('neg_close'), + sessionId: session.sessionId, + windowId, + round: session.rounds, + reason, + }; + + return sendToPeer(peerKey, closeMsg); +} + +async function reconcileNegentropyFrame( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise<{ + nextMsg: string | Uint8Array | null; + haveIds: string[]; + needIds: string[]; +} | null> { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + const windowRounds = session.currentWindowStats?.rounds ?? 0; + if (windowRounds >= session.maxRounds) { + finalizeCurrentWindowStats(session, { completed: false, cappedByRounds: true }); + sendNegClose(peerKey, session, 'max_rounds_reached'); + closePeerSession(peerKey, 'max_rounds_reached'); + return null; + } + + const result = await negentropyAdapter.reconcile(frame); + session.rounds += 1; + if (session.currentWindowStats) { + session.currentWindowStats.rounds += 1; + } + touchPeerSession(peerKey); + + return { + nextMsg: result.nextMsg, + haveIds: normalizeNegentropyIds(result.haveIds), + needIds: normalizeNegentropyIds(result.needIds), + }; +} + +async function maybeFinalizeInitiatorSession(peerKey: string, session: PeerSyncSession): Promise { + if (!session.initiator) { + return; + } + + if (!session.reconciliationComplete) { + return; + } + + if (session.pendingNeedIds.size > 0) { + return; + } + + const advanced = await maybeAdvanceToOlderWindow(peerKey, session); + if (advanced) { + return; + } + + if (!sendNegClose(peerKey, session, 'complete')) { + closePeerSession(peerKey, 'send_neg_close_failed'); + return; + } + + closePeerSession(peerKey, 'complete'); +} + +async function handleNegentropyRoundAsInitiator( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise { + const outcome = await reconcileNegentropyFrame(peerKey, session, frame); + if (!outcome) { + return; + } + + for (const id of outcome.haveIds) { + session.pendingHaveIds.add(id); + } + const newNeedIds: string[] = []; + for (const id of outcome.needIds) { + if (!session.pendingNeedIds.has(id)) { + session.pendingNeedIds.add(id); + newNeedIds.push(id); + } + } + syncStats.negentropyRounds += 1; + syncStats.negentropyHaveIds += outcome.haveIds.length; + syncStats.negentropyNeedIds += outcome.needIds.length; + + if (outcome.haveIds.length > 0) { + await sendOpsPushForIds(peerKey, session, outcome.haveIds); + } + + if (newNeedIds.length > 0) { + await sendOpsReq(peerKey, session, newNeedIds); + } + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + round: session.rounds, + have: outcome.haveIds.length, + need: outcome.needIds.length, + pendingNeed: session.pendingNeedIds.size, + }, + 'negentropy initiator round' + ); + + if (outcome.nextMsg !== null) { + if (!sendNegMsg(peerKey, session, outcome.nextMsg)) { + closePeerSession(peerKey, 'send_neg_msg_failed'); + } + return; + } + + session.reconciliationComplete = true; + const completedWindow = finalizeCurrentWindowStats(session, { completed: true, cappedByRounds: false }); + if (completedWindow) { + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId: session.windowId, + windowName: completedWindow.windowName, + loaded: completedWindow.loaded, + skipped: completedWindow.skipped, + rounds: completedWindow.rounds, + cappedByRecords: completedWindow.cappedByRecords, + }, + 'negentropy window complete (initiator)' + ); + } + await maybeFinalizeInitiatorSession(peerKey, session); +} + +async function handleNegentropyRoundAsResponder( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise { + const outcome = await reconcileNegentropyFrame(peerKey, session, frame); + if (!outcome) { + return; + } + syncStats.negentropyRounds += 1; + syncStats.negentropyHaveIds += outcome.haveIds.length; + syncStats.negentropyNeedIds += outcome.needIds.length; + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + round: session.rounds, + have: outcome.haveIds.length, + need: outcome.needIds.length, + }, + 'negentropy responder round' + ); + + if (outcome.nextMsg !== null) { + if (!sendNegMsg(peerKey, session, outcome.nextMsg)) { + closePeerSession(peerKey, 'send_neg_msg_failed'); + } + return; + } + + session.reconciliationComplete = true; + const completedWindow = finalizeCurrentWindowStats(session, { completed: true, cappedByRounds: false }); + if (completedWindow) { + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId: session.windowId, + windowName: completedWindow.windowName, + loaded: completedWindow.loaded, + skipped: completedWindow.skipped, + rounds: completedWindow.rounds, + cappedByRecords: completedWindow.cappedByRecords, + }, + 'negentropy window complete (responder)' + ); + } +} + function sendBatch(conn: HyperswarmConnection, batch: Operation[]): number { const limit = 8 * 1024 * 1014; // 8 MB limit @@ -172,6 +1339,7 @@ function sendBatch(conn: HyperswarmConnection, batch: Operation[]): number { const json = JSON.stringify(msg); if (json.length < limit) { + syncStats.bytesSent += messageBytes(json); conn.write(json); log.debug(` * sent ${batch.length} ops in ${json.length} bytes`); return batch.length; @@ -248,6 +1416,7 @@ async function relayMsg(msg: HyperMessage): Promise { const lastSeen = `last seen ${minutesSinceLastSeen} minutes ago ${last.toISOString()}`; if (!msg.relays.includes(peerKey)) { + syncStats.bytesSent += messageBytes(json); conn.connection.write(json); log.debug(`* relaying to: ${conn.peerName} (${conn.nodeName}) ${lastSeen} *`); } @@ -257,7 +1426,7 @@ async function relayMsg(msg: HyperMessage): Promise { } } -async function importBatch(batch: Operation[]): Promise { +async function importBatch(batch: Operation[]): Promise { // The batch we receive from other hyperswarm nodes includes just operations. // We have to wrap the operations in new events before submitting to our gatekeeper for importing try { @@ -282,10 +1451,36 @@ async function importBatch(batch: Operation[]): Promise { const importDurationMs = Date.now() - importStart; log.debug({ durationMs: importDurationMs }, 'importBatch'); log.debug(`* ${JSON.stringify(response)}`); + syncStats.opsRejected += response.rejected ?? 0; + + return filterIndexRejectedOperations(batch, response.rejectedIndices); } catch (error) { log.error({ error }, 'importBatch error'); + return []; + } +} + +async function persistAcceptedOperations(operations: Operation[], source: string): Promise { + if (!Array.isArray(operations) || operations.length === 0) { + return; + } + + const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); + if (records.length === 0) { + log.debug({ source, attempted: operations.length, invalid }, 'sync-store persist skipped'); + return; } + + const inserted = await syncStore.upsertMany(records); + if (inserted > 0) { + markNegentropyAdapterDirty(); + maybeStartBackgroundPrebuild(`persist_${source}`); + } + log.debug( + { source, attempted: operations.length, mapped: records.length, invalid, inserted }, + 'sync-store persist accepted ops' + ); } async function mergeBatch(batch: Operation[]): Promise { @@ -295,24 +1490,34 @@ async function mergeBatch(batch: Operation[]): Promise { } let chunk = []; + const acceptedCandidates: Operation[] = []; + for (const operation of batch) { chunk.push(operation); if (chunk.length >= BATCH_SIZE) { - await importBatch(chunk); + const candidates = await importBatch(chunk); + acceptedCandidates.push(...candidates); chunk = []; } } if (chunk.length > 0) { - await importBatch(chunk); + const candidates = await importBatch(chunk); + acceptedCandidates.push(...candidates); } const processStart = Date.now(); const response = await gatekeeper.processEvents(); const processDurationMs = Date.now() - processStart; log.debug({ durationMs: processDurationMs }, 'processEvents'); - log.debug(`mergeBatch: ${JSON.stringify(response)}`); + const { acceptedHashes: _acceptedHashes, ...processSummary } = response; + log.debug(`mergeBatch: ${JSON.stringify(processSummary)}`); + syncStats.opsApplied += (response.added ?? 0) + (response.merged ?? 0); + syncStats.opsRejected += response.rejected ?? 0; + + const acceptedToPersist = filterOperationsByAcceptedHashes(acceptedCandidates, response.acceptedHashes); + await persistAcceptedOperations(acceptedToPersist, 'mergeBatch'); } let importQueue = asyncLib.queue( @@ -328,6 +1533,14 @@ let importQueue = asyncLib.queue( return; } + if (msg.type === 'queue') { + syncStats.queueOpsImported += batch.length; + const samples = collectQueueDelaySamples(batch); + for (const sample of samples) { + addAggregateSample(syncStats.queueDelayMs, sample); + } + } + const nodeName = msg.node || 'anon'; log.debug(`* merging batch (${batch.length} events) from: ${shortName(name)} (${nodeName}) *`); await mergeBatch(batch); @@ -346,6 +1559,11 @@ let exportQueue = asyncLib.queue( const ready = await gatekeeper.isReady(); if (ready) { + const mode = connectionInfo[name]?.syncMode ?? 'unknown'; + if (!shouldAcceptLegacySync(mode, config.legacySyncEnabled)) { + log.debug({ peer: shortName(name), mode }, 'shareDb skipped by sync mode policy'); + return; + } log.debug(`* sharing db with: ${shortName(name)} (${msg.node || 'anon'}) *`); await shareDb(conn); } @@ -425,15 +1643,21 @@ async function addPeer(did: string): Promise { } } -async function receiveMsg(peerKey: string, json: string): Promise { +async function receiveMsg(peerKey: string, json: Buffer | string): Promise { const conn = connectionInfo[peerKey]; - let msg; + if (!conn) { + return; + } + + let msg: HyperMessage; + const payload = typeof json === 'string' ? json : json.toString('utf8'); + syncStats.bytesReceived += messageBytes(payload); try { - msg = JSON.parse(json); + msg = JSON.parse(payload); } catch (error) { - const jsonPreview = json.length > 80 ? `${json.slice(0, 40)}...${json.slice(-40)}` : json; + const jsonPreview = payload.length > 80 ? `${payload.slice(0, 40)}...${payload.slice(-40)}` : payload; log.warn(`received invalid message from: ${conn.peerName}, JSON: ${jsonPreview}`); return; } @@ -444,15 +1668,18 @@ async function receiveMsg(peerKey: string, json: string): Promise { connectionInfo[peerKey].lastSeen = new Date().getTime(); if (msg.type === 'batch') { - if (newBatch(msg.data)) { + if (Array.isArray(msg.data) && newBatch(msg.data)) { importQueue.push({ name: peerKey, msg }); } return; } if (msg.type === 'queue') { - if (newBatch(msg.data)) { + if (Array.isArray(msg.data) && newBatch(msg.data)) { importQueue.push({ name: peerKey, msg }); + if (!Array.isArray(msg.relays)) { + msg.relays = []; + } msg.relays.push(peerKey); await relayMsg(msg); } @@ -460,19 +1687,152 @@ async function receiveMsg(peerKey: string, json: string): Promise { } if (msg.type === 'sync') { + if (!shouldAcceptLegacySync(connectionInfo[peerKey].syncMode, config.legacySyncEnabled)) { + log.debug( + { peer: shortName(peerKey), mode: connectionInfo[peerKey].syncMode }, + 'ignoring legacy sync request' + ); + return; + } exportQueue.push({ name: peerKey, msg, conn: conn.connection }); return; } if (msg.type === 'ping') { connectionInfo[peerKey].nodeName = nodeName; + connectionInfo[peerKey].capabilities = normalizePeerCapabilities(msg.capabilities); - if (msg.peers) { + if (Array.isArray(msg.peers)) { for (const did of msg.peers) { addPeer(did); } } + await maybeStartPeerSync(peerKey); + return; + } + + if (msg.type === 'neg_open') { + if (!negentropyAdapter) { + log.warn('neg_open ignored because adapter is unavailable'); + return; + } + + const window = parseRemoteWindow(msg.window); + if (!window) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_open with invalid window'); + return; + } + if (typeof msg.windowId !== 'string' || msg.windowId.length === 0) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_open with invalid windowId'); + return; + } + + let session = peerSessions.get(peerKey); + if (session && session.sessionId !== msg.sessionId) { + closePeerSession(peerKey, 'replaced_by_remote_open'); + session = undefined; + } + + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + session = createPeerSession(peerKey, 'negentropy', false, msg.sessionId); + } + + session.initiator = false; + session.maxRounds = config.negentropyMaxRoundsPerSession; + const existingIndex = session.windows.findIndex(existingWindow => makeWindowId(existingWindow) === msg.windowId); + if (existingIndex >= 0) { + session.windows[existingIndex] = window; + session.windowIndex = existingIndex; + } else { + session.windows.push(window); + session.windowIndex = session.windows.length - 1; + } + const windowStats = await ensureWindowAdapterFresh(window, 'session_open_responder'); + initializeSessionWindowState(session, window, msg.windowId, windowStats); + touchPeerSession(peerKey); + await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); + return; + } + + if (msg.type === 'neg_msg') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_msg for unknown session'); + return; + } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'neg_msg')) { + return; + } + + touchPeerSession(peerKey); + if (session.initiator) { + await handleNegentropyRoundAsInitiator(peerKey, session, decodeNegentropyFrame(msg.frame)); + } else { + await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); + } + return; + } + + if (msg.type === 'ops_req') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_req for unknown session'); + return; + } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'ops_req')) { + return; + } + + const requestedIds = Array.isArray(msg.ids) + ? Array.from(new Set(msg.ids.map(id => String(id).toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))) + : []; + syncStats.negentropyOpsReqReceived += requestedIds.length; + await sendOpsPushForIds(peerKey, session, requestedIds); + touchPeerSession(peerKey); + return; + } + + if (msg.type === 'ops_push') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_push for unknown session'); + return; + } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'ops_push')) { + return; + } + + const batch = Array.isArray(msg.data) ? msg.data : []; + if (batch.length > 0) { + syncStats.negentropyOpsPushReceived += batch.length; + const pushedIds = new Set(extractOperationHashes(batch)); + for (const id of pushedIds) { + session.pendingNeedIds.delete(id); + } + + if (newBatch(batch)) { + importQueue.push({ + name: peerKey, + msg: { + ...createBaseMessage('batch'), + data: batch, + }, + }); + } + + await maybeFinalizeInitiatorSession(peerKey, session); + } + + touchPeerSession(peerKey); + return; + } + + if (msg.type === 'neg_close') { + const session = peerSessions.get(peerKey); + if (session && session.sessionId === msg.sessionId && (!session.windowId || msg.windowId === session.windowId)) { + closePeerSession(peerKey, msg.reason || 'remote_closed'); + } return; } @@ -481,9 +1841,15 @@ async function receiveMsg(peerKey: string, json: string): Promise { async function flushQueue(): Promise { const batch = await gatekeeper.getQueue(REGISTRY); - log.debug(`${REGISTRY} queue: ${JSON.stringify(batch, null, 4)}`); if (batch.length > 0) { + await persistAcceptedOperations(batch, 'flushQueue'); + syncStats.queueOpsRelayed += batch.length; + const samples = collectQueueDelaySamples(batch); + for (const sample of samples) { + addAggregateSample(syncStats.queueDelayMs, sample); + } + const msg: BatchMessage = { type: 'queue', time: new Date().toISOString(), @@ -522,6 +1888,8 @@ async function exportLoop(): Promise { } async function checkConnections(): Promise { + expireIdlePeerSessions(); + if (Object.keys(connectionInfo).length === 0) { log.warn("No active connections, rejoining the topic..."); await createSwarm(); @@ -537,7 +1905,7 @@ async function checkConnections(): Promise { if (timeSinceLastSeen > expireLimit) { log.info(`Removing stale connection info for: ${conn.peerName} (${conn.nodeName}), last seen ${timeSinceLastSeen / 1000}s ago`); - delete connectionInfo[peerKey]; + closeConnection(peerKey); } } } @@ -549,15 +1917,12 @@ async function connectionLoop(): Promise { await checkConnections(); - const msg: PingMessage = { - type: 'ping', - time: new Date().toISOString(), - node: nodeInfo.name, - relays: [], - peers: Object.keys(knownNodes), - }; + const msg: PingMessage = buildPingMessage(); await relayMsg(msg); + await runPeriodicNegentropyRepair(); + + log.debug({ syncStats: buildSyncStatsSnapshot() }, 'hyperswarm sync stats'); if (config.ipfsEnabled) { const peeringPeers = await ipfs.getPeeringPeers(); @@ -594,6 +1959,8 @@ const networkID = Buffer.from(hash).toString('hex'); const topic = Buffer.from(b4a.from(networkID, 'hex')); async function main(): Promise { + await syncStore.start(); + await gatekeeper.connect({ url: config.gatekeeperURL, waitUntilReady: true, @@ -601,6 +1968,11 @@ async function main(): Promise { chatty: true, }); + const bootstrap = await bootstrapSyncStoreIfEmpty(syncStore, gatekeeper); + log.info({ bootstrap }, 'sync-store bootstrap complete'); + + await initNegentropyAdapter(); + if (config.ipfsEnabled) { await keymaster.connect({ url: config.keymasterURL, @@ -663,4 +2035,65 @@ async function main(): Promise { await connectionLoop(); } -main(); +async function initNegentropyAdapter(): Promise { + if (!config.negentropyEnabled) { + negentropyAdapter = null; + adapterChangeSeq = 0; + adapterBuiltSeq = -1; + adapterBuiltAt = 0; + adapterBuiltWindowId = null; + adapterBuiltWindowStats = null; + rebuildPromise = null; + backgroundPrebuildQueued = false; + log.info('negentropy disabled via KC_HYPR_NEGENTROPY_ENABLE; using legacy sync mode when available'); + return; + } + + negentropyAdapter = await NegentropyAdapter.create({ + syncStore, + frameSizeLimit: config.negentropyFrameSizeLimit, + recentWindowDays: config.negentropyWindowDays, + olderWindowDays: config.negentropyWindowDays, + maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, + maxRoundsPerSession: config.negentropyMaxRoundsPerSession, + deferInitialBuild: true, + }); + adapterChangeSeq = 0; + adapterBuiltSeq = -1; + adapterBuiltAt = 0; + adapterBuiltWindowId = null; + adapterBuiltWindowStats = null; + rebuildPromise = null; + backgroundPrebuildQueued = false; + log.info( + { + stats: negentropyAdapter.getStats(), + windowDays: config.negentropyWindowDays, + maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, + maxRoundsPerSession: config.negentropyMaxRoundsPerSession, + frameSizeLimit: config.negentropyFrameSizeLimit, + }, + 'negentropy adapter initialized' + ); +} + +export async function runMediator(options: MediatorMainOptions = {}): Promise { + if (options.syncStore) { + syncStore = options.syncStore; + } + + if (options.startLoops === false) { + await syncStore.start(); + return; + } + + return main(); +} + +const isDirectRun = !!process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; +if (isDirectRun) { + runMediator().catch(error => { + log.error({ error }, 'fatal mediator error'); + process.exit(1); + }); +} diff --git a/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs b/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs new file mode 100644 index 00000000..ae802bbe --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs @@ -0,0 +1,589 @@ +// (C) 2023 Doug Hoyte. MIT license + +const PROTOCOL_VERSION = 0x61; // Version 1 +const ID_SIZE = 32; +const FINGERPRINT_SIZE = 16; + +const Mode = { + Skip: 0, + Fingerprint: 1, + IdList: 2, +}; + +class WrappedBuffer { + constructor(buffer) { + this._raw = new Uint8Array(buffer || 512); + this.length = buffer ? buffer.length : 0; + } + + unwrap() { + return this._raw.subarray(0, this.length); + } + + get capacity() { + return this._raw.byteLength; + } + + extend(buf) { + if (buf._raw) buf = buf.unwrap(); + if (typeof(buf.length) !== 'number') throw Error("bad length"); + const targetSize = buf.length + this.length; + if (this.capacity < targetSize) { + const oldRaw = this._raw; + const newCapacity = Math.max(this.capacity * 2, targetSize); + this._raw = new Uint8Array(newCapacity); + this._raw.set(oldRaw); + } + + this._raw.set(buf, this.length); + this.length += buf.length; + } + + shift() { + const first = this._raw[0]; + this._raw = this._raw.subarray(1); + this.length--; + return first; + } + + shiftN(n = 1) { + const firstSubarray = this._raw.subarray(0, n); + this._raw = this._raw.subarray(n); + this.length -= n; + return firstSubarray; + } +} + +function decodeVarInt(buf) { + let res = 0; + + while (1) { + if (buf.length === 0) throw Error("parse ends prematurely"); + let byte = buf.shift(); + res = (res << 7) | (byte & 127); + if ((byte & 128) === 0) break; + } + + return res; +} + +function encodeVarInt(n) { + if (n === 0) return new WrappedBuffer([0]); + + let o = []; + + while (n !== 0) { + o.push(n & 127); + n >>>= 7; + } + + o.reverse(); + + for (let i = 0; i < o.length - 1; i++) o[i] |= 128; + + return new WrappedBuffer(o); +} + +function getByte(buf) { + return getBytes(buf, 1)[0]; +} + +function getBytes(buf, n) { + if (buf.length < n) throw Error("parse ends prematurely"); + return buf.shiftN(n); +} + + +class Accumulator { + constructor() { + this.setToZero(); + + if (typeof window === 'undefined') { // node.js + const crypto = require('crypto'); + this.sha256 = async (slice) => new Uint8Array(crypto.createHash('sha256').update(slice).digest()); + } else { // browser + this.sha256 = async (slice) => new Uint8Array(await crypto.subtle.digest("SHA-256", slice)); + } + } + + setToZero() { + this.buf = new Uint8Array(ID_SIZE); + } + + add(otherBuf) { + let currCarry = 0, nextCarry = 0; + let p = new DataView(this.buf.buffer); + let po = new DataView(otherBuf.buffer); + + for (let i = 0; i < 8; i++) { + let offset = i * 4; + let orig = p.getUint32(offset, true); + let otherV = po.getUint32(offset, true); + + let next = orig; + + next += currCarry; + next += otherV; + if (next > 0xFFFFFFFF) nextCarry = 1; + + p.setUint32(offset, next & 0xFFFFFFFF, true); + currCarry = nextCarry; + nextCarry = 0; + } + } + + negate() { + let p = new DataView(this.buf.buffer); + + for (let i = 0; i < 8; i++) { + let offset = i * 4; + p.setUint32(offset, ~p.getUint32(offset, true)); + } + + let one = new Uint8Array(ID_SIZE); + one[0] = 1; + this.add(one); + } + + async getFingerprint(n) { + let input = new WrappedBuffer(); + input.extend(this.buf); + input.extend(encodeVarInt(n)); + + let hash = await this.sha256(input.unwrap()); + + return hash.subarray(0, FINGERPRINT_SIZE); + } +}; + + +class NegentropyStorageVector { + constructor() { + this.items = []; + this.sealed = false; + } + + insert(timestamp, id) { + if (this.sealed) throw Error("already sealed"); + id = loadInputBuffer(id); + if (id.byteLength !== ID_SIZE) throw Error("bad id size for added item"); + this.items.push({ timestamp, id }); + } + + seal() { + if (this.sealed) throw Error("already sealed"); + this.sealed = true; + + this.items.sort(itemCompare); + + for (let i = 1; i < this.items.length; i++) { + if (itemCompare(this.items[i - 1], this.items[i]) === 0) throw Error("duplicate item inserted"); + } + } + + unseal() { + this.sealed = false; + } + + size() { + this._checkSealed(); + return this.items.length; + } + + getItem(i) { + this._checkSealed(); + if (i >= this.items.length) throw Error("out of range"); + return this.items[i]; + } + + iterate(begin, end, cb) { + this._checkSealed(); + this._checkBounds(begin, end); + + for (let i = begin; i < end; ++i) { + if (!cb(this.items[i], i)) break; + } + } + + findLowerBound(begin, end, bound) { + this._checkSealed(); + this._checkBounds(begin, end); + + return this._binarySearch(this.items, begin, end, (a) => itemCompare(a, bound) < 0); + } + + async fingerprint(begin, end) { + let out = new Accumulator(); + out.setToZero(); + + this.iterate(begin, end, (item, i) => { + out.add(item.id); + return true; + }); + + return await out.getFingerprint(end - begin); + } + + _checkSealed() { + if (!this.sealed) throw Error("not sealed"); + } + + _checkBounds(begin, end) { + if (begin > end || end > this.items.length) throw Error("bad range"); + } + + _binarySearch(arr, first, last, cmp) { + let count = last - first; + + while (count > 0) { + let it = first; + let step = Math.floor(count / 2); + it += step; + + if (cmp(arr[it])) { + first = ++it; + count -= step + 1; + } else { + count = step; + } + } + + return first; + } +} + + +class Negentropy { + constructor(storage, frameSizeLimit = 0) { + if (frameSizeLimit !== 0 && frameSizeLimit < 4096) throw Error("frameSizeLimit too small"); + + this.storage = storage; + this.frameSizeLimit = frameSizeLimit; + + this.lastTimestampIn = 0; + this.lastTimestampOut = 0; + } + + _bound(timestamp, id) { + return { timestamp, id: id ? id : new Uint8Array(0) }; + } + + async initiate() { + if (this.isInitiator) throw Error("already initiated"); + this.isInitiator = true; + + let output = new WrappedBuffer(); + output.extend([ PROTOCOL_VERSION ]); + + await this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output); + + return this._renderOutput(output); + } + + setInitiator() { + this.isInitiator = true; + } + + async reconcile(query) { + let haveIds = [], needIds = []; + query = new WrappedBuffer(loadInputBuffer(query)); + + this.lastTimestampIn = this.lastTimestampOut = 0; // reset for each message + + let fullOutput = new WrappedBuffer(); + fullOutput.extend([ PROTOCOL_VERSION ]); + + let protocolVersion = getByte(query); + if (protocolVersion < 0x60 || protocolVersion > 0x6F) throw Error("invalid negentropy protocol version byte"); + if (protocolVersion !== PROTOCOL_VERSION) { + if (this.isInitiator) throw Error("unsupported negentropy protocol version requested: " + (protocolVersion - 0x60)); + else return [this._renderOutput(fullOutput), haveIds, needIds]; + } + + let storageSize = this.storage.size(); + let prevBound = this._bound(0); + let prevIndex = 0; + let skip = false; + + while (query.length !== 0) { + let o = new WrappedBuffer(); + + let doSkip = () => { + if (skip) { + skip = false; + o.extend(this.encodeBound(prevBound)); + o.extend(encodeVarInt(Mode.Skip)); + } + }; + + let currBound = this.decodeBound(query); + let mode = decodeVarInt(query); + + let lower = prevIndex; + let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound); + + if (mode === Mode.Skip) { + skip = true; + } else if (mode === Mode.Fingerprint) { + let theirFingerprint = getBytes(query, FINGERPRINT_SIZE); + let ourFingerprint = await this.storage.fingerprint(lower, upper); + + if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) { + doSkip(); + await this.splitRange(lower, upper, currBound, o); + } else { + skip = true; + } + } else if (mode === Mode.IdList) { + let numIds = decodeVarInt(query); + + let theirElems = {}; // stringified Uint8Array -> original Uint8Array (or hex) + for (let i = 0; i < numIds; i++) { + let e = getBytes(query, ID_SIZE); + if (this.isInitiator) theirElems[e] = e; + } + + if (this.isInitiator) { + skip = true; + + this.storage.iterate(lower, upper, (item) => { + let k = item.id; + + if (!theirElems[k]) { + // ID exists on our side, but not their side + if (this.isInitiator) haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k)); + } else { + // ID exists on both sides + delete theirElems[k]; + } + + return true; + }); + + for (let v of Object.values(theirElems)) { + // ID exists on their side, but not our side + needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v)); + } + } else { + doSkip(); + + let responseIds = new WrappedBuffer(); + let numResponseIds = 0; + let endBound = currBound; + + this.storage.iterate(lower, upper, (item, index) => { + if (this.exceededFrameSizeLimit(fullOutput.length + responseIds.length)) { + endBound = item; + upper = index; // shrink upper so that remaining range gets correct fingerprint + return false; + } + + responseIds.extend(item.id); + numResponseIds++; + return true; + }); + + o.extend(this.encodeBound(endBound)); + o.extend(encodeVarInt(Mode.IdList)); + o.extend(encodeVarInt(numResponseIds)); + o.extend(responseIds); + + fullOutput.extend(o); + o = new WrappedBuffer(); + } + } else { + throw Error("unexpected mode"); + } + + if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) { + // frameSizeLimit exceeded: Stop range processing and return a fingerprint for the remaining range + let remainingFingerprint = await this.storage.fingerprint(upper, storageSize); + + fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE))); + fullOutput.extend(encodeVarInt(Mode.Fingerprint)); + fullOutput.extend(remainingFingerprint); + break; + } else { + fullOutput.extend(o); + } + + prevIndex = upper; + prevBound = currBound; + } + + return [fullOutput.length === 1 && this.isInitiator ? null : this._renderOutput(fullOutput), haveIds, needIds]; + } + + async splitRange(lower, upper, upperBound, o) { + let numElems = upper - lower; + let buckets = 16; + + if (numElems < buckets * 2) { + o.extend(this.encodeBound(upperBound)); + o.extend(encodeVarInt(Mode.IdList)); + + o.extend(encodeVarInt(numElems)); + this.storage.iterate(lower, upper, (item) => { + o.extend(item.id); + return true; + }); + } else { + let itemsPerBucket = Math.floor(numElems / buckets); + let bucketsWithExtra = numElems % buckets; + let curr = lower; + + for (let i = 0; i < buckets; i++) { + let bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0); + let ourFingerprint = await this.storage.fingerprint(curr, curr + bucketSize); + curr += bucketSize; + + let nextBound; + + if (curr === upper) { + nextBound = upperBound; + } else { + let prevItem, currItem; + + this.storage.iterate(curr - 1, curr + 1, (item, index) => { + if (index === curr - 1) prevItem = item; + else currItem = item; + return true; + }); + + nextBound = this.getMinimalBound(prevItem, currItem); + } + + o.extend(this.encodeBound(nextBound)); + o.extend(encodeVarInt(Mode.Fingerprint)); + o.extend(ourFingerprint); + } + } + } + + _renderOutput(o) { + o = o.unwrap(); + if (!this.wantUint8ArrayOutput) o = uint8ArrayToHex(o); + return o; + } + + exceededFrameSizeLimit(n) { + return this.frameSizeLimit && n > this.frameSizeLimit - 200; + } + + // Decoding + + decodeTimestampIn(encoded) { + let timestamp = decodeVarInt(encoded); + timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1; + if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) { + this.lastTimestampIn = Number.MAX_VALUE; + return Number.MAX_VALUE; + } + timestamp += this.lastTimestampIn; + this.lastTimestampIn = timestamp; + return timestamp; + } + + decodeBound(encoded) { + let timestamp = this.decodeTimestampIn(encoded); + let len = decodeVarInt(encoded); + if (len > ID_SIZE) throw Error("bound key too long"); + let id = getBytes(encoded, len); + return { timestamp, id }; + } + + // Encoding + + encodeTimestampOut(timestamp) { + if (timestamp === Number.MAX_VALUE) { + this.lastTimestampOut = Number.MAX_VALUE; + return encodeVarInt(0); + } + + let temp = timestamp; + timestamp -= this.lastTimestampOut; + this.lastTimestampOut = temp; + return encodeVarInt(timestamp + 1); + } + + encodeBound(key) { + let output = new WrappedBuffer(); + + output.extend(this.encodeTimestampOut(key.timestamp)); + output.extend(encodeVarInt(key.id.length)); + output.extend(key.id); + + return output; + } + + getMinimalBound(prev, curr) { + if (curr.timestamp !== prev.timestamp) { + return this._bound(curr.timestamp); + } else { + let sharedPrefixBytes = 0; + let currKey = curr.id; + let prevKey = prev.id; + + for (let i = 0; i < ID_SIZE; i++) { + if (currKey[i] !== prevKey[i]) break; + sharedPrefixBytes++; + } + + return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1)); + } + }; +} + +function loadInputBuffer(inp) { + if (typeof(inp) === 'string') inp = hexToUint8Array(inp); + else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp); // node Buffer? + return inp; +} + +function hexToUint8Array(h) { + if (h.startsWith('0x')) h = h.substr(2); + if (h.length % 2 === 1) throw Error("odd length of hex string"); + let arr = new Uint8Array(h.length / 2); + for (let i = 0; i < arr.length; i++) arr[i] = parseInt(h.substr(i * 2, 2), 16); + return arr; +} + +const uint8ArrayToHexLookupTable = new Array(256); +{ + const hexAlphabet = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + for (let i = 0; i < 256; i++) { + uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xF] + hexAlphabet[i & 0xF]; + } +} + +function uint8ArrayToHex(arr) { + let out = ''; + for (let i = 0, edx = arr.length; i < edx; i++) { + out += uint8ArrayToHexLookupTable[arr[i]]; + } + return out; +} + + +function compareUint8Array(a, b) { + for (let i = 0; i < a.byteLength; i++) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + + if (a.byteLength > b.byteLength) return 1; + if (a.byteLength < b.byteLength) return -1; + + return 0; +} + +function itemCompare(a, b) { + if (a.timestamp === b.timestamp) { + return compareUint8Array(a.id, b.id); + } + + return a.timestamp - b.timestamp; +} + + +module.exports = { Negentropy, NegentropyStorageVector, }; diff --git a/services/mediators/hyperswarm/src/negentropy/adapter.ts b/services/mediators/hyperswarm/src/negentropy/adapter.ts new file mode 100644 index 00000000..f7b1cfe4 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/adapter.ts @@ -0,0 +1,480 @@ +import { createRequire } from 'module'; +import { childLogger } from '@mdip/common/logger'; +import type { SyncStoreCursor } from '../db/types.js'; +import type { OperationSyncStore } from '../db/types.js'; + +const log = childLogger({ service: 'hyperswarm-negentropy-adapter' }); + +const MIN_FRAME_SIZE_LIMIT = 4096; +const DEFAULT_ITERATE_LIMIT = 1000; +const DEFAULT_RECENT_WINDOW_DAYS = 7; +const DEFAULT_OLDER_WINDOW_DAYS = 30; +const DEFAULT_MAX_RECORDS_PER_WINDOW = 25_000; +const DEFAULT_MAX_ROUNDS_PER_SESSION = 64; +const DAY_SECONDS = 24 * 60 * 60; +const require = createRequire(import.meta.url); + +interface NegentropyInstance { + wantUint8ArrayOutput?: boolean; + initiate(): Promise; + reconcile( + msg: string | Uint8Array + ): Promise<[string | Uint8Array | null, Array, Array]>; +} + +interface NegentropyStorageVectorInstance { + insert(timestamp: number, id: string | Uint8Array | Buffer): void; + seal(): void; +} + +interface NegentropyModule { + Negentropy: new (storage: NegentropyStorageVectorInstance, frameSizeLimit?: number) => NegentropyInstance; + NegentropyStorageVector: new () => NegentropyStorageVectorInstance; +} + +export interface NegentropyAdapterOptions { + syncStore: OperationSyncStore; + frameSizeLimit?: number; + iterateLimit?: number; + wantUint8ArrayOutput?: boolean; + recentWindowDays?: number; + olderWindowDays?: number; + maxRecordsPerWindow?: number; + maxRoundsPerSession?: number; + deferInitialBuild?: boolean; +} + +export interface ReconciliationWindow { + name: string; + fromTs: number; + toTs: number; + maxRecords: number; + order: number; +} + +export interface NegentropyReconcileResult { + nextMsg: string | Uint8Array | null; + haveIds: Array; + needIds: Array; +} + +export interface NegentropyAdapterStats { + loaded: number; + skipped: number; + durationMs: number; + frameSizeLimit: number; +} + +export interface NegentropyWindowStats extends NegentropyAdapterStats { + windowName: string; + fromTs: number; + toTs: number; + rounds: number; + completed: boolean; + cappedByRecords: boolean; + cappedByRounds: boolean; +} + +export interface NegentropySessionStats { + windowCount: number; + rounds: number; + loaded: number; + skipped: number; + durationMs: number; + windows: NegentropyWindowStats[]; +} + +export interface NegentropyWindowSessionOptions { + nowTs?: number; + nowMs?: number; + maxRoundsPerSession?: number; +} + +export default class NegentropyAdapter { + private readonly syncStore: OperationSyncStore; + private readonly frameSizeLimit: number; + private readonly iterateLimit: number; + private readonly wantUint8ArrayOutput: boolean; + private readonly recentWindowDays: number; + private readonly olderWindowDays: number; + private readonly maxRecordsPerWindow: number; + private readonly maxRoundsPerSession: number; + private readonly mod: NegentropyModule; + private ne: NegentropyInstance | null = null; + private stats: NegentropyAdapterStats = { + loaded: 0, + skipped: 0, + durationMs: 0, + frameSizeLimit: 0, + }; + private lastWindowStats: NegentropyWindowStats | null = null; + private lastSessionStats: NegentropySessionStats | null = null; + + static async create(options: NegentropyAdapterOptions): Promise { + const adapter = new NegentropyAdapter(options); + if (!options.deferInitialBuild) { + await adapter.rebuildFromStore(); + } + return adapter; + } + + constructor(options: NegentropyAdapterOptions) { + this.syncStore = options.syncStore; + this.frameSizeLimit = options.frameSizeLimit ?? 0; + this.iterateLimit = options.iterateLimit ?? DEFAULT_ITERATE_LIMIT; + this.wantUint8ArrayOutput = options.wantUint8ArrayOutput ?? false; + this.recentWindowDays = options.recentWindowDays ?? DEFAULT_RECENT_WINDOW_DAYS; + this.olderWindowDays = options.olderWindowDays ?? DEFAULT_OLDER_WINDOW_DAYS; + this.maxRecordsPerWindow = options.maxRecordsPerWindow ?? DEFAULT_MAX_RECORDS_PER_WINDOW; + this.maxRoundsPerSession = options.maxRoundsPerSession ?? DEFAULT_MAX_ROUNDS_PER_SESSION; + this.mod = loadNegentropyModule(); + + if (this.frameSizeLimit !== 0 && this.frameSizeLimit < MIN_FRAME_SIZE_LIMIT) { + throw new Error(`negentropy frameSizeLimit must be 0 or >= ${MIN_FRAME_SIZE_LIMIT}`); + } + + assertPositiveInteger(this.iterateLimit, 'iterateLimit'); + assertPositiveInteger(this.recentWindowDays, 'recentWindowDays'); + assertPositiveInteger(this.olderWindowDays, 'olderWindowDays'); + assertPositiveInteger(this.maxRecordsPerWindow, 'maxRecordsPerWindow'); + assertPositiveInteger(this.maxRoundsPerSession, 'maxRoundsPerSession'); + } + + async rebuildFromStore(): Promise { + const stats = await this.rebuildWindowAdapter({ + name: 'full_history', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: Number.MAX_SAFE_INTEGER, + order: 0, + }); + + return { + loaded: stats.loaded, + skipped: stats.skipped, + durationMs: stats.durationMs, + frameSizeLimit: stats.frameSizeLimit, + }; + } + + async rebuildForWindow(window: ReconciliationWindow): Promise { + return this.rebuildWindowAdapter(window); + } + + async planWindows(nowTs: number = currentEpochSeconds(), earliestTsOverride?: number): Promise { + if (!Number.isFinite(nowTs)) { + throw new Error('nowTs must be a finite timestamp'); + } + + const earliestTs = typeof earliestTsOverride === 'number' + ? earliestTsOverride + : await this.getEarliestTimestamp(); + + if (earliestTs == null) { + return []; + } + + const windows: ReconciliationWindow[] = []; + const recentSpanTs = this.recentWindowDays * DAY_SECONDS; + const olderSpanTs = this.olderWindowDays * DAY_SECONDS; + const recentStart = Math.max(earliestTs, nowTs - recentSpanTs); + + windows.push({ + name: 'recent', + fromTs: recentStart, + toTs: nowTs, + maxRecords: this.maxRecordsPerWindow, + order: 0, + }); + + let cursorTo = recentStart - 1; + let order = 1; + while (cursorTo >= earliestTs) { + const fromTs = Math.max(earliestTs, cursorTo - olderSpanTs + 1); + windows.push({ + name: `older_${order}`, + fromTs, + toTs: cursorTo, + maxRecords: this.maxRecordsPerWindow, + order, + }); + cursorTo = fromTs - 1; + order += 1; + } + + return windows; + } + + async runWindowedSessionWithPeer( + peer: NegentropyAdapter, + options: NegentropyWindowSessionOptions = {}, + ): Promise { + const startedAt = Date.now(); + const sessionNowTs = options.nowTs ?? options.nowMs ?? currentEpochSeconds(); + const maxRoundsPerSession = options.maxRoundsPerSession ?? this.maxRoundsPerSession; + assertPositiveInteger(maxRoundsPerSession, 'maxRoundsPerSession'); + + const [localEarliest, peerEarliest] = await Promise.all([ + this.getEarliestTimestamp(), + peer.getEarliestTimestamp(), + ]); + const earliestCandidates = [localEarliest, peerEarliest] + .filter((value): value is number => typeof value === 'number'); + + const windows = earliestCandidates.length === 0 + ? [] + : await this.planWindows(sessionNowTs, Math.min(...earliestCandidates)); + + const windowStats: NegentropyWindowStats[] = []; + let totalLoaded = 0; + let totalSkipped = 0; + let totalRounds = 0; + + for (const window of windows) { + const localWindowStats = await this.rebuildForWindow(window); + await peer.rebuildForWindow(window); + + const roundsResult = await this.reconcileWindowWithPeer(peer, maxRoundsPerSession); + const mergedStats: NegentropyWindowStats = { + ...localWindowStats, + rounds: roundsResult.rounds, + completed: roundsResult.completed, + cappedByRounds: roundsResult.cappedByRounds, + }; + + windowStats.push(mergedStats); + totalLoaded += mergedStats.loaded; + totalSkipped += mergedStats.skipped; + totalRounds += mergedStats.rounds; + + (log as any).debug?.( + { + windowName: mergedStats.windowName, + fromTs: mergedStats.fromTs, + toTs: mergedStats.toTs, + loaded: mergedStats.loaded, + skipped: mergedStats.skipped, + durationMs: mergedStats.durationMs, + rounds: mergedStats.rounds, + completed: mergedStats.completed, + cappedByRecords: mergedStats.cappedByRecords, + cappedByRounds: mergedStats.cappedByRounds, + }, + 'negentropy window session' + ); + } + + const sessionStats: NegentropySessionStats = { + windowCount: windowStats.length, + rounds: totalRounds, + loaded: totalLoaded, + skipped: totalSkipped, + durationMs: Date.now() - startedAt, + windows: windowStats, + }; + + this.lastSessionStats = sessionStats; + (log as any).debug?.( + { + windowCount: sessionStats.windowCount, + rounds: sessionStats.rounds, + loaded: sessionStats.loaded, + skipped: sessionStats.skipped, + durationMs: sessionStats.durationMs, + }, + 'negentropy windowed session summary' + ); + return sessionStats; + } + + getStats(): NegentropyAdapterStats { + return { ...this.stats }; + } + + getLastWindowStats(): NegentropyWindowStats | null { + return this.lastWindowStats ? { ...this.lastWindowStats } : null; + } + + getLastSessionStats(): NegentropySessionStats | null { + if (!this.lastSessionStats) { + return null; + } + + return { + ...this.lastSessionStats, + windows: this.lastSessionStats.windows.map(window => ({ ...window })), + }; + } + + async initiate(): Promise { + if (!this.ne) { + throw new Error('negentropy adapter not initialized'); + } + return this.ne.initiate(); + } + + async reconcile(msg: string | Uint8Array): Promise { + if (!this.ne) { + throw new Error('negentropy adapter not initialized'); + } + + const [nextMsg, haveIds, needIds] = await this.ne.reconcile(msg); + return { nextMsg, haveIds, needIds }; + } + + async respond(msg: string | Uint8Array): Promise { + const result = await this.reconcile(msg); + return result.nextMsg; + } + + private async rebuildWindowAdapter(window: ReconciliationWindow): Promise { + const startedAt = Date.now(); + const storage = new this.mod.NegentropyStorageVector(); + + let loaded = 0; + let skipped = 0; + let processed = 0; + let cappedByRecords = false; + let after: SyncStoreCursor | undefined; + + while (true) { + const rows = await this.syncStore.iterateSorted({ + after, + limit: this.iterateLimit, + fromTs: window.fromTs, + toTs: window.toTs, + }); + + if (rows.length === 0) { + break; + } + + for (const row of rows) { + if (processed >= window.maxRecords) { + cappedByRecords = true; + break; + } + + processed += 1; + if (!isValidSyncId(row.id) || !Number.isFinite(row.ts)) { + skipped += 1; + continue; + } + + storage.insert(row.ts, row.id); + loaded += 1; + } + + const last = rows[rows.length - 1]; + after = { ts: last.ts, id: last.id }; + + if (cappedByRecords) { + break; + } + } + + storage.seal(); + this.ne = new this.mod.Negentropy(storage, this.frameSizeLimit); + if (this.wantUint8ArrayOutput) { + this.ne.wantUint8ArrayOutput = true; + } + + const durationMs = Date.now() - startedAt; + this.stats = { + loaded, + skipped, + durationMs, + frameSizeLimit: this.frameSizeLimit, + }; + + const windowStats: NegentropyWindowStats = { + ...this.stats, + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + rounds: 0, + completed: true, + cappedByRecords, + cappedByRounds: false, + }; + + this.lastWindowStats = windowStats; + + (log as any).debug?.( + { + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + loaded, + skipped, + durationMs, + cappedByRecords, + frameSizeLimit: this.frameSizeLimit, + }, + 'negentropy adapter rebuilt' + ); + + return windowStats; + } + + private async getEarliestTimestamp(): Promise { + const rows = await this.syncStore.iterateSorted({ limit: 1 }); + if (rows.length === 0) { + return null; + } + return rows[0].ts; + } + + private async reconcileWindowWithPeer( + peer: NegentropyAdapter, + maxRounds: number, + ): Promise<{ rounds: number; completed: boolean; cappedByRounds: boolean }> { + let msg: string | Uint8Array | null = await this.initiate(); + let rounds = 0; + + while (msg !== null && rounds < maxRounds) { + rounds += 1; + const response = await peer.respond(msg); + if (response === null) { + msg = null; + break; + } + + const reconcileResult = await this.reconcile(response); + msg = reconcileResult.nextMsg; + } + + const completed = msg === null; + return { + rounds, + completed, + cappedByRounds: !completed && rounds >= maxRounds, + }; + } +} + +function currentEpochSeconds(): number { + return Math.floor(Date.now() / 1000); +} + +function isValidSyncId(id: string): boolean { + return typeof id === 'string' && /^[a-f0-9]{64}$/i.test(id); +} + +function assertPositiveInteger(value: number, name: string): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function loadNegentropyModule(): NegentropyModule { + const cjs = require('./Negentropy.cjs'); + const Negentropy = cjs?.Negentropy; + const NegentropyStorageVector = cjs?.NegentropyStorageVector; + + if (typeof Negentropy !== 'function' || typeof NegentropyStorageVector !== 'function') { + throw new Error('Invalid local negentropy module exports'); + } + + return { Negentropy, NegentropyStorageVector }; +} diff --git a/services/mediators/hyperswarm/src/negentropy/observability.ts b/services/mediators/hyperswarm/src/negentropy/observability.ts new file mode 100644 index 00000000..a4875c6b --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/observability.ts @@ -0,0 +1,78 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface AggregateMetric { + count: number; + total: number; + max: number; +} + +export function createAggregateMetric(): AggregateMetric { + return { + count: 0, + total: 0, + max: 0, + }; +} + +export function addAggregateSample(metric: AggregateMetric, sample: number): void { + if (!Number.isFinite(sample) || sample < 0) { + return; + } + + metric.count += 1; + metric.total += sample; + if (sample > metric.max) { + metric.max = sample; + } +} + +export function averageAggregate(metric: AggregateMetric): number { + if (metric.count === 0) { + return 0; + } + + return metric.total / metric.count; +} + +export function messageBytes(payload: string | Buffer | object): number { + if (typeof payload === 'string') { + return Buffer.byteLength(payload, 'utf8'); + } + + if (Buffer.isBuffer(payload)) { + return payload.byteLength; + } + + return Buffer.byteLength(JSON.stringify(payload), 'utf8'); +} + +export function collectQueueDelaySamples(operations: Operation[], nowMs: number = Date.now()): number[] { + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + const samples: number[] = []; + for (const operation of operations) { + const signed = operation.signature?.signed; + if (typeof signed !== 'string' || signed === '') { + continue; + } + + const ts = Date.parse(signed); + if (!Number.isFinite(ts)) { + continue; + } + + samples.push(Math.max(0, nowMs - ts)); + } + + return samples; +} + +export function safeRate(numerator: number, denominator: number): number { + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) { + return 0; + } + + return numerator / denominator; +} diff --git a/services/mediators/hyperswarm/src/negentropy/policy.ts b/services/mediators/hyperswarm/src/negentropy/policy.ts new file mode 100644 index 00000000..f9d7a1f7 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/policy.ts @@ -0,0 +1,64 @@ +import type { SyncMode } from './protocol.js'; + +export interface RepairSchedulingInput { + syncMode: SyncMode | 'unknown'; + hasActiveSession: boolean; + importQueueLength: number; + activeNegentropySessions: number; + lastAttemptAtMs: number; + nowMs: number; + repairIntervalMs: number; + isInitiator: boolean; + syncCompleted: boolean; +} + +export function shouldAcceptLegacySync( + syncMode: SyncMode | 'unknown', + legacySyncEnabled: boolean, +): boolean { + if (!legacySyncEnabled) { + return false; + } + + return syncMode === 'legacy'; +} + +export function shouldStartConnectTimeNegentropy( + syncMode: SyncMode | 'unknown', + hasActiveSession: boolean, + isInitiator: boolean, +): boolean { + return syncMode === 'negentropy' && !hasActiveSession && isInitiator; +} + +export function shouldSchedulePeriodicRepair(input: RepairSchedulingInput): boolean { + if (input.syncMode !== 'negentropy') { + return false; + } + + if (!input.isInitiator) { + return false; + } + + if (input.hasActiveSession) { + return false; + } + + if (input.importQueueLength > 0) { + return false; + } + + if (input.activeNegentropySessions > 0) { + return false; + } + + if (input.syncCompleted) { + return false; + } + + if (input.lastAttemptAtMs <= 0) { + return true; + } + + return (input.nowMs - input.lastAttemptAtMs) >= input.repairIntervalMs; +} diff --git a/services/mediators/hyperswarm/src/negentropy/protocol.ts b/services/mediators/hyperswarm/src/negentropy/protocol.ts new file mode 100644 index 00000000..43cd9d7b --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/protocol.ts @@ -0,0 +1,151 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export type SyncMode = 'legacy' | 'negentropy'; +export type NegentropyFrameEncoding = 'utf8' | 'base64'; +export const NEG_SYNC_ID_RE = /^[a-f0-9]{64}$/i; + +export interface PeerCapabilities { + negentropy?: boolean; + negentropyVersion?: number; +} + +export interface NegotiatedPeerCapabilities { + advertised: boolean; + negentropy: boolean; + version: number | null; +} + +export type ConnectSyncModeReason = + | 'negentropy_supported' + | 'missing_capabilities' + | 'negentropy_disabled' + | 'version_mismatch' + | 'legacy_disabled'; + +export interface ConnectSyncModeDecision { + mode: SyncMode | null; + reason: ConnectSyncModeReason; +} + +export interface NegentropyFrame { + encoding: NegentropyFrameEncoding; + data: string; +} + +export function normalizePeerCapabilities(capabilities?: PeerCapabilities): NegotiatedPeerCapabilities { + if (!capabilities) { + return { + advertised: false, + negentropy: false, + version: null, + }; + } + + return { + advertised: true, + negentropy: capabilities.negentropy === true, + version: typeof capabilities.negentropyVersion === 'number' + ? capabilities.negentropyVersion + : null, + }; +} + +export function supportsPeerNegentropy( + capabilities: NegotiatedPeerCapabilities, + minVersion: number +): boolean { + return capabilities.advertised + && capabilities.negentropy + && (capabilities.version == null || capabilities.version >= minVersion); +} + +export function chooseSyncMode( + capabilities: NegotiatedPeerCapabilities, + minVersion: number +): SyncMode | null { + if (!capabilities.advertised) { + return null; + } + + return supportsPeerNegentropy(capabilities, minVersion) + ? 'negentropy' + : 'legacy'; +} + +export function chooseConnectSyncMode( + capabilities: NegotiatedPeerCapabilities, + minVersion: number, + legacySyncEnabled: boolean, + negentropyEnabled = true, +): ConnectSyncModeDecision { + if (negentropyEnabled && supportsPeerNegentropy(capabilities, minVersion)) { + return { mode: 'negentropy', reason: 'negentropy_supported' }; + } + + if (!legacySyncEnabled) { + return { mode: null, reason: 'legacy_disabled' }; + } + + if (!negentropyEnabled) { + return { mode: 'legacy', reason: 'negentropy_disabled' }; + } + + if (!capabilities.advertised) { + return { mode: 'legacy', reason: 'missing_capabilities' }; + } + + if (!capabilities.negentropy) { + return { mode: 'legacy', reason: 'negentropy_disabled' }; + } + + return { mode: 'legacy', reason: 'version_mismatch' }; +} + +export function encodeNegentropyFrame(frame: string | Uint8Array): NegentropyFrame { + if (typeof frame === 'string') { + return { + encoding: 'utf8', + data: frame, + }; + } + + return { + encoding: 'base64', + data: Buffer.from(frame).toString('base64'), + }; +} + +export function decodeNegentropyFrame(frame: NegentropyFrame): string | Uint8Array { + if (frame.encoding === 'utf8') { + return frame.data; + } + + return Buffer.from(frame.data, 'base64'); +} + +export function normalizeNegentropyIds(ids: Array): string[] { + const unique = new Set(); + for (const id of ids) { + const hex = typeof id === 'string' + ? id.toLowerCase() + : Buffer.from(id).toString('hex').toLowerCase(); + + if (NEG_SYNC_ID_RE.test(hex)) { + unique.add(hex); + } + } + + return Array.from(unique); +} + +export function extractOperationHashes(operations: Operation[]): string[] { + const unique = new Set(); + for (const operation of operations) { + const hash = operation.signature?.hash?.toLowerCase(); + if (hash && NEG_SYNC_ID_RE.test(hash)) { + unique.add(hash); + } + } + + return Array.from(unique); +} diff --git a/services/mediators/hyperswarm/src/negentropy/transfer.ts b/services/mediators/hyperswarm/src/negentropy/transfer.ts new file mode 100644 index 00000000..be86a4d6 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/transfer.ts @@ -0,0 +1,70 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface OpsPushBatchingOptions { + maxOpsPerPush: number; + maxBytesPerPush: number; +} + +export function chunkIds(ids: string[], maxPerChunk: number): string[][] { + if (!Number.isInteger(maxPerChunk) || maxPerChunk <= 0) { + throw new Error('maxPerChunk must be a positive integer'); + } + + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const unique = Array.from(new Set(ids)); + const chunks: string[][] = []; + for (let i = 0; i < unique.length; i += maxPerChunk) { + chunks.push(unique.slice(i, i + maxPerChunk)); + } + return chunks; +} + +export function chunkOperationsForPush( + operations: Operation[], + options: OpsPushBatchingOptions, +): Operation[][] { + const { maxOpsPerPush, maxBytesPerPush } = options; + if (!Number.isInteger(maxOpsPerPush) || maxOpsPerPush <= 0) { + throw new Error('maxOpsPerPush must be a positive integer'); + } + + if (!Number.isInteger(maxBytesPerPush) || maxBytesPerPush <= 0) { + throw new Error('maxBytesPerPush must be a positive integer'); + } + + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + const batches: Operation[][] = []; + let current: Operation[] = []; + let currentBytes = 0; + + for (const operation of operations) { + const operationBytes = estimateOperationBytes(operation); + const exceedsCount = current.length >= maxOpsPerPush; + const exceedsBytes = current.length > 0 && (currentBytes + operationBytes) > maxBytesPerPush; + + if (exceedsCount || exceedsBytes) { + batches.push(current); + current = []; + currentBytes = 0; + } + + current.push(operation); + currentBytes += operationBytes; + } + + if (current.length > 0) { + batches.push(current); + } + + return batches; +} + +export function estimateOperationBytes(operation: Operation): number { + return Buffer.byteLength(JSON.stringify(operation), 'utf8'); +} diff --git a/services/mediators/hyperswarm/src/stubs.d.ts b/services/mediators/hyperswarm/src/stubs.d.ts index 64a900ff..143be4d2 100644 --- a/services/mediators/hyperswarm/src/stubs.d.ts +++ b/services/mediators/hyperswarm/src/stubs.d.ts @@ -12,6 +12,18 @@ declare module 'hyperswarm' { destroy?(): void; } + export interface HyperswarmOptions { + bootstrap?: string[]; + dht?: any; + keyPair?: { publicKey: Buffer; secretKey?: Buffer }; + seed?: Buffer; + maxPeers?: number; + firewall?: (remotePublicKey: Buffer) => boolean; + debug?: boolean; + backoffs?: number[]; + jitter?: number; + } + interface Discovery { flushed(): Promise; } diff --git a/services/mediators/hyperswarm/src/sync-mapping.ts b/services/mediators/hyperswarm/src/sync-mapping.ts new file mode 100644 index 00000000..ed1fb400 --- /dev/null +++ b/services/mediators/hyperswarm/src/sync-mapping.ts @@ -0,0 +1,98 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +// - id = operation.signature.hash (64 hex chars => 32 bytes) +// - timestamp = Math.floor(Date.parse(operation.signature.signed) / 1000) in epoch seconds +// - if timestamp is before MDIP epoch, clamp it to MDIP epoch (2024-01-01T00:00:00Z) +export const SYNC_ID_HEX_LEN = 64; +export const SYNC_ID_BYTES_LEN = 32; +export const MDIP_EPOCH_SECONDS = 1_704_067_200; // 2024-01-01T00:00:00Z + +export interface SyncMappedOperation { + idHex: string; + idBytes: Buffer; + tsSec: number; + operation: Operation; +} + +export type SyncMappingErrorCode = + | 'missing_signature' + | 'missing_signature_hash' + | 'invalid_signature_hash_type' + | 'invalid_signature_hash_format' + | 'missing_signature_signed' + | 'invalid_signature_signed_type' + | 'invalid_signature_signed_value'; + +export interface SyncMappingFailure { + ok: false; + code: SyncMappingErrorCode; + reason: string; +} + +export interface SyncMappingSuccess { + ok: true; + value: SyncMappedOperation; +} + +export type SyncMappingResult = SyncMappingFailure | SyncMappingSuccess; + +function fail(code: SyncMappingErrorCode, reason: string): SyncMappingFailure { + return { ok: false, code, reason }; +} + +export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { + const signature = operation.signature; + + if (!signature) { + return fail('missing_signature', 'operation.signature is required'); + } + + if (signature.hash == null || signature.hash === '') { + return fail('missing_signature_hash', 'operation.signature.hash is required'); + } + + if (typeof signature.hash !== 'string') { + return fail('invalid_signature_hash_type', 'operation.signature.hash must be a string'); + } + + const idHex = signature.hash.toLowerCase(); + if (!/^[a-f0-9]{64}$/.test(idHex)) { + return fail( + 'invalid_signature_hash_format', + `operation.signature.hash must be ${SYNC_ID_HEX_LEN} hex characters` + ); + } + + if (signature.signed == null || signature.signed === '') { + return fail('missing_signature_signed', 'operation.signature.signed is required'); + } + + if (typeof signature.signed !== 'string') { + return fail('invalid_signature_signed_type', 'operation.signature.signed must be a string'); + } + + const parsedSigned = Date.parse(signature.signed); + if (!Number.isFinite(parsedSigned)) { + return fail('invalid_signature_signed_value', 'operation.signature.signed must be parseable by Date.parse'); + } + const tsSec = Math.floor(parsedSigned / 1000); + const normalizedTsSec = tsSec < MDIP_EPOCH_SECONDS ? MDIP_EPOCH_SECONDS : tsSec; + + const idBytes = Buffer.from(idHex, 'hex'); + if (idBytes.length !== SYNC_ID_BYTES_LEN) { + return fail( + 'invalid_signature_hash_format', + `operation.signature.hash must decode to ${SYNC_ID_BYTES_LEN} bytes` + ); + } + + return { + ok: true, + value: { + idHex, + idBytes, + tsSec: normalizedTsSec, + operation, + }, + }; +} diff --git a/services/mediators/hyperswarm/src/sync-persistence.ts b/services/mediators/hyperswarm/src/sync-persistence.ts new file mode 100644 index 00000000..53ca45ad --- /dev/null +++ b/services/mediators/hyperswarm/src/sync-persistence.ts @@ -0,0 +1,81 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { mapOperationToSyncKey } from './sync-mapping.js'; + +export interface SyncPersistenceRecord { + id: string; + ts: number; + operation: Operation; +} + +export interface MapAcceptedResult { + records: SyncPersistenceRecord[]; + invalid: number; +} + +export function filterIndexRejectedOperations(batch: Operation[], rejectedIndices: number[] = []): Operation[] { + if (!Array.isArray(batch) || batch.length === 0) { + return []; + } + + if (!Array.isArray(rejectedIndices) || rejectedIndices.length === 0) { + return [...batch]; + } + + const rejectedSet = new Set(); + for (const index of rejectedIndices) { + if (Number.isInteger(index) && index >= 0 && index < batch.length) { + rejectedSet.add(index); + } + } + + return batch.filter((_operation, index) => !rejectedSet.has(index)); +} + +export function filterOperationsByAcceptedHashes( + operations: Operation[], + acceptedHashes: string[] = [], +): Operation[] { + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + if (!Array.isArray(acceptedHashes) || acceptedHashes.length === 0) { + return []; + } + + const acceptedSet = new Set( + acceptedHashes + .filter((hash): hash is string => hash !== '') + .map(hash => hash.toLowerCase()) + ); + + if (acceptedSet.size === 0) { + return []; + } + + return operations.filter(operation => { + const hash = operation.signature?.hash; + return typeof hash === 'string' && acceptedSet.has(hash.toLowerCase()); + }); +} + +export function mapAcceptedOperationsToSyncRecords(operations: Operation[]): MapAcceptedResult { + const records: SyncPersistenceRecord[] = []; + let invalid = 0; + + for (const operation of operations) { + const mapped = mapOperationToSyncKey(operation); + if (!mapped.ok) { + invalid += 1; + continue; + } + + records.push({ + id: mapped.value.idHex, + ts: mapped.value.tsSec, + operation, + }); + } + + return { records, invalid }; +} diff --git a/start-node b/start-node index 26dec449..0f0a031b 100755 --- a/start-node +++ b/start-node @@ -11,11 +11,13 @@ if [ -f .env ]; then set +a fi -if [ "${KC_IPFS_ENABLE,,}" != "false" ]; then +KC_IPFS_ENABLE_LC="$(printf '%s' "${KC_IPFS_ENABLE:-}" | tr '[:upper:]' '[:lower:]')" + +if [ "$KC_IPFS_ENABLE_LC" != "false" ]; then PROFILE_ARGS=(--profile ipfs) fi -if [ ${#SERVICES[@]} -gt 0 ] && [ "${KC_IPFS_ENABLE,,}" != "false" ]; then +if [ ${#SERVICES[@]} -gt 0 ] && [ "$KC_IPFS_ENABLE_LC" != "false" ]; then if ! printf '%s\n' "${SERVICES[@]}" | grep -qx "ipfs"; then SERVICES=("ipfs" "${SERVICES[@]}") fi diff --git a/test.sh b/test.sh new file mode 100755 index 00000000..b06ab961 --- /dev/null +++ b/test.sh @@ -0,0 +1,12 @@ +SERVICE=hypr-mediator +CID=$(docker compose ps -q "$SERVICE") +if [ -z "$CID" ]; then + echo "No running container for service: $SERVICE" + docker compose ps + exit 1 +fi + +while :; do + echo "$(date -Is),$(docker stats --no-stream --format '{{.MemUsage}}' "$CID")" + sleep 1 +done | tee /tmp/${SERVICE}-mem.csv diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index 4713d565..1b3e3ae1 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -47,6 +47,13 @@ KC_KEYMASTER_URL=http://localhost:4226 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 +KC_HYPR_NEGENTROPY_ENABLE=true +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 +KC_HYPR_NEGENTROPY_WINDOW_DAYS=30 +KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 +KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 +KC_HYPR_NEGENTROPY_INTERVAL=300 +KC_HYPR_LEGACY_SYNC_ENABLE=true KC_MDIP_PROTOCOL=/MDIP/testing # Bitcoin mediator diff --git a/tests/gatekeeper/client.test.ts b/tests/gatekeeper/client.test.ts index 93a42d2b..317279aa 100644 --- a/tests/gatekeeper/client.test.ts +++ b/tests/gatekeeper/client.test.ts @@ -1,7 +1,6 @@ import nock from 'nock'; import GatekeeperClient from '@mdip/gatekeeper/client'; import { ExpectedExceptionError } from '@mdip/common/errors'; -import {Operation} from "@mdip/gatekeeper/types"; const GatekeeperURL = 'http://gatekeeper.org'; const ServerError = { message: 'Server error' }; @@ -469,13 +468,19 @@ describe('importDIDs', () => { it('should return imported DID results', async () => { nock(GatekeeperURL) .post(Endpoints.dids.import) - .reply(200, { queued: 0, processed: 0 }); + .reply(200, { queued: 0, processed: 0, rejected: 0, total: 0, rejectedIndices: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); // @ts-expect-error Testing without arguments const results = await gatekeeper.importDIDs(); - expect(results).toStrictEqual({ queued: 0, processed: 0 }); + expect(results).toStrictEqual({ + queued: 0, + processed: 0, + rejected: 0, + total: 0, + rejectedIndices: [], + }); }); it('should throw exception on importDIDs server error', async () => { @@ -529,13 +534,19 @@ describe('importBatch', () => { it('should return imported batch results', async () => { nock(GatekeeperURL) .post(Endpoints.batch.import) - .reply(200, { queued: 0, processed: 0 }); + .reply(200, { queued: 0, processed: 0, rejected: 0, total: 0, rejectedIndices: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); // @ts-expect-error Testing without arguments const results = await gatekeeper.importBatch(); - expect(results).toStrictEqual({ queued: 0, processed: 0 }); + expect(results).toStrictEqual({ + queued: 0, + processed: 0, + rejected: 0, + total: 0, + rejectedIndices: [], + }); }); it('should throw exception on importBatch server error', async () => { @@ -628,12 +639,12 @@ describe('processEvents', () => { it('should return process status', async () => { nock(GatekeeperURL) .post(Endpoints.events.process) - .reply(200, { added: 0, merged: 0, pending: 0 }); + .reply(200, { added: 0, merged: 0, pending: 0, acceptedHashes: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); const status = await gatekeeper.processEvents(); - expect(status).toStrictEqual({ added: 0, merged: 0, pending: 0 }); + expect(status).toStrictEqual({ added: 0, merged: 0, pending: 0, acceptedHashes: [] }); }); it('should throw exception on processEvents server error', async () => { diff --git a/tests/gatekeeper/sync.test.ts b/tests/gatekeeper/sync.test.ts index 44d50c64..b63b92eb 100644 --- a/tests/gatekeeper/sync.test.ts +++ b/tests/gatekeeper/sync.test.ts @@ -146,6 +146,25 @@ describe('importDIDs', () => { expect(response.merged).toBe(1); }); + + it('should report rejectedIndices using dids.flat() order', async () => { + const keypair = cipher.generateRandomJwk(); + const op1 = await helper.createAgentOp(keypair); + const did1 = await gatekeeper.createDID(op1); + const did1Events = await gatekeeper.exportDIDs([did1]); + + const op2 = await helper.createAgentOp(keypair); + const did2 = await gatekeeper.createDID(op2); + const did2Events = await gatekeeper.exportDIDs([did2]); + + delete did2Events[0][0].operation.signature; + + const response = await gatekeeper.importDIDs([did1Events[0], did2Events[0]]); + + expect(response.queued).toBe(1); + expect(response.rejected).toBe(1); + expect(response.rejectedIndices).toStrictEqual([1]); + }); }); describe('removeDIDs', () => { @@ -295,6 +314,7 @@ describe('importBatch', () => { const response = await gatekeeper.importBatch([1, 2, 3]); expect(response.rejected).toBe(3); + expect(response.rejectedIndices).toStrictEqual([0, 1, 2]); }); it('should report an error on invalid event time', async () => { @@ -498,6 +518,22 @@ describe('importBatch', () => { expect(response.queued).toBe(1); expect(response.rejected).toBe(1); + expect(response.rejectedIndices).toStrictEqual([1]); + }); + + it('should report sparse rejectedIndices in original submitted order', async () => { + const keypair = cipher.generateRandomJwk(); + const agentOp = await helper.createAgentOp(keypair); + const did = await gatekeeper.createDID(agentOp); + const ops = await gatekeeper.exportDID(did); + + const invalid1 = { ...ops[0], time: 'invalid' } as any; + const invalid2 = { ...ops[0], operation: { ...ops[0].operation, type: 'bad-type' } } as any; + const response = await gatekeeper.importBatch([invalid1, ops[0], invalid2]); + + expect(response.queued).toBe(1); + expect(response.rejected).toBe(2); + expect(response.rejectedIndices).toStrictEqual([0, 2]); }); it('should report an error on invalid update operation missing did', async () => { @@ -547,6 +583,7 @@ describe('processEvents', () => { const response = await gatekeeper.processEvents(); expect(response.merged).toBe(1); + expect(response.acceptedHashes).toContain(ops[0].operation.signature!.hash.toLowerCase()); }); it('should import a valid asset DID export', async () => { @@ -921,6 +958,7 @@ describe('processEvents', () => { const response = await gatekeeper.processEvents(); expect(response.added).toBe(1); expect(response.rejected).toBe(2); + expect(response.acceptedHashes).toStrictEqual([updateOp1.signature!.hash.toLowerCase()]); }); it('should handle a reorg event', async () => { diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts new file mode 100644 index 00000000..8d4406bd --- /dev/null +++ b/tests/hyperswarm/bootstrap.test.ts @@ -0,0 +1,167 @@ +import { jest } from '@jest/globals'; +import { GatekeeperEvent, Operation } from '@mdip/gatekeeper/types'; +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import { bootstrapSyncStoreIfEmpty } from '../../services/mediators/hyperswarm/src/bootstrap.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOperation(hashChar: string, signed: string): Operation { + return { + type: 'create', + signature: { + hash: h(hashChar), + signed, + value: `sig-${hashChar}`, + }, + }; +} + +function makeEvent(operation: Operation): GatekeeperEvent { + return { + registry: 'hyperswarm', + time: new Date().toISOString(), + operation, + }; +} + +function makeDid(index: number): string { + return `did:test:${index}`; +} + +describe('bootstrapSyncStoreIfEmpty', () => { + it('skips rebuild when store drift is within configured percentage tolerance', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + // eslint-disable-next-line sonarjs/no-duplicate-string + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + await store.upsertMany([{ + id: h('a'), + // eslint-disable-next-line sonarjs/no-duplicate-string + ts: Math.floor(Date.parse('2026-02-10T10:00:00.000Z') / 1000), + operation: opA, + }]); + + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), + exportBatch: jest.fn(async () => [makeEvent(opA)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(true); + expect(result.reason).toBe('store_within_drift_tolerance'); + expect(result.driftPct).toBe(0); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledWith([makeDid(1)]); + expect(await store.count()).toBe(1); + }); + + it('rebuilds populated store when drift percentage exceeds threshold', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + await store.upsertMany([{ + id: h('a'), + ts: Math.floor(Date.parse('2026-02-10T10:00:00.000Z') / 1000), + operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), + }]); + + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1), makeDid(2)]), + exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(false); + expect(result.countBefore).toBe(1); + expect(result.countAfter).toBe(2); + expect(result.driftPct).toBeGreaterThan(result.driftThresholdPct); + expect(result.inserted).toBe(2); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(2); + expect(await store.count()).toBe(2); + }); + + it('bootstraps from gatekeeper exportBatch when store is empty', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1), makeDid(2)]), + exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(false); + expect(result.exported).toBe(2); + expect(result.mapped).toBe(2); + expect(result.invalid).toBe(0); + expect(result.inserted).toBe(2); + expect(result.countAfter).toBe(2); + expect(result.driftPct).toBe(1); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledWith([makeDid(1), makeDid(2)]); + expect(await store.count()).toBe(2); + }); + + it('throws when gatekeeper exportBatch fails', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), + exportBatch: jest.fn(async () => { + throw new Error('boom'); + }), + }; + + await expect(bootstrapSyncStoreIfEmpty(store, gatekeeper)).rejects.toThrow('boom'); + }); + + it('handles empty/invalid export payload without upserting', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), + exportBatch: jest.fn(async () => [{ registry: 'hyperswarm', time: new Date().toISOString() }]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper as any); + expect(result.skipped).toBe(false); + expect(result.exported).toBe(0); + expect(result.mapped).toBe(0); + expect(result.invalid).toBe(0); + expect(result.inserted).toBe(0); + expect(result.countAfter).toBe(0); + expect(result.driftPct).toBe(0); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + }); + + it('exports in DID batches to avoid loading the full export in one payload', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const dids = Array.from({ length: 501 }, (_, index) => makeDid(index + 1)); + const gatekeeper = { + getDIDs: jest.fn(async () => dids), + exportBatch: jest.fn(async (batchDids?: string[]) => { + return (batchDids ?? []).map((_, index) => + makeEvent(makeOperation(index % 2 === 0 ? 'a' : 'b', '2026-02-10T10:00:00.000Z')) + ); + }), + }; + + await bootstrapSyncStoreIfEmpty(store, gatekeeper); + + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(2); + expect(gatekeeper.exportBatch).toHaveBeenNthCalledWith(1, dids.slice(0, 500)); + expect(gatekeeper.exportBatch).toHaveBeenNthCalledWith(2, dids.slice(500, 501)); + }); +}); diff --git a/tests/hyperswarm/config.test.ts b/tests/hyperswarm/config.test.ts new file mode 100644 index 00000000..52aa8769 --- /dev/null +++ b/tests/hyperswarm/config.test.ts @@ -0,0 +1,92 @@ +import { jest } from '@jest/globals'; + +const CONFIG_PATH = '../../services/mediators/hyperswarm/src/config.js'; +const ORIGINAL_ENV = { ...process.env }; + +async function importConfigIsolated() { + let loaded: any; + await jest.isolateModulesAsync(async () => { + loaded = (await import(CONFIG_PATH)).default; + }); + return loaded; +} + +describe('hyperswarm config', () => { + afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + jest.resetModules(); + }); + + it('throws when both negentropy and legacy sync are disabled', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'false'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'false'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow( + 'Invalid sync configuration; at least one of KC_HYPR_NEGENTROPY_ENABLE or KC_HYPR_LEGACY_SYNC_ENABLE must be true' + ); + }); + + it('allows startup when at least one sync mode is enabled', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'false'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'true'; + + const config = await importConfigIsolated(); + + expect(config.negentropyEnabled).toBe(false); + expect(config.legacySyncEnabled).toBe(true); + }); + + it('uses defaults when optional env vars are empty', async () => { + process.env.KC_HYPR_EXPORT_INTERVAL = ''; + process.env.KC_HYPR_NEGENTROPY_ENABLE = ''; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = ''; + + const config = await importConfigIsolated(); + + expect(config.exportInterval).toBe(2); + expect(config.negentropyEnabled).toBe(true); + expect(config.legacySyncEnabled).toBe(true); + }); + + it('throws on invalid positive integer env values', async () => { + process.env.KC_HYPR_EXPORT_INTERVAL = '0'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('Invalid KC_HYPR_EXPORT_INTERVAL; expected a positive integer'); + }); + + it('throws when frame size limit is non-zero and below minimum', async () => { + process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '2'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4 (KB)'); + }); + + it('interprets frame size limit env value as KB', async () => { + process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '4'; + + const config = await importConfigIsolated(); + expect(config.negentropyFrameSizeLimit).toBe(4096); + }); + + it('throws on invalid boolean env values', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'maybe'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'true'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('Invalid KC_HYPR_NEGENTROPY_ENABLE; expected true or false'); + }); +}); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts new file mode 100644 index 00000000..2b61f840 --- /dev/null +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -0,0 +1,417 @@ +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import NegentropyAdapter from '../../services/mediators/hyperswarm/src/negentropy/adapter.ts'; +import { Operation } from '@mdip/gatekeeper/types'; +import type { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from '../../services/mediators/hyperswarm/src/db/types.ts'; + +const DAY_SECONDS = 24 * 60 * 60; +const h = (c: string) => c.repeat(64); +const idFromNum = (n: number) => n.toString(16).padStart(64, '0'); +const toEpochSeconds = (iso: string) => Math.floor(Date.parse(iso) / 1000); +const toISOFromEpochSeconds = (ts: number) => new Date(ts * 1000).toISOString(); + +function makeOp(hashChar: string, signed: string): Operation { + return { + type: 'create', + signature: { + signed, + hash: h(hashChar), + value: `sig-${hashChar}`, + }, + }; +} + +function makeOpFromHash(hash: string, signed: string): Operation { + return { + type: 'create', + signature: { + signed, + hash, + value: `sig-${hash.slice(0, 8)}`, + }, + }; +} + +async function seedStore( + store: InMemoryOperationSyncStore, + records: Array<{ id: string; ts: number; op: Operation }> +): Promise { + await store.start(); + await store.reset(); + await store.upsertMany(records.map(item => ({ + id: item.id, + ts: item.ts, + operation: item.op, + }))); +} + +async function seedNumericRange( + store: InMemoryOperationSyncStore, + start: number, + endExclusive: number, + baseTs: number, +): Promise { + const records = []; + for (let i = start; i < endExclusive; i++) { + const id = idFromNum(i); + records.push({ + id, + ts: baseTs + i, + op: makeOpFromHash(id, toISOFromEpochSeconds(baseTs + i)), + }); + } + await seedStore(store, records); +} + +describe('NegentropyAdapter', () => { + it('returns null session/window stats before any rebuilds', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + expect(adapter.getLastWindowStats()).toBeNull(); + expect(adapter.getLastSessionStats()).toBeNull(); + }); + + it('loads and builds from store', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, [ + // eslint-disable-next-line sonarjs/no-duplicate-string + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + // eslint-disable-next-line sonarjs/no-duplicate-string + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + }); + + const stats = adapter.getStats(); + expect(stats.loaded).toBe(2); + expect(stats.skipped).toBe(0); + expect(stats.durationMs).toBeGreaterThanOrEqual(0); + }); + + it('reconciles two stores and reports have/need ids', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + await seedStore(storeA, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + // eslint-disable-next-line sonarjs/no-duplicate-string + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + ]); + + await seedStore(storeB, [ + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + { id: h('d'), ts: 4000, op: makeOp('d', '2026-02-09T13:00:00.000Z') }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + }); + + let msg: string | Uint8Array | null = await adapterA.initiate(); + const have = new Set(); + const need = new Set(); + + for (let i = 0; i < 20 && msg !== null; i++) { + const response = await adapterB.respond(msg); + if (response === null) { + break; + } + const round = await adapterA.reconcile(response); + + for (const id of round.haveIds) { + have.add(String(id)); + } + for (const id of round.needIds) { + need.add(String(id)); + } + + msg = round.nextMsg; + } + + expect(have.has(h('a'))).toBe(true); + expect(need.has(h('d'))).toBe(true); + }); + + it('throws when frameSizeLimit is invalid', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 1024, + })).rejects.toThrow('frameSizeLimit'); + }); + + it('throws when integer options are invalid', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + recentWindowDays: 0, + })).rejects.toThrow('recentWindowDays'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + olderWindowDays: 0, + })).rejects.toThrow('olderWindowDays'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + maxRecordsPerWindow: 0, + })).rejects.toThrow('maxRecordsPerWindow'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + maxRoundsPerSession: 0, + })).rejects.toThrow('maxRoundsPerSession'); + }); + + it('throws if initiate/reconcile called before adapter has been built', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapter.initiate()).rejects.toThrow('not initialized'); + await expect(adapter.reconcile('msg')).rejects.toThrow('not initialized'); + }); + + it('returns empty windows for empty store and rejects non-finite nowTs', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapter.planWindows(Number.NaN)).rejects.toThrow('nowTs must be a finite timestamp'); + await expect(adapter.planWindows(Math.floor(Date.now() / 1000))).resolves.toStrictEqual([]); + }); + + it('skips invalid sync rows when rebuilding a window', async () => { + const validId = h('a'); + const validTs = toEpochSeconds('2026-02-13T00:00:00.000Z'); + const validOp = makeOp('a', '2026-02-13T00:00:00.000Z'); + const rows: SyncOperationRecord[] = [ + { id: 'invalid-id', ts: validTs, operation: validOp, insertedAt: 1 }, + { id: validId, ts: Number.NaN, operation: validOp, insertedAt: 2 }, + { id: validId, ts: validTs, operation: validOp, insertedAt: 3 }, + ]; + + const stubStore: OperationSyncStore = { + start: async () => undefined, + stop: async () => undefined, + reset: async () => undefined, + upsertMany: async () => 0, + getByIds: async () => [], + has: async () => false, + count: async () => rows.length, + iterateSorted: async (options: SyncStoreListOptions = {}) => { + if (options.after) { + return []; + } + return rows; + }, + }; + + const adapter = await NegentropyAdapter.create({ + syncStore: stubStore, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + const stats = await adapter.rebuildForWindow({ + name: 'recent', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: 100, + order: 0, + }); + + expect(stats.loaded).toBe(1); + expect(stats.skipped).toBe(2); + }); + + it('plans recent window first then older windows in descending recency', async () => { + const store = new InMemoryOperationSyncStore(); + // eslint-disable-next-line sonarjs/no-duplicate-string + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); + await seedStore(store, [ + { id: h('a'), ts: nowTs - (10 * DAY_SECONDS), op: makeOp('a', '2026-01-31T00:00:00.000Z') }, + { id: h('b'), ts: nowTs - (2 * DAY_SECONDS), op: makeOp('b', '2026-02-08T00:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + recentWindowDays: 3, + olderWindowDays: 2, + deferInitialBuild: true, + }); + + const windows = await adapter.planWindows(nowTs); + expect(windows.length).toBeGreaterThanOrEqual(2); + expect(windows[0].name).toBe('recent'); + expect(windows[0].toTs).toBe(nowTs); + expect(windows[1].fromTs).toBeLessThanOrEqual(windows[1].toTs); + expect(windows[1].toTs).toBe(windows[0].fromTs - 1); + }); + + it('caps records per window and emits window stats', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + const stats = await adapter.rebuildForWindow({ + name: 'recent', + fromTs: 0, + toTs: 5000, + maxRecords: 2, + order: 0, + }); + + expect(stats.loaded).toBe(2); + expect(stats.skipped).toBe(0); + expect(stats.cappedByRecords).toBe(true); + expect(stats.windowName).toBe('recent'); + }); + + it('applies maxRoundsPerSession cap in windowed sessions', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + const baseTs = toEpochSeconds('2026-02-01T00:00:00.000Z'); + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); + + await seedNumericRange(storeA, 0, 2000, baseTs); + await seedNumericRange(storeB, 1000, 3000, baseTs); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 4096, + recentWindowDays: 14, + olderWindowDays: 14, + maxRecordsPerWindow: 10000, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 4096, + recentWindowDays: 14, + olderWindowDays: 14, + maxRecordsPerWindow: 10000, + deferInitialBuild: true, + }); + + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { + nowTs, + maxRoundsPerSession: 1, + }); + + expect(session.windowCount).toBeGreaterThan(0); + expect(session.windows.some(window => window.cappedByRounds)).toBe(true); + expect(session.windows[0].rounds).toBe(1); + }); + + it('throws for invalid runWindowedSessionWithPeer maxRoundsPerSession option', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + await seedStore(storeA, []); + await seedStore(storeB, []); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapterA.runWindowedSessionWithPeer(adapterB, { + maxRoundsPerSession: 0, + })).rejects.toThrow('maxRoundsPerSession'); + }); + + it('continues into older windows when a newer window hits record cap', async () => { + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + const recentA = nowTs - (6 * 60 * 60); + const recentB = nowTs - (8 * 60 * 60); + const older = nowTs - (2 * DAY_SECONDS); + + await seedStore(storeA, [ + { id: h('a'), ts: recentA, op: makeOp('a', toISOFromEpochSeconds(recentA)) }, + { id: h('b'), ts: recentB, op: makeOp('b', toISOFromEpochSeconds(recentB)) }, + { id: h('c'), ts: older, op: makeOp('c', toISOFromEpochSeconds(older)) }, + ]); + await seedStore(storeB, [ + { id: h('d'), ts: recentA, op: makeOp('d', toISOFromEpochSeconds(recentA)) }, + { id: h('e'), ts: older, op: makeOp('e', toISOFromEpochSeconds(older)) }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + recentWindowDays: 1, + olderWindowDays: 1, + maxRecordsPerWindow: 1, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + recentWindowDays: 1, + olderWindowDays: 1, + maxRecordsPerWindow: 1, + deferInitialBuild: true, + }); + + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { nowTs }); + const recent = session.windows.find(window => window.windowName === 'recent'); + const olderWindow = session.windows.find(window => window.windowName === 'older_1'); + + expect(session.windowCount).toBeGreaterThanOrEqual(2); + expect(recent?.cappedByRecords).toBe(true); + expect(olderWindow).toBeDefined(); + }); +}); diff --git a/tests/hyperswarm/negentropy-observability.test.ts b/tests/hyperswarm/negentropy-observability.test.ts new file mode 100644 index 00000000..945af92d --- /dev/null +++ b/tests/hyperswarm/negentropy-observability.test.ts @@ -0,0 +1,71 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + addAggregateSample, + averageAggregate, + collectQueueDelaySamples, + createAggregateMetric, + messageBytes, + safeRate, +} from '../../services/mediators/hyperswarm/src/negentropy/observability.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(signed?: string): Operation { + return { + type: 'create', + signature: signed ? { + hash: h('a'), + signed, + value: 'sig', + } : undefined, + }; +} + +describe('negentropy observability helpers', () => { + it('returns zero average for an empty aggregate', () => { + expect(averageAggregate(createAggregateMetric())).toBe(0); + }); + + it('aggregates samples and computes average', () => { + const metric = createAggregateMetric(); + addAggregateSample(metric, 10); + addAggregateSample(metric, 20); + addAggregateSample(metric, -5); + + expect(metric).toStrictEqual({ + count: 2, + total: 30, + max: 20, + }); + expect(averageAggregate(metric)).toBe(15); + }); + + it('collects queue delay samples from signed operations', () => { + const now = Date.parse('2026-02-13T10:00:00.000Z'); + const samples = collectQueueDelaySamples([ + makeOp('2026-02-13T09:59:00.000Z'), + makeOp('invalid-date'), + makeOp(), + ], now); + + expect(samples).toStrictEqual([60_000]); + }); + + it('returns empty samples for empty input', () => { + expect(collectQueueDelaySamples([])).toStrictEqual([]); + }); + + it('computes message bytes for string, buffer, and object', () => { + expect(messageBytes('abc')).toBe(3); + expect(messageBytes(Buffer.from([1, 2, 3]))).toBe(3); + expect(messageBytes({ hello: 'world' })).toBeGreaterThan(0); + }); + + it('returns safe rate for invalid denominator', () => { + expect(safeRate(Number.NaN, 10)).toBe(0); + expect(safeRate(10, Number.NaN)).toBe(0); + expect(safeRate(1, 0)).toBe(0); + expect(safeRate(1, -1)).toBe(0); + expect(safeRate(2, 4)).toBe(0.5); + }); +}); diff --git a/tests/hyperswarm/negentropy-policy.test.ts b/tests/hyperswarm/negentropy-policy.test.ts new file mode 100644 index 00000000..8e8d85a9 --- /dev/null +++ b/tests/hyperswarm/negentropy-policy.test.ts @@ -0,0 +1,47 @@ +import { + shouldAcceptLegacySync, + shouldSchedulePeriodicRepair, + shouldStartConnectTimeNegentropy, +} from '../../services/mediators/hyperswarm/src/negentropy/policy.ts'; + +describe('negentropy sync policy', () => { + it('accepts legacy sync only when enabled and mode is explicitly legacy', () => { + expect(shouldAcceptLegacySync('legacy', true)).toBe(true); + expect(shouldAcceptLegacySync('unknown', true)).toBe(false); + expect(shouldAcceptLegacySync('negentropy', true)).toBe(false); + expect(shouldAcceptLegacySync('legacy', false)).toBe(false); + }); + + it('starts connect-time negentropy only for initiator without active session', () => { + expect(shouldStartConnectTimeNegentropy('negentropy', false, true)).toBe(true); + expect(shouldStartConnectTimeNegentropy('negentropy', true, true)).toBe(false); + expect(shouldStartConnectTimeNegentropy('negentropy', false, false)).toBe(false); + expect(shouldStartConnectTimeNegentropy('legacy', false, true)).toBe(false); + }); + + it('schedules periodic repair only when all guards pass', () => { + const base = { + syncMode: 'negentropy' as const, + hasActiveSession: false, + importQueueLength: 0, + activeNegentropySessions: 0, + lastAttemptAtMs: 0, + nowMs: 1_000_000, + repairIntervalMs: 300_000, + isInitiator: true, + syncCompleted: false, + }; + + expect(shouldSchedulePeriodicRepair(base)).toBe(true); + expect(shouldSchedulePeriodicRepair({ ...base, isInitiator: false })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, hasActiveSession: true })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, importQueueLength: 1 })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, activeNegentropySessions: 1 })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, syncMode: 'legacy' })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, syncCompleted: true })).toBe(false); + expect(shouldSchedulePeriodicRepair({ + ...base, + lastAttemptAtMs: base.nowMs - 10_000, + })).toBe(false); + }); +}); diff --git a/tests/hyperswarm/negentropy-protocol.test.ts b/tests/hyperswarm/negentropy-protocol.test.ts new file mode 100644 index 00000000..2850b0ea --- /dev/null +++ b/tests/hyperswarm/negentropy-protocol.test.ts @@ -0,0 +1,115 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + NEG_SYNC_ID_RE, + chooseConnectSyncMode, + chooseSyncMode, + decodeNegentropyFrame, + encodeNegentropyFrame, + extractOperationHashes, + normalizeNegentropyIds, + normalizePeerCapabilities, +} from '../../services/mediators/hyperswarm/src/negentropy/protocol.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(hash: string): Operation { + return { + type: 'create', + signature: { + hash, + signed: '2026-02-11T00:00:00.000Z', + value: 'sig', + }, + }; +} + +describe('negentropy protocol helpers', () => { + it('encodes and decodes utf8 frame payloads', () => { + const payload = encodeNegentropyFrame('frame-message'); + expect(payload.encoding).toBe('utf8'); + expect(decodeNegentropyFrame(payload)).toBe('frame-message'); + }); + + it('encodes and decodes binary frame payloads', () => { + const bytes = Uint8Array.from([1, 2, 3, 250]); + const payload = encodeNegentropyFrame(bytes); + expect(payload.encoding).toBe('base64'); + const decoded = decodeNegentropyFrame(payload); + expect(Buffer.from(decoded as Uint8Array)).toStrictEqual(Buffer.from(bytes)); + }); + + it('normalizes peer capabilities and chooses sync mode', () => { + const unknown = normalizePeerCapabilities(); + expect(chooseSyncMode(unknown, 1)).toBeNull(); + + const legacy = normalizePeerCapabilities({ negentropy: false }); + expect(chooseSyncMode(legacy, 1)).toBe('legacy'); + + const negentropy = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 1 }); + expect(chooseSyncMode(negentropy, 1)).toBe('negentropy'); + + const oldVersion = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 0 }); + expect(chooseSyncMode(oldVersion, 1)).toBe('legacy'); + }); + + it('chooses connect sync mode with explicit fallback reasons', () => { + const missing = normalizePeerCapabilities(); + expect(chooseConnectSyncMode(missing, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'missing_capabilities', + }); + expect(chooseConnectSyncMode(missing, 1, false)).toStrictEqual({ + mode: null, + reason: 'legacy_disabled', + }); + + const disabled = normalizePeerCapabilities({ negentropy: false, negentropyVersion: 1 }); + expect(chooseConnectSyncMode(disabled, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'negentropy_disabled', + }); + + const oldVersion = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 0 }); + expect(chooseConnectSyncMode(oldVersion, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'version_mismatch', + }); + + const supported = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 1 }); + expect(chooseConnectSyncMode(supported, 1, true)).toStrictEqual({ + mode: 'negentropy', + reason: 'negentropy_supported', + }); + expect(chooseConnectSyncMode(supported, 1, true, false)).toStrictEqual({ + mode: 'legacy', + reason: 'negentropy_disabled', + }); + }); + + it('normalizes negentropy ids and filters invalid values', () => { + const a = h('a'); + const b = h('b'); + const ids = normalizeNegentropyIds([ + a.toUpperCase(), + Buffer.from(b, 'hex'), + 'not-a-sync-id', + ]); + + expect(ids).toStrictEqual([a, b]); + expect(ids.every(id => NEG_SYNC_ID_RE.test(id))).toBe(true); + }); + + it('extracts unique valid operation hashes', () => { + const a = h('a'); + const b = h('b'); + const operations: Operation[] = [ + makeOp(a.toUpperCase()), + makeOp(b), + makeOp('invalid'), + { type: 'create' }, + ]; + + const hashes = extractOperationHashes(operations); + expect(hashes).toStrictEqual([a, b]); + }); +}); diff --git a/tests/hyperswarm/negentropy-transfer.test.ts b/tests/hyperswarm/negentropy-transfer.test.ts new file mode 100644 index 00000000..576dafdb --- /dev/null +++ b/tests/hyperswarm/negentropy-transfer.test.ts @@ -0,0 +1,86 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + chunkIds, + chunkOperationsForPush, + estimateOperationBytes, +} from '../../services/mediators/hyperswarm/src/negentropy/transfer.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(hashChar: string, sizeTag: string = ''): Operation { + return { + type: 'create', + data: { note: `${sizeTag}${'x'.repeat(sizeTag.length * 20)}` }, + signature: { + hash: h(hashChar), + signed: '2026-02-12T00:00:00.000Z', + value: `sig-${hashChar}`, + }, + }; +} + +describe('negentropy transfer batching helpers', () => { + it('throws on invalid chunking options', () => { + expect(() => chunkIds(['a'], 0)).toThrow('maxPerChunk'); + expect(() => chunkOperationsForPush([makeOp('a')], { + maxOpsPerPush: 0, + maxBytesPerPush: 1024, + })).toThrow('maxOpsPerPush'); + expect(() => chunkOperationsForPush([makeOp('a')], { + maxOpsPerPush: 1, + maxBytesPerPush: 0, + })).toThrow('maxBytesPerPush'); + }); + + it('chunks id lists with de-duplication', () => { + const ids = ['a', 'b', 'c', 'c', 'd', 'e']; + const chunks = chunkIds(ids, 2); + expect(chunks).toStrictEqual([ + ['a', 'b'], + ['c', 'd'], + ['e'], + ]); + }); + + it('returns empty id chunks when input ids are empty', () => { + expect(chunkIds([], 2)).toStrictEqual([]); + }); + + it('splits operations by count and bytes', () => { + const ops = [ + makeOp('a', 'small'), + makeOp('b', 'small'), + makeOp('c', 'this-is-a-larger-payload'), + makeOp('d', 'small'), + ]; + const bytesA = estimateOperationBytes(ops[0]); + const bytesB = estimateOperationBytes(ops[1]); + const bytesC = estimateOperationBytes(ops[2]); + + const batches = chunkOperationsForPush(ops, { + maxOpsPerPush: 2, + maxBytesPerPush: bytesA + bytesB + Math.floor(bytesC / 2), + }); + + expect(batches.length).toBeGreaterThanOrEqual(2); + expect(batches[0].length).toBeLessThanOrEqual(2); + expect(batches.flat().length).toBe(ops.length); + }); + + it('keeps a single oversized op in its own batch', () => { + const op = makeOp('a', 'x'.repeat(200)); + const batches = chunkOperationsForPush([op], { + maxOpsPerPush: 10, + maxBytesPerPush: 32, + }); + + expect(batches).toStrictEqual([[op]]); + }); + + it('returns empty when operations input is empty', () => { + expect(chunkOperationsForPush([], { + maxOpsPerPush: 10, + maxBytesPerPush: 1024, + })).toStrictEqual([]); + }); +}); diff --git a/tests/hyperswarm/sync-mapping.test.ts b/tests/hyperswarm/sync-mapping.test.ts new file mode 100644 index 00000000..974fb67a --- /dev/null +++ b/tests/hyperswarm/sync-mapping.test.ts @@ -0,0 +1,89 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + MDIP_EPOCH_SECONDS, + SYNC_ID_BYTES_LEN, + mapOperationToSyncKey, + type SyncMappingErrorCode, +} from '../../services/mediators/hyperswarm/src/sync-mapping.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(overrides: Partial> = {}): Operation { + return { + type: 'create', + signature: { + hash: h('a'), + signed: '2026-02-13T00:00:00.000Z', + value: 'sig-a', + ...overrides, + }, + }; +} + +function expectFailure(operation: Operation, code: SyncMappingErrorCode): void { + const result = mapOperationToSyncKey(operation); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(code); + } +} + +describe('sync-mapping', () => { + it('maps valid operation to sync key and normalizes hash case', () => { + const op = makeOp({ hash: h('A') }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.idHex).toBe(h('a')); + expect(result.value.idBytes.length).toBe(SYNC_ID_BYTES_LEN); + expect(result.value.tsSec).toBe(Math.floor(Date.parse('2026-02-13T00:00:00.000Z') / 1000)); + expect(result.value.operation).toBe(op); + } + }); + + it('clamps unix epoch signed timestamp to MDIP epoch', () => { + const op = makeOp({ signed: '1970-01-01T00:00:00.000Z' }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.tsSec).toBe(MDIP_EPOCH_SECONDS); + } + }); + + it('clamps pre-MDIP signed timestamps to MDIP epoch', () => { + const op = makeOp({ signed: '1971-01-01T00:00:00.000Z' }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.tsSec).toBe(MDIP_EPOCH_SECONDS); + } + }); + + it('rejects missing signature', () => { + expectFailure({ type: 'create' }, 'missing_signature'); + }); + + it('rejects missing signature hash', () => { + expectFailure(makeOp({ hash: '' }), 'missing_signature_hash'); + }); + + it('rejects non-string signature hash', () => { + expectFailure(makeOp({ hash: 123 as unknown as string }), 'invalid_signature_hash_type'); + }); + + it('rejects invalid signature hash format', () => { + expectFailure(makeOp({ hash: 'xyz' }), 'invalid_signature_hash_format'); + }); + + it('rejects missing signature signed', () => { + expectFailure(makeOp({ signed: '' }), 'missing_signature_signed'); + }); + + it('rejects non-string signature signed', () => { + expectFailure(makeOp({ signed: 123 as unknown as string }), 'invalid_signature_signed_type'); + }); + + it('rejects unparseable signature signed value', () => { + expectFailure(makeOp({ signed: 'not-a-date' }), 'invalid_signature_signed_value'); + }); +}); diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts new file mode 100644 index 00000000..0085d5e0 --- /dev/null +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -0,0 +1,111 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + filterOperationsByAcceptedHashes, + filterIndexRejectedOperations, + mapAcceptedOperationsToSyncRecords, +} from '../../services/mediators/hyperswarm/src/sync-persistence.ts'; +import { + MDIP_EPOCH_SECONDS, +} from '../../services/mediators/hyperswarm/src/sync-mapping.ts'; + +const h = (c: string) => c.repeat(64); + +function makeCreateOp(hashChar: string, signed: string): Operation { + return { + type: 'create', + created: signed, + mdip: { + version: 1, + type: 'agent', + registry: 'hyperswarm', + }, + signature: { + signed, + hash: h(hashChar), + value: `sig-${hashChar}`, + }, + }; +} + +describe('sync-persistence helpers', () => { + it('returns empty array when filterIndexRejectedOperations receives empty batch', () => { + expect(filterIndexRejectedOperations([], [0, 1])).toStrictEqual([]); + }); + + it('filters rejected indices in original submitted order', () => { + // eslint-disable-next-line sonarjs/no-duplicate-string + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + // eslint-disable-next-line sonarjs/no-duplicate-string + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); + + const filtered = filterIndexRejectedOperations([a, b, c], [1, 0, 999, -1, 1]); + + expect(filtered).toStrictEqual([c]); + }); + + it('returns original batch when rejected indices are missing/empty', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + + expect(filterIndexRejectedOperations([a, b], undefined)).toStrictEqual([a, b]); + expect(filterIndexRejectedOperations([a, b], [])).toStrictEqual([a, b]); + }); + + it('maps accepted operations to sync records and counts invalid operations', () => { + const valid = makeCreateOp('A', '2026-02-10T10:00:00.000Z'); + const invalid = makeCreateOp('b', 'not-a-date'); + + const result = mapAcceptedOperationsToSyncRecords([valid, invalid]); + + expect(result.records.length).toBe(1); + expect(result.records[0].id).toBe(h('a')); + expect(result.records[0].ts).toBe(Math.floor(Date.parse(valid.signature!.signed) / 1000)); + expect(result.invalid).toBe(1); + }); + + it('maps pre-MDIP signed timestamp to MDIP epoch seconds', () => { + const legacyEpoch = makeCreateOp('a', '1971-01-01T00:00:00.000Z'); + const result = mapAcceptedOperationsToSyncRecords([legacyEpoch]); + expect(result.records.length).toBe(1); + expect(result.records[0].ts).toBe(MDIP_EPOCH_SECONDS); + expect(result.invalid).toBe(0); + }); + + it('maps unix epoch signed timestamp to MDIP epoch seconds', () => { + const legacyEpoch = makeCreateOp('a', '1970-01-01T00:00:00.000Z'); + const result = mapAcceptedOperationsToSyncRecords([legacyEpoch]); + expect(result.records.length).toBe(1); + expect(result.records[0].ts).toBe(MDIP_EPOCH_SECONDS); + expect(result.invalid).toBe(0); + }); + + it('returns no records when all operations are invalid', () => { + const invalidA = makeCreateOp('a', 'not-a-date'); + const invalidB = { type: 'create' } as Operation; + + const result = mapAcceptedOperationsToSyncRecords([invalidA, invalidB]); + expect(result.records).toStrictEqual([]); + expect(result.invalid).toBe(2); + }); + + it('filters operations by accepted hashes', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); + + const accepted = filterOperationsByAcceptedHashes([a, b, c], [h('a').toUpperCase(), h('c')]); + expect(accepted).toStrictEqual([a, c]); + }); + + it('returns empty when accepted hashes are empty or normalize to empty set', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + expect(filterOperationsByAcceptedHashes([a], [])).toStrictEqual([]); + expect(filterOperationsByAcceptedHashes([a], ['', '' as unknown as string])).toStrictEqual([]); + }); + + it('returns empty when operations input is empty or not an array', () => { + expect(filterOperationsByAcceptedHashes([], [h('a')])).toStrictEqual([]); + expect(filterOperationsByAcceptedHashes('not-an-array' as unknown as Operation[], [h('a')])).toStrictEqual([]); + }); +}); diff --git a/tests/hyperswarm/sync-store.test.ts b/tests/hyperswarm/sync-store.test.ts new file mode 100644 index 00000000..4c704852 --- /dev/null +++ b/tests/hyperswarm/sync-store.test.ts @@ -0,0 +1,190 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { Operation } from '@mdip/gatekeeper/types'; +import SqliteOperationSyncStore from '../../services/mediators/hyperswarm/src/db/sqlite.ts'; +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import { OperationSyncStore } from '../../services/mediators/hyperswarm/src/db/types.ts'; + +const h = (c: string) => c.repeat(64); + +const opA: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T10:00:00.000Z', + hash: h('a'), + value: 'sig-a', + }, +}; + +const opB: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T11:00:00.000Z', + hash: h('b'), + value: 'sig-b', + }, +}; + +const opC: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T11:30:00.000Z', + hash: h('c'), + value: 'sig-c', + }, +}; + +const recA = { id: h('a'), ts: 1000, operation: opA }; +const recB = { id: h('b'), ts: 2000, operation: opB }; +const recC = { id: h('c'), ts: 2000, operation: opC }; + +async function runStoreContractTests(store: OperationSyncStore): Promise { + await store.start(); + await store.reset(); + + const inserted1 = await store.upsertMany([recB, recA, recC]); + expect(inserted1).toBe(3); + + const inserted2 = await store.upsertMany([recA, recC]); + expect(inserted2).toBe(0); + + expect(await store.count()).toBe(3); + expect(await store.has(recA.id)).toBe(true); + expect(await store.has(h('d'))).toBe(false); + + const byIds = await store.getByIds([recC.id, recA.id, h('d')]); + expect(byIds.map(item => item.id)).toStrictEqual([recC.id, recA.id]); + + const sortedAll = await store.iterateSorted(); + expect(sortedAll.map(item => item.id)).toStrictEqual([recA.id, recB.id, recC.id]); + expect(sortedAll.every(item => Number.isFinite(item.insertedAt))).toBe(true); + + const firstTwo = await store.iterateSorted({ limit: 2 }); + expect(firstTwo.map(item => item.id)).toStrictEqual([recA.id, recB.id]); + + const rangeTs = await store.iterateSorted({ fromTs: 1500, toTs: 2000 }); + expect(rangeTs.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + const afterA = await store.iterateSorted({ after: { ts: 1000, id: recA.id } }); + expect(afterA.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + const afterB = await store.iterateSorted({ after: { ts: 2000, id: recB.id } }); + expect(afterB.map(item => item.id)).toStrictEqual([recC.id]); + + const afterRange = await store.iterateSorted({ + after: { ts: 1500, id: h('0') }, + fromTs: 1500, + toTs: 2000, + }); + expect(afterRange.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + await store.reset(); + expect(await store.count()).toBe(0); + + await store.stop(); +} + +describe('InMemoryOperationSyncStore', () => { + it('implements sync-store contract', async () => { + const store = new InMemoryOperationSyncStore(); + await runStoreContractTests(store); + }); + + it('preserves explicit insertedAt and handles empty getByIds input', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + await store.reset(); + + const insertedAt = 123456789; + const inserted = await store.upsertMany([{ + ...recA, + insertedAt, + }]); + + expect(inserted).toBe(1); + const rows = await store.getByIds([recA.id]); + expect(rows[0].insertedAt).toBe(insertedAt); + expect(await store.getByIds([])).toStrictEqual([]); + }); +}); + +describe('SqliteOperationSyncStore', () => { + let tmpRoot = ''; + + beforeEach(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hypr-sync-store-')); + }); + + afterEach(async () => { + if (tmpRoot) { + await fs.rm(tmpRoot, { recursive: true, force: true }); + } + }); + + it('implements sync-store contract', async () => { + // eslint-disable-next-line sonarjs/no-duplicate-string + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await runStoreContractTests(store); + }); + + it('is safe to stop before start and idempotent to start twice', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await expect(store.stop()).resolves.toBeUndefined(); + await store.start(); + await store.start(); + await expect(store.stop()).resolves.toBeUndefined(); + }); + + it('throws from data APIs when start was not called', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + + // eslint-disable-next-line sonarjs/no-duplicate-string + await expect(store.reset()).rejects.toThrow('Call start() first'); + await expect(store.upsertMany([recA])).rejects.toThrow('Call start() first'); + await expect(store.getByIds([recA.id])).rejects.toThrow('Call start() first'); + await expect(store.iterateSorted()).rejects.toThrow('Call start() first'); + await expect(store.has(recA.id)).rejects.toThrow('Call start() first'); + await expect(store.count()).rejects.toThrow('Call start() first'); + }); + + it('returns early for empty inputs', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + await store.reset(); + + expect(await store.upsertMany([])).toBe(0); + expect(await store.getByIds([])).toStrictEqual([]); + }); + + it('rolls back transaction when an upsert item fails serialization', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + await store.reset(); + + const circular: any = { type: 'create' }; + circular.self = circular; + + await expect(store.upsertMany([ + recA, + { + id: h('d'), + ts: 4000, + operation: circular, + } as any, + ])).rejects.toThrow(); + + expect(await store.count()).toBe(0); + }); + + it('returns zero count when sqlite get returns no row', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + + const db = (store as any).db; + const originalGet = db.get.bind(db); + db.get = async () => undefined; + await expect(store.count()).resolves.toBe(0); + db.get = originalGet; + }); +});