From d83f8da93f723985dd3cae1687814061f5b19be1 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 11:17:27 +0000 Subject: [PATCH 01/31] feat(hyperswarm): add negentropy sync with sqlite store --- docker-compose.yml | 2 + jest.config.js | 2 + packages/gatekeeper/src/gatekeeper.ts | 30 +- packages/gatekeeper/src/types.ts | 3 + .../gatekeeper/server/src/gatekeeper-api.ts | 15 + services/mediators/hyperswarm/README.md | 73 +- .../mediators/hyperswarm/package-lock.json | 1554 ++++++++++++++++- services/mediators/hyperswarm/package.json | 6 +- .../mediators/hyperswarm/src/bootstrap.ts | 61 + services/mediators/hyperswarm/src/config.js | 54 +- .../mediators/hyperswarm/src/db/memory.ts | 98 ++ .../mediators/hyperswarm/src/db/sqlite.ts | 231 +++ services/mediators/hyperswarm/src/db/types.ts | 31 + .../hyperswarm/src/hyperswarm-mediator.ts | 981 ++++++++++- .../hyperswarm/src/negentropy/Negentropy.cjs | 589 +++++++ .../hyperswarm/src/negentropy/adapter.ts | 475 +++++ .../src/negentropy/observability.ts | 78 + .../hyperswarm/src/negentropy/policy.ts | 60 + .../hyperswarm/src/negentropy/protocol.ts | 110 ++ .../hyperswarm/src/negentropy/transfer.ts | 70 + .../mediators/hyperswarm/src/sync-mapping.ts | 95 + .../hyperswarm/src/sync-persistence.ts | 81 + tests/gatekeeper/client.test.ts | 24 +- tests/gatekeeper/sync.test.ts | 38 + tests/hyperswarm/bootstrap.test.ts | 81 + tests/hyperswarm/negentropy-adapter.test.ts | 273 +++ .../negentropy-observability.test.ts | 61 + tests/hyperswarm/negentropy-policy.test.ts | 46 + tests/hyperswarm/negentropy-protocol.test.ts | 80 + tests/hyperswarm/negentropy-transfer.test.ts | 63 + tests/hyperswarm/sync-persistence.test.ts | 58 + tests/hyperswarm/sync-store.test.ts | 112 ++ 32 files changed, 5414 insertions(+), 121 deletions(-) create mode 100644 services/mediators/hyperswarm/src/bootstrap.ts create mode 100644 services/mediators/hyperswarm/src/db/memory.ts create mode 100644 services/mediators/hyperswarm/src/db/sqlite.ts create mode 100644 services/mediators/hyperswarm/src/db/types.ts create mode 100644 services/mediators/hyperswarm/src/negentropy/Negentropy.cjs create mode 100644 services/mediators/hyperswarm/src/negentropy/adapter.ts create mode 100644 services/mediators/hyperswarm/src/negentropy/observability.ts create mode 100644 services/mediators/hyperswarm/src/negentropy/policy.ts create mode 100644 services/mediators/hyperswarm/src/negentropy/protocol.ts create mode 100644 services/mediators/hyperswarm/src/negentropy/transfer.ts create mode 100644 services/mediators/hyperswarm/src/sync-mapping.ts create mode 100644 services/mediators/hyperswarm/src/sync-persistence.ts create mode 100644 tests/hyperswarm/bootstrap.test.ts create mode 100644 tests/hyperswarm/negentropy-adapter.test.ts create mode 100644 tests/hyperswarm/negentropy-observability.test.ts create mode 100644 tests/hyperswarm/negentropy-policy.test.ts create mode 100644 tests/hyperswarm/negentropy-protocol.test.ts create mode 100644 tests/hyperswarm/negentropy-transfer.test.ts create mode 100644 tests/hyperswarm/sync-persistence.test.ts create mode 100644 tests/hyperswarm/sync-store.test.ts diff --git a/docker-compose.yml b/docker-compose.yml index 89e7e434b..d7aa645df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,8 @@ services: - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} - KC_LOG_LEVEL=${KC_LOG_LEVEL} + volumes: + - ./data:/app/hyperswarm/data user: "${KC_UID}:${KC_GID}" depends_on: - gatekeeper diff --git a/jest.config.js b/jest.config.js index 81faae45d..83d4a6028 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,6 +32,8 @@ const config = { '^pino$': '/tests/common/pino.mock.ts', '^\\.\\/typeGuards\\.js$': '/packages/keymaster/src/db/typeGuards.ts', '^\\.\\/db\\/typeGuards\\.js$': '/packages/keymaster/src/db/typeGuards.ts', + '^\\.\\/sync-mapping\\.js$': '/services/mediators/hyperswarm/src/sync-mapping.ts', + '^\\.\\/sync-persistence\\.js$': '/services/mediators/hyperswarm/src/sync-persistence.ts', '^\\.\\/abstract-json\\.js$': '/packages/gatekeeper/src/db/abstract-json.ts', '^\\.\\/cipher-base\\.js$': '/packages/cipher/src/cipher-base.ts', '^\\.\\/abstract-base\\.js$': '/packages/keymaster/src/db/abstract-base.ts', diff --git a/packages/gatekeeper/src/gatekeeper.ts b/packages/gatekeeper/src/gatekeeper.ts index 55965561b..9094b9e55 100644 --- a/packages/gatekeeper/src/gatekeeper.ts +++ b/packages/gatekeeper/src/gatekeeper.ts @@ -1026,6 +1026,7 @@ export default class Gatekeeper implements GatekeeperInterface { let added = 0; let merged = 0; let rejected = 0; + const acceptedHashes: string[] = []; this.eventsQueue = []; @@ -1036,10 +1037,16 @@ export default class Gatekeeper implements GatekeeperInterface { if (status === ImportStatus.ADDED) { added += 1; + if (event.operation.signature?.hash) { + acceptedHashes.push(event.operation.signature.hash.toLowerCase()); + } this.log.debug(`import ${i}/${total}: added event for ${event.did}`); } else if (status === ImportStatus.MERGED) { merged += 1; + if (event.operation.signature?.hash) { + acceptedHashes.push(event.operation.signature.hash.toLowerCase()); + } this.log.debug(`import ${i}/${total}: merged event for ${event.did}`); } else if (status === ImportStatus.REJECTED) { @@ -1054,7 +1061,7 @@ export default class Gatekeeper implements GatekeeperInterface { event = tempQueue.shift(); } - return { added, merged, rejected }; + return { added, merged, rejected, acceptedHashes }; } async processEvents(): Promise { @@ -1066,6 +1073,7 @@ export default class Gatekeeper implements GatekeeperInterface { let merged = 0; let rejected = 0; let done = false; + const acceptedHashes = new Set(); try { this.isProcessingEvents = true; @@ -1076,6 +1084,9 @@ export default class Gatekeeper implements GatekeeperInterface { added += response.added; merged += response.merged; rejected += response.rejected; + for (const hash of response.acceptedHashes) { + acceptedHashes.add(hash); + } done = (response.added === 0 && response.merged === 0); } @@ -1089,7 +1100,13 @@ export default class Gatekeeper implements GatekeeperInterface { } const pending = this.eventsQueue.length; - const response = { added, merged, rejected, pending }; + const response = { + added, + merged, + rejected, + pending, + acceptedHashes: Array.from(acceptedHashes), + }; this.log.debug(`processEvents: ${JSON.stringify(response)}`); @@ -1181,8 +1198,8 @@ export default class Gatekeeper implements GatekeeperInterface { } let queued = 0; - let rejected = 0; let processed = 0; + const rejectedIndices: number[] = []; for (let i = 0; i < batch.length; i++) { const event = batch[i]; @@ -1200,15 +1217,16 @@ export default class Gatekeeper implements GatekeeperInterface { } } else { - rejected += 1; + rejectedIndices.push(i); } } return { queued, processed, - rejected, - total: this.eventsQueue.length + rejected: rejectedIndices.length, + total: this.eventsQueue.length, + rejectedIndices, }; } diff --git a/packages/gatekeeper/src/types.ts b/packages/gatekeeper/src/types.ts index f0f1b6593..e7cd1f4e4 100644 --- a/packages/gatekeeper/src/types.ts +++ b/packages/gatekeeper/src/types.ts @@ -13,6 +13,7 @@ export interface ImportBatchResult { processed: number; rejected: number; total: number; + rejectedIndices: number[]; } export interface ProcessEventsResult { @@ -21,6 +22,7 @@ export interface ProcessEventsResult { merged?: number; rejected?: number; pending?: number; + acceptedHashes?: string[]; } export interface VerifyDbResult { @@ -109,6 +111,7 @@ export interface ImportEventsResult { added: number; merged: number; rejected: number; + acceptedHashes: string[]; } export interface GatekeeperClientOptions { diff --git a/services/gatekeeper/server/src/gatekeeper-api.ts b/services/gatekeeper/server/src/gatekeeper-api.ts index 2c81c8397..77d6ced21 100644 --- a/services/gatekeeper/server/src/gatekeeper-api.ts +++ b/services/gatekeeper/server/src/gatekeeper-api.ts @@ -1114,6 +1114,11 @@ v1router.post('/dids/export', async (req, res) => { * rejected: * type: integer * description: Number of events that failed validation (bad signature, size limit, etc.). + * rejectedIndices: + * type: array + * description: Zero-based indexes of rejected events in the original submitted batch order (for importDIDs this is `dids.flat()` order). + * items: + * type: integer * total: * type: integer * description: Total number of events in the queue after this import. @@ -1290,6 +1295,11 @@ v1router.post('/batch/export', async (req, res) => { * rejected: * type: integer * description: Number of events that failed validation. + * rejectedIndices: + * type: array + * description: Zero-based indexes of rejected events in the original submitted batch order. + * items: + * type: integer * total: * type: integer * description: The total event queue size after this import. @@ -1608,6 +1618,11 @@ v1router.get('/db/verify', async (req, res) => { * pending: * type: integer * description: Number of events still left in the queue after processing. + * acceptedHashes: + * type: array + * description: Lower-case signature hashes of events accepted during this processing run (added or merged). + * items: + * type: string * 500: * description: Internal Server Error. * content: diff --git a/services/mediators/hyperswarm/README.md b/services/mediators/hyperswarm/README.md index 648abdd9f..106787269 100644 --- a/services/mediators/hyperswarm/README.md +++ b/services/mediators/hyperswarm/README.md @@ -2,28 +2,71 @@ The Hyperswarm mediator is responsible for distributing unconfirmed MDIP operations to the network and for organizing an IPFS peer network for file-sharing. -When a node gets a new connection, it sends the connection a `sync` message and the connection replies with a series of `batch` messages containing all the operations in the connection's DID database. The nodes imports these operations into its Gatekeeper. The Gatekeeper will add any new operations it hasn't seen before, merge any operations it has already seen, and reject invalid operations. +The mediator supports two synchronization modes: -While running the mediator will poll the Gatekeeper's hyperswarm queue for new operations, and relay them to all of its connections with a `queue` message. -When a node receives a `queue` message it will import the operations like during a `batch` but also relay the message to all of its connections, distributing the new operations with a "gossip protocol". +- `negentropy` mode (preferred): connect-time catch-up and periodic anti-entropy repair using `neg_open`/`neg_msg`/`ops_req`/`ops_push`/`neg_close`. +- `legacy` mode (compatibility): classic `sync` -> full-history `batch` transfer (`shareDb`). + +Realtime propagation is always handled by the Gatekeeper queue gossip path: +- mediator polls `gatekeeper.getQueue('hyperswarm')` +- relays queue operations with a `queue` message +- peers import and further relay `queue` messages + +This keeps low latency for new operations while negentropy handles catch-up/repair. + +## Sync mode behavior + +| peer mode | connect-time behavior | periodic behavior | queue gossip | +| --- | --- | --- | --- | +| `negentropy` | negotiate + run negentropy session | periodic anti-entropy repair sessions | enabled | +| `legacy` | `sync` + `shareDb` full-history export | n/a | enabled | + +`shareDb` is intentionally retained for backward compatibility and can be disabled once compatibility validation is complete. + +## Observability + +The mediator emits periodic structured sync metrics in `connectionLoop` including: +- session mode selection counts (`legacy` vs `negentropy`) and fallback rate +- negentropy rounds and have/need totals +- ops requested/pushed sent and received +- gatekeeper apply/reject totals +- bytes sent/received +- session duration aggregates +- queue delay aggregates (from operation `signature.signed` to relay/import time) ## Environment variables -| variable | default | description | -| ------------------------- | ---------------------- | ----------------------------- | -| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | -| `KC_KEYMASTER_URL` | http://localhost:4226 | MDIP keymaster service URL | -| `KC_IPFS_URL` | http://localhost:5001/api/v0 | IPFS RPC URL | -| `KC_IPFS_ENABLE` | true | Enable IPFS + Keymaster peering integration | -| `KC_NODE_ID ` | (no default) | Keymaster node agent name | -| `KC_NODE_NAME` | anon | Human-readable name for the node | -| `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | -| `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | -| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | +| variable | default | description | +| ------------------------- |------------------------------| ----------------------------- | +| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | +| `KC_KEYMASTER_URL` | http://localhost:4226 | MDIP keymaster service URL | +| `KC_IPFS_URL` | http://localhost:5001/api/v0 | IPFS RPC URL | +| `KC_IPFS_ENABLE` | true | Enable IPFS + Keymaster peering integration | +| `KC_NODE_ID ` | (no default) | Keymaster node agent name | +| `KC_NODE_NAME` | anon | Human-readable name for the node | +| `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | +| `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | +| `KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT` | 0 | Negentropy frame-size limit (0 or >= 4096) | +| `KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS` | 7 | First reconciliation window size in days (recent-first) | +| `KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS` | 30 | Older reconciliation window size in days | +| `KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW` | 25000 | Maximum operations loaded into a single window adapter | +| `KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION` | 64 | Maximum negentropy rounds per window session | +| `KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS` | 300 | Seconds between periodic negentropy repair attempts per peer | +| `KC_HYPR_NEGENTROPY_MAX_CONCURRENT_SESSIONS` | 1 | Maximum concurrent negentropy sessions on this node | +| `KC_HYPR_LEGACY_SYNC_ENABLE` | true | Allow legacy `sync`/`shareDb` compatibility path | +| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | ## IPFS disabled mode Set `KC_IPFS_ENABLE=false` to run the mediator without IPFS or Keymaster integration. In this mode: -- operations still sync and relay over Hyperswarm (batch/queue/sync/ping) +- operations still sync and relay over Hyperswarm (queue + negentropy; legacy sync if enabled) - IPFS peering is disabled and node IPFS info is not published - `KC_NODE_ID` is not required because Keymaster is not used + +## Sync Store Scaffolding + +The mediator now includes a sync-store abstraction in `src/db/` with: +- `SqliteOperationSyncStore` for persistent ordered storage +- `InMemoryOperationSyncStore` for tests + +The SQLite implementation uses a fixed data path under `data/hyperswarm` (relative to the mediator working directory), with an index on `(ts, id)` to use SQLite's native B-tree ordering for range queries. diff --git a/services/mediators/hyperswarm/package-lock.json b/services/mediators/hyperswarm/package-lock.json index 3736aac31..3120fe2ce 100644 --- a/services/mediators/hyperswarm/package-lock.json +++ b/services/mediators/hyperswarm/package-lock.json @@ -1,12 +1,12 @@ { "name": "hyperswarm-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hyperswarm-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "license": "MIT", "dependencies": { "@noble/hashes": "^1.3.3", @@ -16,13 +16,21 @@ "express": "^4.21.0", "graceful-goodbye": "^1.3.0", "hyperswarm": "^4.7.14", - "morgan": "^1.10.0" + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7" }, "devDependencies": { "@types/async": "^3.2.24", "@types/b4a": "^1.6.5" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "license": "MIT", + "optional": true + }, "node_modules/@hyperswarm/secret-stream": { "version": "6.6.3", "resolved": "https://registry.npmjs.org/@hyperswarm/secret-stream/-/secret-stream-6.6.3.tgz", @@ -49,6 +57,42 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "license": "MIT", + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/@types/async": { "version": "3.2.24", "resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.24.tgz", @@ -73,6 +117,13 @@ "undici-types": "~6.21.0" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC", + "optional": true + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -85,6 +136,103 @@ "node": ">= 0.6" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/agent-base/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/agent-base/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "optional": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/aproba": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", + "integrity": "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==", + "license": "ISC", + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -100,27 +248,47 @@ "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT", + "optional": true + }, "node_modules/bare-events": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==" }, - "node_modules/basic-auth": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", - "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.8" + "file-uri-to-path": "1.0.0" } }, - "node_modules/basic-auth/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/bits-to-bytes": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bits-to-bytes/-/bits-to-bytes-1.3.0.tgz", @@ -129,6 +297,17 @@ "b4a": "^1.5.0" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/blind-relay": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/blind-relay/-/blind-relay-1.3.3.tgz", @@ -177,6 +356,41 @@ "compact-encoding-net": "^1.2.0" } }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -185,6 +399,36 @@ "node": ">= 0.8" } }, + "node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -203,6 +447,35 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/compact-encoding": { "version": "2.15.0", "resolved": "https://registry.npmjs.org/compact-encoding/-/compact-encoding-2.15.0.tgz", @@ -227,6 +500,20 @@ "compact-encoding": "^2.4.1" } }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT", + "optional": true + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "license": "ISC", + "optional": true + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -276,6 +563,30 @@ "streamx": "^2.12.4" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -292,6 +603,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT", + "optional": true + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -309,6 +627,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/dht-rpc": { "version": "6.14.1", "resolved": "https://registry.npmjs.org/dht-rpc/-/dht-rpc-6.14.1.tgz", @@ -343,6 +670,13 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT", + "optional": true + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -351,6 +685,55 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "license": "MIT", + "optional": true + }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", @@ -383,6 +766,15 @@ "node": ">= 0.6" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.21.2", "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", @@ -434,6 +826,12 @@ "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -467,6 +865,31 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC", + "optional": true + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -475,6 +898,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -493,6 +937,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "optional": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -504,6 +976,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC", + "optional": true + }, "node_modules/graceful-goodbye": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/graceful-goodbye/-/graceful-goodbye-1.3.0.tgz", @@ -545,6 +1024,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "license": "ISC", + "optional": true + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -556,6 +1042,13 @@ "node": ">= 0.4" } }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause", + "optional": true + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -571,40 +1064,129 @@ "node": ">= 0.8" } }, - "node_modules/hypercore-crypto": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/hypercore-crypto/-/hypercore-crypto-3.4.2.tgz", - "integrity": "sha512-16ii4M6T1dFfRa41Szv3IR0wXfImJMYJ8ysZEGwHEDH7sMeWVEBck6tg1GCNutYl39E+H7wMY2p3ndCRfj+XdQ==", + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "license": "MIT", + "optional": true, "dependencies": { - "b4a": "^1.6.6", - "compact-encoding": "^2.15.0", - "sodium-universal": "^4.0.1" + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" } }, - "node_modules/hypercore-id-encoding": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/hypercore-id-encoding/-/hypercore-id-encoding-1.3.0.tgz", - "integrity": "sha512-W6sHdGo5h7LXEsoWfKf/KfuROZmZRQDlGqJF2EPHW+noCK66Vvr0+zE6cL0vqQi18s0kQPeN7Sq3QyR0Ytc2VQ==", + "node_modules/http-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, "dependencies": { - "b4a": "^1.5.3", - "z32": "^1.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/hyperdht": { - "version": "6.17.2", - "resolved": "https://registry.npmjs.org/hyperdht/-/hyperdht-6.17.2.tgz", - "integrity": "sha512-Oxxmj0khgroLR8MizK8BNdOQBRmJp7H+vOSpPv6IDACAN6oaJiqi5m9ValCph7Gc85LhqloExcxep02hOvMamA==", + "node_modules/http-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "optional": true, "dependencies": { - "@hyperswarm/secret-stream": "^6.6.2", - "b4a": "^1.3.1", - "bare-events": "^2.2.0", - "blind-relay": "^1.3.0", - "bogon": "^1.0.0", - "compact-encoding": "^2.4.1", - "compact-encoding-net": "^1.0.1", - "debugging-stream": "^2.0.0", - "dht-rpc": "^6.14.0", - "hypercore-crypto": "^3.3.0", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/https-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/hypercore-crypto": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/hypercore-crypto/-/hypercore-crypto-3.4.2.tgz", + "integrity": "sha512-16ii4M6T1dFfRa41Szv3IR0wXfImJMYJ8ysZEGwHEDH7sMeWVEBck6tg1GCNutYl39E+H7wMY2p3ndCRfj+XdQ==", + "dependencies": { + "b4a": "^1.6.6", + "compact-encoding": "^2.15.0", + "sodium-universal": "^4.0.1" + } + }, + "node_modules/hypercore-id-encoding": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/hypercore-id-encoding/-/hypercore-id-encoding-1.3.0.tgz", + "integrity": "sha512-W6sHdGo5h7LXEsoWfKf/KfuROZmZRQDlGqJF2EPHW+noCK66Vvr0+zE6cL0vqQi18s0kQPeN7Sq3QyR0Ytc2VQ==", + "dependencies": { + "b4a": "^1.5.3", + "z32": "^1.0.0" + } + }, + "node_modules/hyperdht": { + "version": "6.17.2", + "resolved": "https://registry.npmjs.org/hyperdht/-/hyperdht-6.17.2.tgz", + "integrity": "sha512-Oxxmj0khgroLR8MizK8BNdOQBRmJp7H+vOSpPv6IDACAN6oaJiqi5m9ValCph7Gc85LhqloExcxep02hOvMamA==", + "dependencies": { + "@hyperswarm/secret-stream": "^6.6.2", + "b4a": "^1.3.1", + "bare-events": "^2.2.0", + "blind-relay": "^1.3.0", + "bogon": "^1.0.0", + "compact-encoding": "^2.4.1", + "compact-encoding-net": "^1.0.1", + "debugging-stream": "^2.0.0", + "dht-rpc": "^6.14.0", + "hypercore-crypto": "^3.3.0", "hypercore-id-encoding": "^1.2.0", "hypertrace": "^1.3.0", "noise-curve-ed": "^2.0.0", @@ -650,11 +1232,86 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "license": "ISC", + "optional": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "optional": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -663,6 +1320,30 @@ "node": ">= 0.10" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "license": "MIT", + "optional": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, "node_modules/kademlia-routing-table": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/kademlia-routing-table/-/kademlia-routing-table-1.0.3.tgz", @@ -671,6 +1352,47 @@ "bare-events": "^2.2.0" } }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "license": "ISC", + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -725,33 +1447,153 @@ "node": ">= 0.6" } }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "optional": true, "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" + "brace-expansion": "^1.1.7" }, "engines": { - "node": ">= 0.8.0" + "node": "*" } }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", "dependencies": { - "ee-first": "1.1.1" + "yallist": "^4.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "license": "MIT", + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" } }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -762,6 +1604,12 @@ "resolved": "https://registry.npmjs.org/nanoassert/-/nanoassert-2.0.0.tgz", "integrity": "sha512-7vO7n28+aYO4J+8w96AzhmU8G+Y/xpPDJz/se19ICsqj/momRbb9mh9ZUtkoJ5X3nTnPdhEJyc0qnM6yAsHBaA==" }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -780,6 +1628,49 @@ "node": ">= 0.6" } }, + "node_modules/node-abi": { + "version": "3.87.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.87.0.tgz", + "integrity": "sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "license": "MIT", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, "node_modules/node-gyp-build": { "version": "4.8.2", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.2.tgz", @@ -810,6 +1701,39 @@ "sodium-universal": "^4.0.0" } }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/object-inspect": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", @@ -832,13 +1756,29 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", - "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", "license": "MIT", + "optional": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, "engines": { - "node": ">= 0.8" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/parseurl": { @@ -849,12 +1789,69 @@ "node": ">= 0.8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-to-regexp": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "license": "ISC", + "optional": true + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", + "optional": true, + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/protomux": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/protomux/-/protomux-3.9.2.tgz", @@ -880,6 +1877,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -921,6 +1928,35 @@ "node": ">= 0.8" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/record-cache": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/record-cache/-/record-cache-1.2.0.tgz", @@ -929,6 +1965,33 @@ "b4a": "^1.3.1" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -958,6 +2021,18 @@ "resolved": "https://registry.npmjs.org/safety-catch/-/safety-catch-1.0.2.tgz", "integrity": "sha512-C1UYVZ4dtbBxEtvOcpjBaaD27nP8MlvyAQEp2fOTOEe6pfUpk1cDUxij6BR1jZup6rSyUTaBBplK7LanskrULA==" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -1008,6 +2083,13 @@ "node": ">= 0.8.0" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "license": "ISC", + "optional": true + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -1054,11 +2136,129 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC", + "optional": true + }, "node_modules/signal-promise": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/signal-promise/-/signal-promise-1.0.3.tgz", "integrity": "sha512-WBgv0UnIq2C+Aeh0/n+IRpP6967eIx9WpynTUoiW3isPpfe1zu2LJzyfXdo9Tgef8yR/sGjcMvoUXD7EYdiz+g==" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/socks-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socks-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, "node_modules/sodium-native": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sodium-native/-/sodium-native-4.2.0.tgz", @@ -1093,6 +2293,49 @@ } } }, + "node_modules/sqlite": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-5.1.1.tgz", + "integrity": "sha512-oBkezXa2hnkfuJwUo44Hl9hS3er+YFtueifoajrgidvqsJRQFpc5fKoAkAor1O5ZnLoa28GBScfHXs8j0K358Q==", + "license": "MIT" + }, + "node_modules/sqlite3": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.7.tgz", + "integrity": "sha512-GGIyOiFaG+TUra3JIfkI/zGP8yZYLPQ0pl1bH+ODjiX57sPhrLU5sQJn1y9bDKZUFYkX1crlrPfSYt0BKKdkog==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, + "node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1114,6 +2357,113 @@ "bare-events": "^2.2.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, "node_modules/text-decoder": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.0.tgz", @@ -1140,6 +2490,18 @@ "node": ">=0.6" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1171,6 +2533,26 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "license": "ISC", + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, "node_modules/unordered-set": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unordered-set/-/unordered-set-2.0.1.tgz", @@ -1192,6 +2574,12 @@ "b4a": "^1.6.6" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1208,11 +2596,49 @@ "node": ">= 0.8" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/xache": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/xache/-/xache-1.2.1.tgz", "integrity": "sha512-igRS6jPreJ54ABdzhh4mCDXcz+XMaWO2q1ABRV2yWYuk29jlp8VT7UBdCqNkX7rpYBbXsebVVKkwIuYZjyZNqA==" }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, "node_modules/z32": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/z32/-/z32-1.1.0.tgz", diff --git a/services/mediators/hyperswarm/package.json b/services/mediators/hyperswarm/package.json index a94d1132c..607197098 100644 --- a/services/mediators/hyperswarm/package.json +++ b/services/mediators/hyperswarm/package.json @@ -5,7 +5,7 @@ "description": "MDIP hyperswarm network mediator", "main": "src/index.js", "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && mkdir -p dist/negentropy && cp src/negentropy/Negentropy.cjs dist/negentropy/Negentropy.cjs", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -18,7 +18,9 @@ "dotenv": "^16.4.5", "express": "^4.21.0", "graceful-goodbye": "^1.3.0", - "hyperswarm": "^4.7.14" + "hyperswarm": "^4.7.14", + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7" }, "devDependencies": { "@types/async": "^3.2.24", diff --git a/services/mediators/hyperswarm/src/bootstrap.ts b/services/mediators/hyperswarm/src/bootstrap.ts new file mode 100644 index 000000000..7f78c0c87 --- /dev/null +++ b/services/mediators/hyperswarm/src/bootstrap.ts @@ -0,0 +1,61 @@ +import type { GatekeeperEvent } from '@mdip/gatekeeper/types'; +import type { OperationSyncStore } from './db/types.js'; +import { mapAcceptedOperationsToSyncRecords } from './sync-persistence.js'; + +export interface BootstrapGatekeeper { + exportBatch(dids?: string[]): Promise; +} + +export interface BootstrapResult { + skipped: boolean; + reason?: 'store_not_empty'; + countBefore: number; + countAfter: number; + exported: number; + mapped: number; + invalid: number; + inserted: number; + durationMs: number; +} + +export async function bootstrapSyncStoreIfEmpty( + syncStore: OperationSyncStore, + gatekeeper: BootstrapGatekeeper, +): Promise { + const startedAt = Date.now(); + const countBefore = await syncStore.count(); + + if (countBefore > 0) { + return { + skipped: true, + reason: 'store_not_empty', + countBefore, + countAfter: countBefore, + exported: 0, + mapped: 0, + invalid: 0, + inserted: 0, + durationMs: Date.now() - startedAt, + }; + } + + const exportedEvents = await gatekeeper.exportBatch(); + const operations = exportedEvents + .map(event => event.operation) + .filter((operation): operation is NonNullable => !!operation); + + const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); + const inserted = records.length > 0 ? await syncStore.upsertMany(records) : 0; + const countAfter = await syncStore.count(); + + return { + skipped: false, + countBefore, + countAfter, + exported: operations.length, + mapped: records.length, + invalid, + inserted, + durationMs: Date.now() - startedAt, + }; +} diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index 3cb8d1072..3d4876e88 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -2,6 +2,50 @@ import dotenv from 'dotenv'; dotenv.config(); +function parsePositiveIntEnv(varName, defaultValue, options = {}) { + const allowZero = options.allowZero === true; + const raw = process.env[varName]; + if (raw == null || raw === '') { + return defaultValue; + } + + const value = Number.parseInt(raw, 10); + if (!Number.isInteger(value) || value < 0 || (!allowZero && value === 0)) { + const expected = allowZero ? 'a non-negative integer' : 'a positive integer'; + throw new Error(`Invalid ${varName}; expected ${expected}`); + } + + return value; +} + +function parseFrameSizeLimit() { + const value = parsePositiveIntEnv('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT', 0, { allowZero: true }); + + if (value > 0 && value < 4096) { + throw new Error('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4096'); + } + + return value; +} + +function parseBooleanEnv(varName, defaultValue) { + const raw = process.env[varName]; + if (raw == null || raw === '') { + return defaultValue; + } + + const normalized = raw.trim().toLowerCase(); + if (normalized === 'true') { + return true; + } + + if (normalized === 'false') { + return false; + } + + throw new Error(`Invalid ${varName}; expected true or false`); +} + const config = { debug: process.env.KC_DEBUG ? process.env.KC_DEBUG === 'true' : false, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', @@ -11,7 +55,15 @@ const config = { nodeID: process.env.KC_NODE_ID || '', nodeName: process.env.KC_NODE_NAME || 'anon', protocol: process.env.KC_MDIP_PROTOCOL || '/MDIP/v1.0-public', - exportInterval: process.env.KC_HYPR_EXPORT_INTERVAL ? parseInt(process.env.KC_HYPR_EXPORT_INTERVAL) : 2, + exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), + negentropyFrameSizeLimit: parseFrameSizeLimit(), + negentropyRecentWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS', 7), + negentropyOlderWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS', 30), + negentropyMaxRecordsPerWindow: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW', 25000), + negentropyMaxRoundsPerSession: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION', 64), + negentropyRepairIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS', 300), + negentropyMaxConcurrentSessions: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_CONCURRENT_SESSIONS', 1), + legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), }; export default config; diff --git a/services/mediators/hyperswarm/src/db/memory.ts b/services/mediators/hyperswarm/src/db/memory.ts new file mode 100644 index 000000000..9e68fd407 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/memory.ts @@ -0,0 +1,98 @@ +import { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from './types.js'; + +export default class InMemoryOperationSyncStore implements OperationSyncStore { + private readonly records = new Map(); + + async start(): Promise { + // no-op + } + + async stop(): Promise { + // no-op + } + + async reset(): Promise { + this.records.clear(); + } + + async upsertMany(records: Array | SyncOperationRecord>): Promise { + if (!Array.isArray(records) || records.length === 0) { + return 0; + } + + let inserted = 0; + const now = Date.now(); + + for (const record of records) { + if (this.records.has(record.id)) { + continue; + } + + inserted += 1; + const insertedAt = 'insertedAt' in record ? record.insertedAt : now; + this.records.set(record.id, { + id: record.id, + ts: record.ts, + operation: record.operation, + insertedAt, + }); + } + + return inserted; + } + + async getByIds(ids: string[]): Promise { + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const out: SyncOperationRecord[] = []; + for (const id of ids) { + const row = this.records.get(id); + if (row) { + out.push(row); + } + } + return out; + } + + async iterateSorted(options: SyncStoreListOptions = {}): Promise { + const limit = options.limit ?? 1000; + const after = options.after; + const fromTs = options.fromTs; + const toTs = options.toTs; + + const sorted = Array.from(this.records.values()).sort((a, b) => { + if (a.ts !== b.ts) { + return a.ts - b.ts; + } + return a.id.localeCompare(b.id); + }); + + const filtered = sorted.filter(item => { + if (after && !(item.ts > after.ts || (item.ts === after.ts && item.id > after.id))) { + return false; + } + + if (typeof fromTs === 'number' && item.ts < fromTs) { + return false; + } + + if (typeof toTs === 'number' && item.ts > toTs) { + return false; + } + + return true; + }); + + return filtered.slice(0, limit); + } + + async has(id: string): Promise { + return this.records.has(id); + } + + async count(): Promise { + return this.records.size; + } +} diff --git a/services/mediators/hyperswarm/src/db/sqlite.ts b/services/mediators/hyperswarm/src/db/sqlite.ts new file mode 100644 index 000000000..6838a3a06 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/sqlite.ts @@ -0,0 +1,231 @@ +import fs from 'fs/promises'; +import path from 'path'; +import * as sqlite from 'sqlite'; +import sqlite3 from 'sqlite3'; +import { childLogger } from '@mdip/common/logger'; +import { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from './types.js'; + +interface SyncRow { + id: string; + ts: number; + operation_json: string; + inserted_at: number; +} + +const SQLITE_NOT_STARTED_ERROR = 'Sync SQLite DB not open. Call start() first.'; +const log = childLogger({ service: 'hyperswarm-sync-db', module: 'sqlite' }); + +export default class SqliteOperationSyncStore implements OperationSyncStore { + private readonly dbName: string; + private readonly dataFolder: string; + private db: sqlite.Database | null; + private _lock: Promise = Promise.resolve(); + + constructor(dbName: string = 'operations.db', dataFolder: string = 'data/hyperswarm') { + this.dbName = dbName; + this.dataFolder = dataFolder; + this.db = null; + } + + private get dbPath(): string { + return path.join(this.dataFolder, this.dbName); + } + + private runExclusive(fn: () => Promise | T): Promise { + const run = async () => await fn(); + const chained = this._lock.then(run, run); + this._lock = chained.then(() => undefined, () => undefined); + return chained; + } + + private async withTx(fn: () => Promise): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + await this.db.exec('BEGIN IMMEDIATE'); + try { + const result = await fn(); + await this.db.exec('COMMIT'); + return result; + } catch (error) { + try { + await this.db.exec('ROLLBACK'); + } catch {} + throw error; + } + } + + async start(): Promise { + if (this.db) { + return; + } + + await fs.mkdir(this.dataFolder, { recursive: true }); + + this.db = await sqlite.open({ + filename: this.dbPath, + driver: sqlite3.Database + }); + + await this.db.exec(` + CREATE TABLE IF NOT EXISTS operations ( + id TEXT PRIMARY KEY, + ts INTEGER NOT NULL, + operation_json TEXT NOT NULL, + inserted_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_operations_ts_id ON operations (ts, id); + `); + } + + async stop(): Promise { + if (this.db) { + await this.db.close(); + this.db = null; + } + } + + async reset(): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + await this.runExclusive(async () => { + await this.withTx(async () => { + await this.db!.run('DELETE FROM operations'); + }); + }); + } + + async upsertMany(records: Array | SyncOperationRecord>): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + if (!Array.isArray(records) || records.length === 0) { + return 0; + } + + return this.runExclusive(async () => { + try { + return await this.withTx(async () => { + let inserted = 0; + const now = Date.now(); + + for (const record of records) { + const insertedAt = 'insertedAt' in record ? record.insertedAt : now; + const res = await this.db!.run( + `INSERT OR IGNORE INTO operations(id, ts, operation_json, inserted_at) VALUES(?, ?, ?, ?)`, + record.id, + record.ts, + JSON.stringify(record.operation), + insertedAt, + ); + + inserted += res.changes ?? 0; + } + + return inserted; + }); + } catch (error) { + log.error({ error }, 'upsertMany failed'); + throw error; + } + }); + } + + async getByIds(ids: string[]): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const placeholders = ids.map(() => '?').join(', '); + const rows = await this.db.all( + `SELECT id, ts, operation_json, inserted_at FROM operations WHERE id IN (${placeholders})`, + ...ids + ); + + const byId = new Map(rows.map(row => [row.id, this.mapRow(row)])); + return ids + .map(id => byId.get(id)) + .filter((item): item is SyncOperationRecord => !!item); + } + + async iterateSorted(options: SyncStoreListOptions = {}): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const limit = options.limit ?? 1000; + const after = options.after; + const fromTs = options.fromTs; + const toTs = options.toTs; + const params: Array = []; + const predicates: string[] = []; + + if (after) { + predicates.push('(ts > ? OR (ts = ? AND id > ?))'); + params.push(after.ts, after.ts, after.id); + } + + if (typeof fromTs === 'number') { + predicates.push('ts >= ?'); + params.push(fromTs); + } + + if (typeof toTs === 'number') { + predicates.push('ts <= ?'); + params.push(toTs); + } + + const where = predicates.length > 0 + ? `WHERE ${predicates.join(' AND ')}` + : ''; + + params.push(limit); + + const rows = await this.db.all( + `SELECT id, ts, operation_json, inserted_at + FROM operations + ${where} + ORDER BY ts ASC, id ASC + LIMIT ?`, + ...params + ); + + return rows.map(row => this.mapRow(row)); + } + + async has(id: string): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const row = await this.db.get<{ id: string }>('SELECT id FROM operations WHERE id = ? LIMIT 1', id); + return !!row; + } + + async count(): Promise { + if (!this.db) { + throw new Error(SQLITE_NOT_STARTED_ERROR); + } + + const row = await this.db.get<{ count: number }>('SELECT COUNT(*) AS count FROM operations'); + return row?.count ?? 0; + } + + private mapRow(row: SyncRow): SyncOperationRecord { + return { + id: row.id, + ts: row.ts, + operation: JSON.parse(row.operation_json), + insertedAt: row.inserted_at, + }; + } +} diff --git a/services/mediators/hyperswarm/src/db/types.ts b/services/mediators/hyperswarm/src/db/types.ts new file mode 100644 index 000000000..ed5786460 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/types.ts @@ -0,0 +1,31 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface SyncOperationRecord { + id: string; + ts: number; + operation: Operation; + insertedAt: number; +} + +export interface SyncStoreCursor { + ts: number; + id: string; +} + +export interface SyncStoreListOptions { + after?: SyncStoreCursor; + limit?: number; + fromTs?: number; + toTs?: number; +} + +export interface OperationSyncStore { + start(): Promise; + stop(): Promise; + reset(): Promise; + upsertMany(records: Array | SyncOperationRecord>): Promise; + getByIds(ids: string[]): Promise; + iterateSorted(options?: SyncStoreListOptions): Promise; + has(id: string): Promise; + count(): Promise; +} diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index b1fe49c93..e2f6f6b2e 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -12,31 +12,127 @@ import { Operation } from '@mdip/gatekeeper/types'; import CipherNode from '@mdip/cipher/node'; import { childLogger } from '@mdip/common/logger'; import config from './config.js'; +import type { OperationSyncStore } from './db/types.js'; +import SqliteOperationSyncStore from './db/sqlite.js'; +import NegentropyAdapter from './negentropy/adapter.js'; +import { + NEG_SYNC_ID_RE, + chooseSyncMode as chooseNegotiatedSyncMode, + decodeNegentropyFrame, + encodeNegentropyFrame, + extractOperationHashes, + normalizeNegentropyIds, + normalizePeerCapabilities, + type NegentropyFrame, + type NegotiatedPeerCapabilities, + type PeerCapabilities, + type SyncMode, +} from './negentropy/protocol.js'; +import { + shouldAcceptLegacySync, + shouldSchedulePeriodicRepair, + shouldStartConnectTimeNegentropy, +} from './negentropy/policy.js'; +import { + addAggregateSample, + averageAggregate, + collectQueueDelaySamples, + createAggregateMetric, + messageBytes, + safeRate, + type AggregateMetric, +} from './negentropy/observability.js'; +import { + chunkIds, + chunkOperationsForPush, +} from './negentropy/transfer.js'; +import { bootstrapSyncStoreIfEmpty } from './bootstrap.js'; +import { + filterOperationsByAcceptedHashes, + filterIndexRejectedOperations, + mapAcceptedOperationsToSyncRecords, +} from './sync-persistence.js'; import { exit } from 'process'; +import path from 'path'; +import { pathToFileURL } from 'url'; const log = childLogger({ service: 'hyperswarm-mediator' }); -interface HyperMessage { - type: 'batch' | 'queue' | 'sync' | 'ping'; +interface HyperMessageBase { + type: string; time: string; node: string; relays: string[]; } -interface PingMessage extends HyperMessage { +interface PingMessage extends HyperMessageBase { + type: 'ping'; peers: string[]; + capabilities?: PeerCapabilities; } -interface BatchMessage extends HyperMessage { +interface BatchMessage extends HyperMessageBase { + type: 'batch' | 'queue'; data: Operation[]; } +interface SyncMessage extends HyperMessageBase { + type: 'sync'; +} + +interface NegOpenMessage extends HyperMessageBase { + type: 'neg_open'; + sessionId: string; + round: number; + frame: NegentropyFrame; +} + +interface NegMsgMessage extends HyperMessageBase { + type: 'neg_msg'; + sessionId: string; + round: number; + frame: NegentropyFrame; +} + +interface NegCloseMessage extends HyperMessageBase { + type: 'neg_close'; + sessionId: string; + round: number; + reason?: string; +} + +interface OpsReqMessage extends HyperMessageBase { + type: 'ops_req'; + sessionId: string; + round: number; + ids: string[]; +} + +interface OpsPushMessage extends HyperMessageBase { + type: 'ops_push'; + sessionId: string; + round: number; + data: Operation[]; +} + +type HyperMessage = + | BatchMessage + | SyncMessage + | PingMessage + | NegOpenMessage + | NegMsgMessage + | NegCloseMessage + | OpsReqMessage + | OpsPushMessage; + interface ImportQueueTask { name: string; msg: BatchMessage; } -interface ExportQueueTask extends ImportQueueTask { +interface ExportQueueTask { + name: string; + msg: SyncMessage; conn: HyperswarmConnection; } @@ -52,23 +148,106 @@ interface ConnectionInfo { nodeName: string; did: string; lastSeen: number; + capabilities: NegotiatedPeerCapabilities; + syncMode: SyncMode | 'unknown'; + syncStarted: boolean; + lastNegentropyRepairAt: number; +} + +interface PeerSyncSession { + sessionId: string; + peerKey: string; + mode: SyncMode; + initiator: boolean; + startedAt: number; + lastActivity: number; + pendingHaveIds: string[]; + pendingNeedIds: string[]; + rounds: number; + maxRounds: number; + reconciliationComplete: boolean; + localClosed: boolean; +} + +interface MediatorSyncStats { + modeSelectionsTotal: number; + modeSelectionsLegacy: number; + modeSelectionsNegentropy: number; + queueOpsRelayed: number; + queueOpsImported: number; + queueDelayMs: AggregateMetric; + negentropySessionsStarted: number; + negentropySessionsClosed: number; + negentropySessionsCompleted: number; + negentropySessionsFailed: number; + negentropyRounds: number; + negentropyHaveIds: number; + negentropyNeedIds: number; + negentropyOpsReqSent: number; + negentropyOpsReqReceived: number; + negentropyOpsPushSent: number; + negentropyOpsPushReceived: number; + opsApplied: number; + opsRejected: number; + bytesSent: number; + bytesReceived: number; + syncDurationMs: AggregateMetric; +} + +export interface MediatorMainOptions { + syncStore?: OperationSyncStore; + startLoops?: boolean; } const gatekeeper = new GatekeeperClient(); const keymaster = new KeymasterClient(); const ipfs = new KuboClient(); const cipher = new CipherNode(); +let syncStore: OperationSyncStore = new SqliteOperationSyncStore(); +let negentropyAdapter: NegentropyAdapter | null = null; EventEmitter.defaultMaxListeners = 100; const REGISTRY = 'hyperswarm'; const BATCH_SIZE = 100; +const NEGENTROPY_VERSION = 1; +const NEG_SESSION_IDLE_TIMEOUT_MS = 2 * 60 * 1000; +const NEG_MAX_IDS_PER_OPS_REQ = 1_000; +const NEG_MAX_IDS_PER_LOOKUP = 1_000; +const NEG_MAX_OPS_PER_PUSH = 256; +const NEG_MAX_BYTES_PER_PUSH = 512 * 1024; +const NEG_REPAIR_INTERVAL_MS = config.negentropyRepairIntervalSeconds * 1000; const connectionInfo: Record = {}; const knownNodes: Record = {}; const knownPeers: Record = {}; const addedPeers: Record = {}; const badPeers: Record = {}; +const peerSessions = new Map(); +const syncStats: MediatorSyncStats = { + modeSelectionsTotal: 0, + modeSelectionsLegacy: 0, + modeSelectionsNegentropy: 0, + queueOpsRelayed: 0, + queueOpsImported: 0, + queueDelayMs: createAggregateMetric(), + negentropySessionsStarted: 0, + negentropySessionsClosed: 0, + negentropySessionsCompleted: 0, + negentropySessionsFailed: 0, + negentropyRounds: 0, + negentropyHaveIds: 0, + negentropyNeedIds: 0, + negentropyOpsReqSent: 0, + negentropyOpsReqReceived: 0, + negentropyOpsPushSent: 0, + negentropyOpsPushReceived: 0, + opsApplied: 0, + opsRejected: 0, + bytesSent: 0, + bytesReceived: 0, + syncDurationMs: createAggregateMetric(), +}; let swarm: Hyperswarm | null = null; let nodeKey = ''; @@ -78,6 +257,10 @@ goodbye(() => { if (swarm) { swarm.destroy(); } + + void syncStore.stop().catch(error => { + log.error({ error }, 'syncStore stop error'); + }); }); async function createSwarm(): Promise { @@ -114,6 +297,7 @@ let syncQueue = asyncLib.queue( }; const json = JSON.stringify(msg); + syncStats.bytesSent += messageBytes(json); conn.write(json); } catch (error) { @@ -131,9 +315,6 @@ function addConnection(conn: HyperswarmConnection): void { log.info(`received connection from: ${peerName}`); - // Push the connection to the syncQueue instead of writing directly - syncQueue.push(conn); - connectionInfo[peerKey] = { connection: conn, key: peerKey, @@ -141,23 +322,562 @@ function addConnection(conn: HyperswarmConnection): void { nodeName: 'anon', did: '', lastSeen: new Date().getTime(), + capabilities: { + advertised: false, + negentropy: false, + version: null, + }, + syncMode: 'unknown', + syncStarted: false, + lastNegentropyRepairAt: 0, }; const peerNames = Object.values(connectionInfo).map(info => info.peerName); log.debug(`--- ${peerNames.length} nodes connected, detected nodes: ${peerNames.join(', ')}`); + + void sendPingToPeer(peerKey); } function closeConnection(peerKey: string): void { const conn = connectionInfo[peerKey]; + if (!conn) { + return; + } log.info(`* connection closed with: ${conn.peerName} (${conn.nodeName}) *`); delete connectionInfo[peerKey]; + closePeerSession(peerKey, 'connection_closed'); } function shortName(peerKey: string): string { return peerKey.slice(0, 4) + '-' + peerKey.slice(-4); } +function createBaseMessage(type: T): Omit & { type: T } { + return { + type, + time: new Date().toISOString(), + node: nodeInfo?.name || config.nodeName, + relays: [], + }; +} + +function buildPingMessage(): PingMessage { + return { + ...createBaseMessage('ping'), + peers: Object.keys(knownNodes), + capabilities: { + negentropy: true, + negentropyVersion: NEGENTROPY_VERSION, + }, + }; +} + +function sendToPeer(peerKey: string, msg: HyperMessage): boolean { + const conn = connectionInfo[peerKey]; + if (!conn) { + return false; + } + + const json = JSON.stringify(msg); + syncStats.bytesSent += messageBytes(json); + conn.connection.write(json); + return true; +} + +async function sendPingToPeer(peerKey: string): Promise { + const ping = buildPingMessage(); + if (sendToPeer(peerKey, ping)) { + log.debug(`* sent ping to: ${shortName(peerKey)}`); + } +} + +function createSessionId(peerKey: string): string { + const nonce = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36); + return `${Date.now().toString(36)}-${shortName(nodeKey)}-${shortName(peerKey)}-${nonce}`; +} + +function createPeerSession(peerKey: string, mode: SyncMode, initiator: boolean, sessionId?: string): PeerSyncSession { + const now = Date.now(); + const session: PeerSyncSession = { + sessionId: sessionId ?? createSessionId(peerKey), + peerKey, + mode, + initiator, + startedAt: now, + lastActivity: now, + pendingHaveIds: [], + pendingNeedIds: [], + rounds: 0, + maxRounds: config.negentropyMaxRoundsPerSession, + reconciliationComplete: false, + localClosed: false, + }; + peerSessions.set(peerKey, session); + connectionInfo[peerKey].syncMode = mode; + connectionInfo[peerKey].syncStarted = true; + if (mode === 'negentropy') { + syncStats.negentropySessionsStarted += 1; + connectionInfo[peerKey].lastNegentropyRepairAt = now; + } + return session; +} + +function touchPeerSession(peerKey: string): void { + const session = peerSessions.get(peerKey); + if (!session) { + return; + } + session.lastActivity = Date.now(); +} + +function closePeerSession(peerKey: string, reason: string): void { + const session = peerSessions.get(peerKey); + if (!session) { + return; + } + + peerSessions.delete(peerKey); + addAggregateSample(syncStats.syncDurationMs, Date.now() - session.startedAt); + const conn = connectionInfo[peerKey]; + if (conn && session.mode === 'negentropy') { + conn.lastNegentropyRepairAt = Date.now(); + syncStats.negentropySessionsClosed += 1; + if (reason === 'complete' || reason === 'remote_closed') { + syncStats.negentropySessionsCompleted += 1; + } else { + syncStats.negentropySessionsFailed += 1; + } + } + + log.debug({ + peer: shortName(peerKey), + mode: session.mode, + rounds: session.rounds, + pendingHave: session.pendingHaveIds.length, + pendingNeed: session.pendingNeedIds.length, + reason, + }, 'peer sync session closed'); +} + +function expireIdlePeerSessions(): void { + const now = Date.now(); + for (const [peerKey, session] of peerSessions.entries()) { + if (now - session.lastActivity > NEG_SESSION_IDLE_TIMEOUT_MS) { + sendNegClose(peerKey, session, 'idle_timeout'); + closePeerSession(peerKey, 'idle_timeout'); + } + } +} + +function choosePeerSyncMode(peerKey: string): SyncMode | null { + const conn = connectionInfo[peerKey]; + if (!conn) { + return null; + } + + return chooseNegotiatedSyncMode(conn.capabilities, NEGENTROPY_VERSION); +} + +function getActiveNegentropySessions(): number { + let count = 0; + for (const session of peerSessions.values()) { + if (session.mode === 'negentropy') { + count += 1; + } + } + return count; +} + +async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic' = 'connect'): Promise { + const conn = connectionInfo[peerKey]; + if (!conn) { + return; + } + + if (source === 'connect' && conn.syncStarted) { + return; + } + + const mode = source === 'connect' + ? choosePeerSyncMode(peerKey) + : conn.syncMode; + + if (!mode) { + return; + } + + if (source === 'connect' && mode === 'negentropy' && importQueue.length() > 0) { + setTimeout(() => void maybeStartPeerSync(peerKey, source), 1_000); + return; + } + + if (source === 'connect') { + syncStats.modeSelectionsTotal += 1; + if (mode === 'legacy') { + syncStats.modeSelectionsLegacy += 1; + } else { + syncStats.modeSelectionsNegentropy += 1; + } + } + + if (mode === 'legacy') { + if (!shouldAcceptLegacySync(conn.syncMode, config.legacySyncEnabled)) { + return; + } + + if (source !== 'connect') { + return; + } + + createPeerSession(peerKey, 'legacy', true, `legacy-${Date.now().toString(36)}`); + syncQueue.push(conn.connection); + log.info({ peer: shortName(peerKey), mode }, 'peer sync mode selected'); + return; + } + + const initiator = nodeKey.localeCompare(peerKey) < 0; + const hasActiveSession = peerSessions.has(peerKey); + const activeNegentropySessions = getActiveNegentropySessions(); + + const shouldStart = source === 'connect' + ? shouldStartConnectTimeNegentropy(mode, hasActiveSession, initiator) + : shouldSchedulePeriodicRepair({ + syncMode: mode, + hasActiveSession, + importQueueLength: importQueue.length(), + activeNegentropySessions, + maxConcurrentNegentropySessions: config.negentropyMaxConcurrentSessions, + lastRepairAtMs: conn.lastNegentropyRepairAt, + nowMs: Date.now(), + repairIntervalMs: NEG_REPAIR_INTERVAL_MS, + isInitiator: initiator, + }); + + conn.syncMode = 'negentropy'; + conn.syncStarted = true; + + if (!shouldStart) { + return; + } + + if (activeNegentropySessions >= config.negentropyMaxConcurrentSessions) { + log.debug( + { peer: shortName(peerKey), activeNegentropySessions, max: config.negentropyMaxConcurrentSessions, source }, + 'negentropy session deferred by concurrency limit' + ); + return; + } + + const session = createPeerSession(peerKey, 'negentropy', initiator); + log.info( + { peer: shortName(peerKey), mode, initiator, sessionId: session.sessionId, source }, + 'peer sync mode selected' + ); + await startNegentropyOpen(peerKey, session); +} + +async function runPeriodicNegentropyRepair(): Promise { + for (const peerKey in connectionInfo) { + try { + await maybeStartPeerSync(peerKey, 'periodic'); + } catch (error) { + log.error({ error, peer: shortName(peerKey) }, 'periodic negentropy repair error'); + } + } +} + +function buildSyncStatsSnapshot(): object { + return { + modeSelections: { + total: syncStats.modeSelectionsTotal, + legacy: syncStats.modeSelectionsLegacy, + negentropy: syncStats.modeSelectionsNegentropy, + fallbackCount: syncStats.modeSelectionsLegacy, + fallbackRate: safeRate(syncStats.modeSelectionsLegacy, syncStats.modeSelectionsTotal), + }, + queue: { + relayed: syncStats.queueOpsRelayed, + imported: syncStats.queueOpsImported, + delayMs: { + avg: averageAggregate(syncStats.queueDelayMs), + max: syncStats.queueDelayMs.max, + samples: syncStats.queueDelayMs.count, + }, + }, + negentropy: { + sessionsStarted: syncStats.negentropySessionsStarted, + sessionsClosed: syncStats.negentropySessionsClosed, + sessionsCompleted: syncStats.negentropySessionsCompleted, + sessionsFailed: syncStats.negentropySessionsFailed, + rounds: syncStats.negentropyRounds, + haveIds: syncStats.negentropyHaveIds, + needIds: syncStats.negentropyNeedIds, + opsRequested: syncStats.negentropyOpsReqSent, + opsRequestedReceived: syncStats.negentropyOpsReqReceived, + opsPushed: syncStats.negentropyOpsPushSent, + opsPushedReceived: syncStats.negentropyOpsPushReceived, + }, + gatekeeper: { + opsApplied: syncStats.opsApplied, + opsRejected: syncStats.opsRejected, + }, + transport: { + bytesSent: syncStats.bytesSent, + bytesReceived: syncStats.bytesReceived, + }, + syncDurationMs: { + avg: averageAggregate(syncStats.syncDurationMs), + max: syncStats.syncDurationMs.max, + sessions: syncStats.syncDurationMs.count, + }, + }; +} + +async function startNegentropyOpen(peerKey: string, session: PeerSyncSession): Promise { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + await negentropyAdapter.rebuildFromStore(); + const firstFrame = await negentropyAdapter.initiate(); + const msg: NegOpenMessage = { + ...createBaseMessage('neg_open'), + sessionId: session.sessionId, + round: session.rounds, + frame: encodeNegentropyFrame(firstFrame), + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_neg_open_failed'); + } +} + +async function sendOpsReq(peerKey: string, session: PeerSyncSession, ids: string[]): Promise { + const normalized = Array.from(new Set(ids.map(id => id.toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))); + const batches = chunkIds(normalized, NEG_MAX_IDS_PER_OPS_REQ); + + for (const batch of batches) { + const msg: OpsReqMessage = { + ...createBaseMessage('ops_req'), + sessionId: session.sessionId, + round: session.rounds, + ids: batch, + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_ops_req_failed'); + return; + } + syncStats.negentropyOpsReqSent += batch.length; + } +} + +async function sendOpsPushForIds(peerKey: string, session: PeerSyncSession, ids: string[]): Promise { + const normalized = Array.from(new Set(ids.map(id => id.toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))); + const idLookupBatches = chunkIds(normalized, NEG_MAX_IDS_PER_LOOKUP); + + for (const idBatch of idLookupBatches) { + const rows = await syncStore.getByIds(idBatch); + const operations = rows.map(row => row.operation); + if (operations.length === 0) { + continue; + } + + const opBatches = chunkOperationsForPush(operations, { + maxOpsPerPush: NEG_MAX_OPS_PER_PUSH, + maxBytesPerPush: NEG_MAX_BYTES_PER_PUSH, + }); + + for (const opBatch of opBatches) { + const msg: OpsPushMessage = { + ...createBaseMessage('ops_push'), + sessionId: session.sessionId, + round: session.rounds, + data: opBatch, + }; + + if (!sendToPeer(peerKey, msg)) { + closePeerSession(peerKey, 'send_ops_push_failed'); + return; + } + syncStats.negentropyOpsPushSent += opBatch.length; + } + } +} + +function sendNegMsg(peerKey: string, session: PeerSyncSession, frame: string | Uint8Array): boolean { + const msg: NegMsgMessage = { + ...createBaseMessage('neg_msg'), + sessionId: session.sessionId, + round: session.rounds, + frame: encodeNegentropyFrame(frame), + }; + + return sendToPeer(peerKey, msg); +} + +function sendNegClose(peerKey: string, session: PeerSyncSession, reason: string): boolean { + session.localClosed = true; + const closeMsg: NegCloseMessage = { + ...createBaseMessage('neg_close'), + sessionId: session.sessionId, + round: session.rounds, + reason, + }; + + return sendToPeer(peerKey, closeMsg); +} + +function mergeUniqueIds(current: string[], incoming: string[]): string[] { + if (incoming.length === 0) { + return current; + } + return Array.from(new Set([...current, ...incoming])); +} + +function removeKnownIds(current: string[], known: string[]): string[] { + if (known.length === 0 || current.length === 0) { + return current; + } + + const knownSet = new Set(known); + return current.filter(id => !knownSet.has(id)); +} + +async function reconcileNegentropyFrame( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise<{ + nextMsg: string | Uint8Array | null; + haveIds: string[]; + needIds: string[]; +} | null> { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + if (session.rounds >= session.maxRounds) { + sendNegClose(peerKey, session, 'max_rounds_reached'); + closePeerSession(peerKey, 'max_rounds_reached'); + return null; + } + + const result = await negentropyAdapter.reconcile(frame); + session.rounds += 1; + touchPeerSession(peerKey); + + return { + nextMsg: result.nextMsg, + haveIds: normalizeNegentropyIds(result.haveIds), + needIds: normalizeNegentropyIds(result.needIds), + }; +} + +function maybeFinalizeInitiatorSession(peerKey: string, session: PeerSyncSession): void { + if (!session.initiator) { + return; + } + + if (!session.reconciliationComplete) { + return; + } + + if (session.pendingNeedIds.length > 0) { + return; + } + + if (!sendNegClose(peerKey, session, 'complete')) { + closePeerSession(peerKey, 'send_neg_close_failed'); + return; + } + + closePeerSession(peerKey, 'complete'); +} + +async function handleNegentropyRoundAsInitiator( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise { + const outcome = await reconcileNegentropyFrame(peerKey, session, frame); + if (!outcome) { + return; + } + + session.pendingHaveIds = mergeUniqueIds(session.pendingHaveIds, outcome.haveIds); + const newNeedIds = removeKnownIds(outcome.needIds, session.pendingNeedIds); + session.pendingNeedIds = mergeUniqueIds(session.pendingNeedIds, newNeedIds); + syncStats.negentropyRounds += 1; + syncStats.negentropyHaveIds += outcome.haveIds.length; + syncStats.negentropyNeedIds += outcome.needIds.length; + + if (outcome.haveIds.length > 0) { + await sendOpsPushForIds(peerKey, session, outcome.haveIds); + } + + if (newNeedIds.length > 0) { + await sendOpsReq(peerKey, session, newNeedIds); + } + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + round: session.rounds, + have: outcome.haveIds.length, + need: outcome.needIds.length, + pendingNeed: session.pendingNeedIds.length, + }, + 'negentropy initiator round' + ); + + if (outcome.nextMsg !== null) { + if (!sendNegMsg(peerKey, session, outcome.nextMsg)) { + closePeerSession(peerKey, 'send_neg_msg_failed'); + } + return; + } + + session.reconciliationComplete = true; + maybeFinalizeInitiatorSession(peerKey, session); +} + +async function handleNegentropyRoundAsResponder( + peerKey: string, + session: PeerSyncSession, + frame: string | Uint8Array, +): Promise { + const outcome = await reconcileNegentropyFrame(peerKey, session, frame); + if (!outcome) { + return; + } + syncStats.negentropyRounds += 1; + syncStats.negentropyHaveIds += outcome.haveIds.length; + syncStats.negentropyNeedIds += outcome.needIds.length; + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + round: session.rounds, + have: outcome.haveIds.length, + need: outcome.needIds.length, + }, + 'negentropy responder round' + ); + + if (outcome.nextMsg !== null) { + if (!sendNegMsg(peerKey, session, outcome.nextMsg)) { + closePeerSession(peerKey, 'send_neg_msg_failed'); + } + return; + } + + session.reconciliationComplete = true; +} + function sendBatch(conn: HyperswarmConnection, batch: Operation[]): number { const limit = 8 * 1024 * 1014; // 8 MB limit @@ -172,6 +892,7 @@ function sendBatch(conn: HyperswarmConnection, batch: Operation[]): number { const json = JSON.stringify(msg); if (json.length < limit) { + syncStats.bytesSent += messageBytes(json); conn.write(json); log.debug(` * sent ${batch.length} ops in ${json.length} bytes`); return batch.length; @@ -248,6 +969,7 @@ async function relayMsg(msg: HyperMessage): Promise { const lastSeen = `last seen ${minutesSinceLastSeen} minutes ago ${last.toISOString()}`; if (!msg.relays.includes(peerKey)) { + syncStats.bytesSent += messageBytes(json); conn.connection.write(json); log.debug(`* relaying to: ${conn.peerName} (${conn.nodeName}) ${lastSeen} *`); } @@ -257,7 +979,7 @@ async function relayMsg(msg: HyperMessage): Promise { } } -async function importBatch(batch: Operation[]): Promise { +async function importBatch(batch: Operation[]): Promise { // The batch we receive from other hyperswarm nodes includes just operations. // We have to wrap the operations in new events before submitting to our gatekeeper for importing try { @@ -282,12 +1004,34 @@ async function importBatch(batch: Operation[]): Promise { const importDurationMs = Date.now() - importStart; log.debug({ durationMs: importDurationMs }, 'importBatch'); log.debug(`* ${JSON.stringify(response)}`); + syncStats.opsRejected += response.rejected ?? 0; + + return filterIndexRejectedOperations(batch, response.rejectedIndices); } catch (error) { log.error({ error }, 'importBatch error'); + return []; } } +async function persistAcceptedOperations(operations: Operation[], source: string): Promise { + if (!Array.isArray(operations) || operations.length === 0) { + return; + } + + const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); + if (records.length === 0) { + log.debug({ source, attempted: operations.length, invalid }, 'sync-store persist skipped'); + return; + } + + const inserted = await syncStore.upsertMany(records); + log.debug( + { source, attempted: operations.length, mapped: records.length, invalid, inserted }, + 'sync-store persist accepted ops' + ); +} + async function mergeBatch(batch: Operation[]): Promise { if (!batch) { @@ -295,17 +1039,21 @@ async function mergeBatch(batch: Operation[]): Promise { } let chunk = []; + const acceptedCandidates: Operation[] = []; + for (const operation of batch) { chunk.push(operation); if (chunk.length >= BATCH_SIZE) { - await importBatch(chunk); + const candidates = await importBatch(chunk); + acceptedCandidates.push(...candidates); chunk = []; } } if (chunk.length > 0) { - await importBatch(chunk); + const candidates = await importBatch(chunk); + acceptedCandidates.push(...candidates); } const processStart = Date.now(); @@ -313,6 +1061,11 @@ async function mergeBatch(batch: Operation[]): Promise { const processDurationMs = Date.now() - processStart; log.debug({ durationMs: processDurationMs }, 'processEvents'); log.debug(`mergeBatch: ${JSON.stringify(response)}`); + syncStats.opsApplied += (response.added ?? 0) + (response.merged ?? 0); + syncStats.opsRejected += response.rejected ?? 0; + + const acceptedToPersist = filterOperationsByAcceptedHashes(acceptedCandidates, response.acceptedHashes); + await persistAcceptedOperations(acceptedToPersist, 'mergeBatch'); } let importQueue = asyncLib.queue( @@ -328,6 +1081,14 @@ let importQueue = asyncLib.queue( return; } + if (msg.type === 'queue') { + syncStats.queueOpsImported += batch.length; + const samples = collectQueueDelaySamples(batch); + for (const sample of samples) { + addAggregateSample(syncStats.queueDelayMs, sample); + } + } + const nodeName = msg.node || 'anon'; log.debug(`* merging batch (${batch.length} events) from: ${shortName(name)} (${nodeName}) *`); await mergeBatch(batch); @@ -346,6 +1107,11 @@ let exportQueue = asyncLib.queue( const ready = await gatekeeper.isReady(); if (ready) { + const mode = connectionInfo[name]?.syncMode ?? 'unknown'; + if (!shouldAcceptLegacySync(mode, config.legacySyncEnabled)) { + log.debug({ peer: shortName(name), mode }, 'shareDb skipped by sync mode policy'); + return; + } log.debug(`* sharing db with: ${shortName(name)} (${msg.node || 'anon'}) *`); await shareDb(conn); } @@ -425,15 +1191,21 @@ async function addPeer(did: string): Promise { } } -async function receiveMsg(peerKey: string, json: string): Promise { +async function receiveMsg(peerKey: string, json: Buffer | string): Promise { const conn = connectionInfo[peerKey]; - let msg; + if (!conn) { + return; + } + + let msg: HyperMessage; + const payload = typeof json === 'string' ? json : json.toString('utf8'); + syncStats.bytesReceived += messageBytes(payload); try { - msg = JSON.parse(json); + msg = JSON.parse(payload); } catch (error) { - const jsonPreview = json.length > 80 ? `${json.slice(0, 40)}...${json.slice(-40)}` : json; + const jsonPreview = payload.length > 80 ? `${payload.slice(0, 40)}...${payload.slice(-40)}` : payload; log.warn(`received invalid message from: ${conn.peerName}, JSON: ${jsonPreview}`); return; } @@ -444,15 +1216,18 @@ async function receiveMsg(peerKey: string, json: string): Promise { connectionInfo[peerKey].lastSeen = new Date().getTime(); if (msg.type === 'batch') { - if (newBatch(msg.data)) { + if (Array.isArray(msg.data) && newBatch(msg.data)) { importQueue.push({ name: peerKey, msg }); } return; } if (msg.type === 'queue') { - if (newBatch(msg.data)) { + if (Array.isArray(msg.data) && newBatch(msg.data)) { importQueue.push({ name: peerKey, msg }); + if (!Array.isArray(msg.relays)) { + msg.relays = []; + } msg.relays.push(peerKey); await relayMsg(msg); } @@ -460,19 +1235,116 @@ async function receiveMsg(peerKey: string, json: string): Promise { } if (msg.type === 'sync') { + if (!shouldAcceptLegacySync(connectionInfo[peerKey].syncMode, config.legacySyncEnabled)) { + log.debug( + { peer: shortName(peerKey), mode: connectionInfo[peerKey].syncMode }, + 'ignoring legacy sync request' + ); + return; + } exportQueue.push({ name: peerKey, msg, conn: conn.connection }); return; } if (msg.type === 'ping') { connectionInfo[peerKey].nodeName = nodeName; + connectionInfo[peerKey].capabilities = normalizePeerCapabilities(msg.capabilities); - if (msg.peers) { + if (Array.isArray(msg.peers)) { for (const did of msg.peers) { addPeer(did); } } + await maybeStartPeerSync(peerKey); + return; + } + + if (msg.type === 'neg_open') { + if (!negentropyAdapter) { + log.warn('neg_open ignored because adapter is unavailable'); + return; + } + + const existing = peerSessions.get(peerKey); + if (existing && existing.sessionId !== msg.sessionId) { + closePeerSession(peerKey, 'replaced_by_remote_open'); + } + + await negentropyAdapter.rebuildFromStore(); + const session = createPeerSession(peerKey, 'negentropy', false, msg.sessionId); + touchPeerSession(peerKey); + await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); + return; + } + + if (msg.type === 'neg_msg') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_msg for unknown session'); + return; + } + + touchPeerSession(peerKey); + if (session.initiator) { + await handleNegentropyRoundAsInitiator(peerKey, session, decodeNegentropyFrame(msg.frame)); + } else { + await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); + } + return; + } + + if (msg.type === 'ops_req') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_req for unknown session'); + return; + } + + const requestedIds = Array.isArray(msg.ids) + ? Array.from(new Set(msg.ids.map(id => String(id).toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))) + : []; + syncStats.negentropyOpsReqReceived += requestedIds.length; + await sendOpsPushForIds(peerKey, session, requestedIds); + touchPeerSession(peerKey); + return; + } + + if (msg.type === 'ops_push') { + const session = peerSessions.get(peerKey); + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_push for unknown session'); + return; + } + + const batch = Array.isArray(msg.data) ? msg.data : []; + if (batch.length > 0) { + syncStats.negentropyOpsPushReceived += batch.length; + const pushedIds = new Set(extractOperationHashes(batch)); + session.pendingNeedIds = session.pendingNeedIds.filter(id => !pushedIds.has(id)); + + if (newBatch(batch)) { + importQueue.push({ + name: peerKey, + msg: { + ...createBaseMessage('batch'), + data: batch, + }, + }); + } + + maybeFinalizeInitiatorSession(peerKey, session); + } + + touchPeerSession(peerKey); + return; + } + + if (msg.type === 'neg_close') { + const session = peerSessions.get(peerKey); + if (session && session.sessionId === msg.sessionId) { + closePeerSession(peerKey, msg.reason || 'remote_closed'); + } return; } @@ -484,6 +1356,13 @@ async function flushQueue(): Promise { log.debug(`${REGISTRY} queue: ${JSON.stringify(batch, null, 4)}`); if (batch.length > 0) { + await persistAcceptedOperations(batch, 'flushQueue'); + syncStats.queueOpsRelayed += batch.length; + const samples = collectQueueDelaySamples(batch); + for (const sample of samples) { + addAggregateSample(syncStats.queueDelayMs, sample); + } + const msg: BatchMessage = { type: 'queue', time: new Date().toISOString(), @@ -522,6 +1401,8 @@ async function exportLoop(): Promise { } async function checkConnections(): Promise { + expireIdlePeerSessions(); + if (Object.keys(connectionInfo).length === 0) { log.warn("No active connections, rejoining the topic..."); await createSwarm(); @@ -537,7 +1418,7 @@ async function checkConnections(): Promise { if (timeSinceLastSeen > expireLimit) { log.info(`Removing stale connection info for: ${conn.peerName} (${conn.nodeName}), last seen ${timeSinceLastSeen / 1000}s ago`); - delete connectionInfo[peerKey]; + closeConnection(peerKey); } } } @@ -549,15 +1430,12 @@ async function connectionLoop(): Promise { await checkConnections(); - const msg: PingMessage = { - type: 'ping', - time: new Date().toISOString(), - node: nodeInfo.name, - relays: [], - peers: Object.keys(knownNodes), - }; + const msg: PingMessage = buildPingMessage(); await relayMsg(msg); + await runPeriodicNegentropyRepair(); + + log.debug({ syncStats: buildSyncStatsSnapshot() }, 'hyperswarm sync stats'); if (config.ipfsEnabled) { const peeringPeers = await ipfs.getPeeringPeers(); @@ -594,6 +1472,8 @@ const networkID = Buffer.from(hash).toString('hex'); const topic = Buffer.from(b4a.from(networkID, 'hex')); async function main(): Promise { + await syncStore.start(); + await gatekeeper.connect({ url: config.gatekeeperURL, waitUntilReady: true, @@ -601,6 +1481,11 @@ async function main(): Promise { chatty: true, }); + const bootstrap = await bootstrapSyncStoreIfEmpty(syncStore, gatekeeper); + log.info({ bootstrap }, 'sync-store bootstrap complete'); + + await initNegentropyAdapter(); + if (config.ipfsEnabled) { await keymaster.connect({ url: config.keymasterURL, @@ -663,4 +1548,46 @@ async function main(): Promise { await connectionLoop(); } -main(); +async function initNegentropyAdapter(): Promise { + negentropyAdapter = await NegentropyAdapter.create({ + syncStore, + frameSizeLimit: config.negentropyFrameSizeLimit, + recentWindowDays: config.negentropyRecentWindowDays, + olderWindowDays: config.negentropyOlderWindowDays, + maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, + maxRoundsPerSession: config.negentropyMaxRoundsPerSession, + deferInitialBuild: true, + }); + log.info( + { + stats: negentropyAdapter.getStats(), + recentWindowDays: config.negentropyRecentWindowDays, + olderWindowDays: config.negentropyOlderWindowDays, + maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, + maxRoundsPerSession: config.negentropyMaxRoundsPerSession, + frameSizeLimit: config.negentropyFrameSizeLimit, + }, + 'negentropy adapter initialized' + ); +} + +export async function runMediator(options: MediatorMainOptions = {}): Promise { + if (options.syncStore) { + syncStore = options.syncStore; + } + + if (options.startLoops === false) { + await syncStore.start(); + return; + } + + return main(); +} + +const isDirectRun = !!process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; +if (isDirectRun) { + runMediator().catch(error => { + log.error({ error }, 'fatal mediator error'); + process.exit(1); + }); +} diff --git a/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs b/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs new file mode 100644 index 000000000..ae802bbe3 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/Negentropy.cjs @@ -0,0 +1,589 @@ +// (C) 2023 Doug Hoyte. MIT license + +const PROTOCOL_VERSION = 0x61; // Version 1 +const ID_SIZE = 32; +const FINGERPRINT_SIZE = 16; + +const Mode = { + Skip: 0, + Fingerprint: 1, + IdList: 2, +}; + +class WrappedBuffer { + constructor(buffer) { + this._raw = new Uint8Array(buffer || 512); + this.length = buffer ? buffer.length : 0; + } + + unwrap() { + return this._raw.subarray(0, this.length); + } + + get capacity() { + return this._raw.byteLength; + } + + extend(buf) { + if (buf._raw) buf = buf.unwrap(); + if (typeof(buf.length) !== 'number') throw Error("bad length"); + const targetSize = buf.length + this.length; + if (this.capacity < targetSize) { + const oldRaw = this._raw; + const newCapacity = Math.max(this.capacity * 2, targetSize); + this._raw = new Uint8Array(newCapacity); + this._raw.set(oldRaw); + } + + this._raw.set(buf, this.length); + this.length += buf.length; + } + + shift() { + const first = this._raw[0]; + this._raw = this._raw.subarray(1); + this.length--; + return first; + } + + shiftN(n = 1) { + const firstSubarray = this._raw.subarray(0, n); + this._raw = this._raw.subarray(n); + this.length -= n; + return firstSubarray; + } +} + +function decodeVarInt(buf) { + let res = 0; + + while (1) { + if (buf.length === 0) throw Error("parse ends prematurely"); + let byte = buf.shift(); + res = (res << 7) | (byte & 127); + if ((byte & 128) === 0) break; + } + + return res; +} + +function encodeVarInt(n) { + if (n === 0) return new WrappedBuffer([0]); + + let o = []; + + while (n !== 0) { + o.push(n & 127); + n >>>= 7; + } + + o.reverse(); + + for (let i = 0; i < o.length - 1; i++) o[i] |= 128; + + return new WrappedBuffer(o); +} + +function getByte(buf) { + return getBytes(buf, 1)[0]; +} + +function getBytes(buf, n) { + if (buf.length < n) throw Error("parse ends prematurely"); + return buf.shiftN(n); +} + + +class Accumulator { + constructor() { + this.setToZero(); + + if (typeof window === 'undefined') { // node.js + const crypto = require('crypto'); + this.sha256 = async (slice) => new Uint8Array(crypto.createHash('sha256').update(slice).digest()); + } else { // browser + this.sha256 = async (slice) => new Uint8Array(await crypto.subtle.digest("SHA-256", slice)); + } + } + + setToZero() { + this.buf = new Uint8Array(ID_SIZE); + } + + add(otherBuf) { + let currCarry = 0, nextCarry = 0; + let p = new DataView(this.buf.buffer); + let po = new DataView(otherBuf.buffer); + + for (let i = 0; i < 8; i++) { + let offset = i * 4; + let orig = p.getUint32(offset, true); + let otherV = po.getUint32(offset, true); + + let next = orig; + + next += currCarry; + next += otherV; + if (next > 0xFFFFFFFF) nextCarry = 1; + + p.setUint32(offset, next & 0xFFFFFFFF, true); + currCarry = nextCarry; + nextCarry = 0; + } + } + + negate() { + let p = new DataView(this.buf.buffer); + + for (let i = 0; i < 8; i++) { + let offset = i * 4; + p.setUint32(offset, ~p.getUint32(offset, true)); + } + + let one = new Uint8Array(ID_SIZE); + one[0] = 1; + this.add(one); + } + + async getFingerprint(n) { + let input = new WrappedBuffer(); + input.extend(this.buf); + input.extend(encodeVarInt(n)); + + let hash = await this.sha256(input.unwrap()); + + return hash.subarray(0, FINGERPRINT_SIZE); + } +}; + + +class NegentropyStorageVector { + constructor() { + this.items = []; + this.sealed = false; + } + + insert(timestamp, id) { + if (this.sealed) throw Error("already sealed"); + id = loadInputBuffer(id); + if (id.byteLength !== ID_SIZE) throw Error("bad id size for added item"); + this.items.push({ timestamp, id }); + } + + seal() { + if (this.sealed) throw Error("already sealed"); + this.sealed = true; + + this.items.sort(itemCompare); + + for (let i = 1; i < this.items.length; i++) { + if (itemCompare(this.items[i - 1], this.items[i]) === 0) throw Error("duplicate item inserted"); + } + } + + unseal() { + this.sealed = false; + } + + size() { + this._checkSealed(); + return this.items.length; + } + + getItem(i) { + this._checkSealed(); + if (i >= this.items.length) throw Error("out of range"); + return this.items[i]; + } + + iterate(begin, end, cb) { + this._checkSealed(); + this._checkBounds(begin, end); + + for (let i = begin; i < end; ++i) { + if (!cb(this.items[i], i)) break; + } + } + + findLowerBound(begin, end, bound) { + this._checkSealed(); + this._checkBounds(begin, end); + + return this._binarySearch(this.items, begin, end, (a) => itemCompare(a, bound) < 0); + } + + async fingerprint(begin, end) { + let out = new Accumulator(); + out.setToZero(); + + this.iterate(begin, end, (item, i) => { + out.add(item.id); + return true; + }); + + return await out.getFingerprint(end - begin); + } + + _checkSealed() { + if (!this.sealed) throw Error("not sealed"); + } + + _checkBounds(begin, end) { + if (begin > end || end > this.items.length) throw Error("bad range"); + } + + _binarySearch(arr, first, last, cmp) { + let count = last - first; + + while (count > 0) { + let it = first; + let step = Math.floor(count / 2); + it += step; + + if (cmp(arr[it])) { + first = ++it; + count -= step + 1; + } else { + count = step; + } + } + + return first; + } +} + + +class Negentropy { + constructor(storage, frameSizeLimit = 0) { + if (frameSizeLimit !== 0 && frameSizeLimit < 4096) throw Error("frameSizeLimit too small"); + + this.storage = storage; + this.frameSizeLimit = frameSizeLimit; + + this.lastTimestampIn = 0; + this.lastTimestampOut = 0; + } + + _bound(timestamp, id) { + return { timestamp, id: id ? id : new Uint8Array(0) }; + } + + async initiate() { + if (this.isInitiator) throw Error("already initiated"); + this.isInitiator = true; + + let output = new WrappedBuffer(); + output.extend([ PROTOCOL_VERSION ]); + + await this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output); + + return this._renderOutput(output); + } + + setInitiator() { + this.isInitiator = true; + } + + async reconcile(query) { + let haveIds = [], needIds = []; + query = new WrappedBuffer(loadInputBuffer(query)); + + this.lastTimestampIn = this.lastTimestampOut = 0; // reset for each message + + let fullOutput = new WrappedBuffer(); + fullOutput.extend([ PROTOCOL_VERSION ]); + + let protocolVersion = getByte(query); + if (protocolVersion < 0x60 || protocolVersion > 0x6F) throw Error("invalid negentropy protocol version byte"); + if (protocolVersion !== PROTOCOL_VERSION) { + if (this.isInitiator) throw Error("unsupported negentropy protocol version requested: " + (protocolVersion - 0x60)); + else return [this._renderOutput(fullOutput), haveIds, needIds]; + } + + let storageSize = this.storage.size(); + let prevBound = this._bound(0); + let prevIndex = 0; + let skip = false; + + while (query.length !== 0) { + let o = new WrappedBuffer(); + + let doSkip = () => { + if (skip) { + skip = false; + o.extend(this.encodeBound(prevBound)); + o.extend(encodeVarInt(Mode.Skip)); + } + }; + + let currBound = this.decodeBound(query); + let mode = decodeVarInt(query); + + let lower = prevIndex; + let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound); + + if (mode === Mode.Skip) { + skip = true; + } else if (mode === Mode.Fingerprint) { + let theirFingerprint = getBytes(query, FINGERPRINT_SIZE); + let ourFingerprint = await this.storage.fingerprint(lower, upper); + + if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) { + doSkip(); + await this.splitRange(lower, upper, currBound, o); + } else { + skip = true; + } + } else if (mode === Mode.IdList) { + let numIds = decodeVarInt(query); + + let theirElems = {}; // stringified Uint8Array -> original Uint8Array (or hex) + for (let i = 0; i < numIds; i++) { + let e = getBytes(query, ID_SIZE); + if (this.isInitiator) theirElems[e] = e; + } + + if (this.isInitiator) { + skip = true; + + this.storage.iterate(lower, upper, (item) => { + let k = item.id; + + if (!theirElems[k]) { + // ID exists on our side, but not their side + if (this.isInitiator) haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k)); + } else { + // ID exists on both sides + delete theirElems[k]; + } + + return true; + }); + + for (let v of Object.values(theirElems)) { + // ID exists on their side, but not our side + needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v)); + } + } else { + doSkip(); + + let responseIds = new WrappedBuffer(); + let numResponseIds = 0; + let endBound = currBound; + + this.storage.iterate(lower, upper, (item, index) => { + if (this.exceededFrameSizeLimit(fullOutput.length + responseIds.length)) { + endBound = item; + upper = index; // shrink upper so that remaining range gets correct fingerprint + return false; + } + + responseIds.extend(item.id); + numResponseIds++; + return true; + }); + + o.extend(this.encodeBound(endBound)); + o.extend(encodeVarInt(Mode.IdList)); + o.extend(encodeVarInt(numResponseIds)); + o.extend(responseIds); + + fullOutput.extend(o); + o = new WrappedBuffer(); + } + } else { + throw Error("unexpected mode"); + } + + if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) { + // frameSizeLimit exceeded: Stop range processing and return a fingerprint for the remaining range + let remainingFingerprint = await this.storage.fingerprint(upper, storageSize); + + fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE))); + fullOutput.extend(encodeVarInt(Mode.Fingerprint)); + fullOutput.extend(remainingFingerprint); + break; + } else { + fullOutput.extend(o); + } + + prevIndex = upper; + prevBound = currBound; + } + + return [fullOutput.length === 1 && this.isInitiator ? null : this._renderOutput(fullOutput), haveIds, needIds]; + } + + async splitRange(lower, upper, upperBound, o) { + let numElems = upper - lower; + let buckets = 16; + + if (numElems < buckets * 2) { + o.extend(this.encodeBound(upperBound)); + o.extend(encodeVarInt(Mode.IdList)); + + o.extend(encodeVarInt(numElems)); + this.storage.iterate(lower, upper, (item) => { + o.extend(item.id); + return true; + }); + } else { + let itemsPerBucket = Math.floor(numElems / buckets); + let bucketsWithExtra = numElems % buckets; + let curr = lower; + + for (let i = 0; i < buckets; i++) { + let bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0); + let ourFingerprint = await this.storage.fingerprint(curr, curr + bucketSize); + curr += bucketSize; + + let nextBound; + + if (curr === upper) { + nextBound = upperBound; + } else { + let prevItem, currItem; + + this.storage.iterate(curr - 1, curr + 1, (item, index) => { + if (index === curr - 1) prevItem = item; + else currItem = item; + return true; + }); + + nextBound = this.getMinimalBound(prevItem, currItem); + } + + o.extend(this.encodeBound(nextBound)); + o.extend(encodeVarInt(Mode.Fingerprint)); + o.extend(ourFingerprint); + } + } + } + + _renderOutput(o) { + o = o.unwrap(); + if (!this.wantUint8ArrayOutput) o = uint8ArrayToHex(o); + return o; + } + + exceededFrameSizeLimit(n) { + return this.frameSizeLimit && n > this.frameSizeLimit - 200; + } + + // Decoding + + decodeTimestampIn(encoded) { + let timestamp = decodeVarInt(encoded); + timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1; + if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) { + this.lastTimestampIn = Number.MAX_VALUE; + return Number.MAX_VALUE; + } + timestamp += this.lastTimestampIn; + this.lastTimestampIn = timestamp; + return timestamp; + } + + decodeBound(encoded) { + let timestamp = this.decodeTimestampIn(encoded); + let len = decodeVarInt(encoded); + if (len > ID_SIZE) throw Error("bound key too long"); + let id = getBytes(encoded, len); + return { timestamp, id }; + } + + // Encoding + + encodeTimestampOut(timestamp) { + if (timestamp === Number.MAX_VALUE) { + this.lastTimestampOut = Number.MAX_VALUE; + return encodeVarInt(0); + } + + let temp = timestamp; + timestamp -= this.lastTimestampOut; + this.lastTimestampOut = temp; + return encodeVarInt(timestamp + 1); + } + + encodeBound(key) { + let output = new WrappedBuffer(); + + output.extend(this.encodeTimestampOut(key.timestamp)); + output.extend(encodeVarInt(key.id.length)); + output.extend(key.id); + + return output; + } + + getMinimalBound(prev, curr) { + if (curr.timestamp !== prev.timestamp) { + return this._bound(curr.timestamp); + } else { + let sharedPrefixBytes = 0; + let currKey = curr.id; + let prevKey = prev.id; + + for (let i = 0; i < ID_SIZE; i++) { + if (currKey[i] !== prevKey[i]) break; + sharedPrefixBytes++; + } + + return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1)); + } + }; +} + +function loadInputBuffer(inp) { + if (typeof(inp) === 'string') inp = hexToUint8Array(inp); + else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp); // node Buffer? + return inp; +} + +function hexToUint8Array(h) { + if (h.startsWith('0x')) h = h.substr(2); + if (h.length % 2 === 1) throw Error("odd length of hex string"); + let arr = new Uint8Array(h.length / 2); + for (let i = 0; i < arr.length; i++) arr[i] = parseInt(h.substr(i * 2, 2), 16); + return arr; +} + +const uint8ArrayToHexLookupTable = new Array(256); +{ + const hexAlphabet = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + for (let i = 0; i < 256; i++) { + uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xF] + hexAlphabet[i & 0xF]; + } +} + +function uint8ArrayToHex(arr) { + let out = ''; + for (let i = 0, edx = arr.length; i < edx; i++) { + out += uint8ArrayToHexLookupTable[arr[i]]; + } + return out; +} + + +function compareUint8Array(a, b) { + for (let i = 0; i < a.byteLength; i++) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + + if (a.byteLength > b.byteLength) return 1; + if (a.byteLength < b.byteLength) return -1; + + return 0; +} + +function itemCompare(a, b) { + if (a.timestamp === b.timestamp) { + return compareUint8Array(a.id, b.id); + } + + return a.timestamp - b.timestamp; +} + + +module.exports = { Negentropy, NegentropyStorageVector, }; diff --git a/services/mediators/hyperswarm/src/negentropy/adapter.ts b/services/mediators/hyperswarm/src/negentropy/adapter.ts new file mode 100644 index 000000000..2e6f99d4b --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/adapter.ts @@ -0,0 +1,475 @@ +import { createRequire } from 'module'; +import { childLogger } from '@mdip/common/logger'; +import type { SyncStoreCursor } from '../db/types.js'; +import type { OperationSyncStore } from '../db/types.js'; + +const log = childLogger({ service: 'hyperswarm-negentropy-adapter' }); + +const MIN_FRAME_SIZE_LIMIT = 4096; +const DEFAULT_ITERATE_LIMIT = 1000; +const DEFAULT_RECENT_WINDOW_DAYS = 7; +const DEFAULT_OLDER_WINDOW_DAYS = 30; +const DEFAULT_MAX_RECORDS_PER_WINDOW = 25_000; +const DEFAULT_MAX_ROUNDS_PER_SESSION = 64; +const DAY_MS = 24 * 60 * 60 * 1000; +const require = createRequire(import.meta.url); + +interface NegentropyInstance { + wantUint8ArrayOutput?: boolean; + initiate(): Promise; + reconcile( + msg: string | Uint8Array + ): Promise<[string | Uint8Array | null, Array, Array]>; +} + +interface NegentropyStorageVectorInstance { + insert(timestamp: number, id: string | Uint8Array | Buffer): void; + seal(): void; +} + +interface NegentropyModule { + Negentropy: new (storage: NegentropyStorageVectorInstance, frameSizeLimit?: number) => NegentropyInstance; + NegentropyStorageVector: new () => NegentropyStorageVectorInstance; +} + +export interface NegentropyAdapterOptions { + syncStore: OperationSyncStore; + frameSizeLimit?: number; + iterateLimit?: number; + wantUint8ArrayOutput?: boolean; + recentWindowDays?: number; + olderWindowDays?: number; + maxRecordsPerWindow?: number; + maxRoundsPerSession?: number; + deferInitialBuild?: boolean; +} + +export interface ReconciliationWindow { + name: string; + fromTs: number; + toTs: number; + maxRecords: number; + order: number; +} + +export interface NegentropyReconcileResult { + nextMsg: string | Uint8Array | null; + haveIds: Array; + needIds: Array; +} + +export interface NegentropyAdapterStats { + loaded: number; + skipped: number; + durationMs: number; + frameSizeLimit: number; +} + +export interface NegentropyWindowStats extends NegentropyAdapterStats { + windowName: string; + fromTs: number; + toTs: number; + rounds: number; + completed: boolean; + cappedByRecords: boolean; + cappedByRounds: boolean; +} + +export interface NegentropySessionStats { + windowCount: number; + rounds: number; + loaded: number; + skipped: number; + durationMs: number; + windows: NegentropyWindowStats[]; +} + +export interface NegentropyWindowSessionOptions { + nowMs?: number; + maxRoundsPerSession?: number; +} + +export default class NegentropyAdapter { + private readonly syncStore: OperationSyncStore; + private readonly frameSizeLimit: number; + private readonly iterateLimit: number; + private readonly wantUint8ArrayOutput: boolean; + private readonly recentWindowDays: number; + private readonly olderWindowDays: number; + private readonly maxRecordsPerWindow: number; + private readonly maxRoundsPerSession: number; + private readonly mod: NegentropyModule; + private ne: NegentropyInstance | null = null; + private stats: NegentropyAdapterStats = { + loaded: 0, + skipped: 0, + durationMs: 0, + frameSizeLimit: 0, + }; + private lastWindowStats: NegentropyWindowStats | null = null; + private lastSessionStats: NegentropySessionStats | null = null; + + static async create(options: NegentropyAdapterOptions): Promise { + const adapter = new NegentropyAdapter(options); + if (!options.deferInitialBuild) { + await adapter.rebuildFromStore(); + } + return adapter; + } + + constructor(options: NegentropyAdapterOptions) { + this.syncStore = options.syncStore; + this.frameSizeLimit = options.frameSizeLimit ?? 0; + this.iterateLimit = options.iterateLimit ?? DEFAULT_ITERATE_LIMIT; + this.wantUint8ArrayOutput = options.wantUint8ArrayOutput ?? false; + this.recentWindowDays = options.recentWindowDays ?? DEFAULT_RECENT_WINDOW_DAYS; + this.olderWindowDays = options.olderWindowDays ?? DEFAULT_OLDER_WINDOW_DAYS; + this.maxRecordsPerWindow = options.maxRecordsPerWindow ?? DEFAULT_MAX_RECORDS_PER_WINDOW; + this.maxRoundsPerSession = options.maxRoundsPerSession ?? DEFAULT_MAX_ROUNDS_PER_SESSION; + this.mod = loadNegentropyModule(); + + if (this.frameSizeLimit !== 0 && this.frameSizeLimit < MIN_FRAME_SIZE_LIMIT) { + throw new Error(`negentropy frameSizeLimit must be 0 or >= ${MIN_FRAME_SIZE_LIMIT}`); + } + + assertPositiveInteger(this.iterateLimit, 'iterateLimit'); + assertPositiveInteger(this.recentWindowDays, 'recentWindowDays'); + assertPositiveInteger(this.olderWindowDays, 'olderWindowDays'); + assertPositiveInteger(this.maxRecordsPerWindow, 'maxRecordsPerWindow'); + assertPositiveInteger(this.maxRoundsPerSession, 'maxRoundsPerSession'); + } + + async rebuildFromStore(): Promise { + const stats = await this.rebuildWindowAdapter({ + name: 'full_history', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: Number.MAX_SAFE_INTEGER, + order: 0, + }); + + return { + loaded: stats.loaded, + skipped: stats.skipped, + durationMs: stats.durationMs, + frameSizeLimit: stats.frameSizeLimit, + }; + } + + async rebuildForWindow(window: ReconciliationWindow): Promise { + return this.rebuildWindowAdapter(window); + } + + async planWindows(nowMs: number = Date.now(), earliestTsOverride?: number): Promise { + if (!Number.isFinite(nowMs)) { + throw new Error('nowMs must be a finite timestamp'); + } + + const earliestTs = typeof earliestTsOverride === 'number' + ? earliestTsOverride + : await this.getEarliestTimestamp(); + + if (earliestTs == null) { + return []; + } + + const windows: ReconciliationWindow[] = []; + const recentSpanMs = this.recentWindowDays * DAY_MS; + const olderSpanMs = this.olderWindowDays * DAY_MS; + const recentStart = Math.max(earliestTs, nowMs - recentSpanMs); + + windows.push({ + name: 'recent', + fromTs: recentStart, + toTs: nowMs, + maxRecords: this.maxRecordsPerWindow, + order: 0, + }); + + let cursorTo = recentStart - 1; + let order = 1; + while (cursorTo >= earliestTs) { + const fromTs = Math.max(earliestTs, cursorTo - olderSpanMs + 1); + windows.push({ + name: `older_${order}`, + fromTs, + toTs: cursorTo, + maxRecords: this.maxRecordsPerWindow, + order, + }); + cursorTo = fromTs - 1; + order += 1; + } + + return windows; + } + + async runWindowedSessionWithPeer( + peer: NegentropyAdapter, + options: NegentropyWindowSessionOptions = {}, + ): Promise { + const startedAt = Date.now(); + const sessionNowMs = options.nowMs ?? Date.now(); + const maxRoundsPerSession = options.maxRoundsPerSession ?? this.maxRoundsPerSession; + assertPositiveInteger(maxRoundsPerSession, 'maxRoundsPerSession'); + + const [localEarliest, peerEarliest] = await Promise.all([ + this.getEarliestTimestamp(), + peer.getEarliestTimestamp(), + ]); + const earliestCandidates = [localEarliest, peerEarliest] + .filter((value): value is number => typeof value === 'number'); + + const windows = earliestCandidates.length === 0 + ? [] + : await this.planWindows(sessionNowMs, Math.min(...earliestCandidates)); + + const windowStats: NegentropyWindowStats[] = []; + let totalLoaded = 0; + let totalSkipped = 0; + let totalRounds = 0; + + for (const window of windows) { + const localWindowStats = await this.rebuildForWindow(window); + await peer.rebuildForWindow(window); + + const roundsResult = await this.reconcileWindowWithPeer(peer, maxRoundsPerSession); + const mergedStats: NegentropyWindowStats = { + ...localWindowStats, + rounds: roundsResult.rounds, + completed: roundsResult.completed, + cappedByRounds: roundsResult.cappedByRounds, + }; + + windowStats.push(mergedStats); + totalLoaded += mergedStats.loaded; + totalSkipped += mergedStats.skipped; + totalRounds += mergedStats.rounds; + + (log as any).debug?.( + { + windowName: mergedStats.windowName, + fromTs: mergedStats.fromTs, + toTs: mergedStats.toTs, + loaded: mergedStats.loaded, + skipped: mergedStats.skipped, + durationMs: mergedStats.durationMs, + rounds: mergedStats.rounds, + completed: mergedStats.completed, + cappedByRecords: mergedStats.cappedByRecords, + cappedByRounds: mergedStats.cappedByRounds, + }, + 'negentropy window session' + ); + } + + const sessionStats: NegentropySessionStats = { + windowCount: windowStats.length, + rounds: totalRounds, + loaded: totalLoaded, + skipped: totalSkipped, + durationMs: Date.now() - startedAt, + windows: windowStats, + }; + + this.lastSessionStats = sessionStats; + (log as any).debug?.( + { + windowCount: sessionStats.windowCount, + rounds: sessionStats.rounds, + loaded: sessionStats.loaded, + skipped: sessionStats.skipped, + durationMs: sessionStats.durationMs, + }, + 'negentropy windowed session summary' + ); + return sessionStats; + } + + getStats(): NegentropyAdapterStats { + return { ...this.stats }; + } + + getLastWindowStats(): NegentropyWindowStats | null { + return this.lastWindowStats ? { ...this.lastWindowStats } : null; + } + + getLastSessionStats(): NegentropySessionStats | null { + if (!this.lastSessionStats) { + return null; + } + + return { + ...this.lastSessionStats, + windows: this.lastSessionStats.windows.map(window => ({ ...window })), + }; + } + + async initiate(): Promise { + if (!this.ne) { + throw new Error('negentropy adapter not initialized'); + } + return this.ne.initiate(); + } + + async reconcile(msg: string | Uint8Array): Promise { + if (!this.ne) { + throw new Error('negentropy adapter not initialized'); + } + + const [nextMsg, haveIds, needIds] = await this.ne.reconcile(msg); + return { nextMsg, haveIds, needIds }; + } + + async respond(msg: string | Uint8Array): Promise { + const result = await this.reconcile(msg); + return result.nextMsg; + } + + private async rebuildWindowAdapter(window: ReconciliationWindow): Promise { + const startedAt = Date.now(); + const storage = new this.mod.NegentropyStorageVector(); + + let loaded = 0; + let skipped = 0; + let processed = 0; + let cappedByRecords = false; + let after: SyncStoreCursor | undefined; + + while (true) { + const rows = await this.syncStore.iterateSorted({ + after, + limit: this.iterateLimit, + fromTs: window.fromTs, + toTs: window.toTs, + }); + + if (rows.length === 0) { + break; + } + + for (const row of rows) { + if (processed >= window.maxRecords) { + cappedByRecords = true; + break; + } + + processed += 1; + if (!isValidSyncId(row.id) || !Number.isFinite(row.ts)) { + skipped += 1; + continue; + } + + storage.insert(row.ts, row.id); + loaded += 1; + } + + const last = rows[rows.length - 1]; + after = { ts: last.ts, id: last.id }; + + if (cappedByRecords) { + break; + } + } + + storage.seal(); + this.ne = new this.mod.Negentropy(storage, this.frameSizeLimit); + if (this.wantUint8ArrayOutput) { + this.ne.wantUint8ArrayOutput = true; + } + + const durationMs = Date.now() - startedAt; + this.stats = { + loaded, + skipped, + durationMs, + frameSizeLimit: this.frameSizeLimit, + }; + + const windowStats: NegentropyWindowStats = { + ...this.stats, + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + rounds: 0, + completed: true, + cappedByRecords, + cappedByRounds: false, + }; + + this.lastWindowStats = windowStats; + + (log as any).debug?.( + { + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + loaded, + skipped, + durationMs, + cappedByRecords, + frameSizeLimit: this.frameSizeLimit, + }, + 'negentropy adapter rebuilt' + ); + + return windowStats; + } + + private async getEarliestTimestamp(): Promise { + const rows = await this.syncStore.iterateSorted({ limit: 1 }); + if (rows.length === 0) { + return null; + } + return rows[0].ts; + } + + private async reconcileWindowWithPeer( + peer: NegentropyAdapter, + maxRounds: number, + ): Promise<{ rounds: number; completed: boolean; cappedByRounds: boolean }> { + let msg: string | Uint8Array | null = await this.initiate(); + let rounds = 0; + + while (msg !== null && rounds < maxRounds) { + rounds += 1; + const response = await peer.respond(msg); + if (response === null) { + msg = null; + break; + } + + const reconcileResult = await this.reconcile(response); + msg = reconcileResult.nextMsg; + } + + const completed = msg === null; + return { + rounds, + completed, + cappedByRounds: !completed && rounds >= maxRounds, + }; + } +} + +function isValidSyncId(id: string): boolean { + return typeof id === 'string' && /^[a-f0-9]{64}$/i.test(id); +} + +function assertPositiveInteger(value: number, name: string): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function loadNegentropyModule(): NegentropyModule { + const cjs = require('./Negentropy.cjs'); + const Negentropy = cjs?.Negentropy; + const NegentropyStorageVector = cjs?.NegentropyStorageVector; + + if (typeof Negentropy !== 'function' || typeof NegentropyStorageVector !== 'function') { + throw new Error('Invalid local negentropy module exports'); + } + + return { Negentropy, NegentropyStorageVector }; +} diff --git a/services/mediators/hyperswarm/src/negentropy/observability.ts b/services/mediators/hyperswarm/src/negentropy/observability.ts new file mode 100644 index 000000000..a4875c6b3 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/observability.ts @@ -0,0 +1,78 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface AggregateMetric { + count: number; + total: number; + max: number; +} + +export function createAggregateMetric(): AggregateMetric { + return { + count: 0, + total: 0, + max: 0, + }; +} + +export function addAggregateSample(metric: AggregateMetric, sample: number): void { + if (!Number.isFinite(sample) || sample < 0) { + return; + } + + metric.count += 1; + metric.total += sample; + if (sample > metric.max) { + metric.max = sample; + } +} + +export function averageAggregate(metric: AggregateMetric): number { + if (metric.count === 0) { + return 0; + } + + return metric.total / metric.count; +} + +export function messageBytes(payload: string | Buffer | object): number { + if (typeof payload === 'string') { + return Buffer.byteLength(payload, 'utf8'); + } + + if (Buffer.isBuffer(payload)) { + return payload.byteLength; + } + + return Buffer.byteLength(JSON.stringify(payload), 'utf8'); +} + +export function collectQueueDelaySamples(operations: Operation[], nowMs: number = Date.now()): number[] { + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + const samples: number[] = []; + for (const operation of operations) { + const signed = operation.signature?.signed; + if (typeof signed !== 'string' || signed === '') { + continue; + } + + const ts = Date.parse(signed); + if (!Number.isFinite(ts)) { + continue; + } + + samples.push(Math.max(0, nowMs - ts)); + } + + return samples; +} + +export function safeRate(numerator: number, denominator: number): number { + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) { + return 0; + } + + return numerator / denominator; +} diff --git a/services/mediators/hyperswarm/src/negentropy/policy.ts b/services/mediators/hyperswarm/src/negentropy/policy.ts new file mode 100644 index 000000000..cd4203b10 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/policy.ts @@ -0,0 +1,60 @@ +import type { SyncMode } from './protocol.js'; + +export interface RepairSchedulingInput { + syncMode: SyncMode | 'unknown'; + hasActiveSession: boolean; + importQueueLength: number; + activeNegentropySessions: number; + maxConcurrentNegentropySessions: number; + lastRepairAtMs: number; + nowMs: number; + repairIntervalMs: number; + isInitiator: boolean; +} + +export function shouldAcceptLegacySync( + syncMode: SyncMode | 'unknown', + legacySyncEnabled: boolean, +): boolean { + if (!legacySyncEnabled) { + return false; + } + + return syncMode !== 'negentropy'; +} + +export function shouldStartConnectTimeNegentropy( + syncMode: SyncMode | 'unknown', + hasActiveSession: boolean, + isInitiator: boolean, +): boolean { + return syncMode === 'negentropy' && !hasActiveSession && isInitiator; +} + +export function shouldSchedulePeriodicRepair(input: RepairSchedulingInput): boolean { + if (input.syncMode !== 'negentropy') { + return false; + } + + if (!input.isInitiator) { + return false; + } + + if (input.hasActiveSession) { + return false; + } + + if (input.importQueueLength > 0) { + return false; + } + + if (input.activeNegentropySessions >= input.maxConcurrentNegentropySessions) { + return false; + } + + if (input.lastRepairAtMs <= 0) { + return true; + } + + return (input.nowMs - input.lastRepairAtMs) >= input.repairIntervalMs; +} diff --git a/services/mediators/hyperswarm/src/negentropy/protocol.ts b/services/mediators/hyperswarm/src/negentropy/protocol.ts new file mode 100644 index 000000000..204eaf679 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/protocol.ts @@ -0,0 +1,110 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export type SyncMode = 'legacy' | 'negentropy'; +export type NegentropyFrameEncoding = 'utf8' | 'base64'; +export const NEG_SYNC_ID_RE = /^[a-f0-9]{64}$/i; + +export interface PeerCapabilities { + negentropy?: boolean; + negentropyVersion?: number; +} + +export interface NegotiatedPeerCapabilities { + advertised: boolean; + negentropy: boolean; + version: number | null; +} + +export interface NegentropyFrame { + encoding: NegentropyFrameEncoding; + data: string; +} + +export function normalizePeerCapabilities(capabilities?: PeerCapabilities): NegotiatedPeerCapabilities { + if (!capabilities) { + return { + advertised: false, + negentropy: false, + version: null, + }; + } + + return { + advertised: true, + negentropy: capabilities.negentropy === true, + version: typeof capabilities.negentropyVersion === 'number' + ? capabilities.negentropyVersion + : null, + }; +} + +export function supportsPeerNegentropy( + capabilities: NegotiatedPeerCapabilities, + minVersion: number +): boolean { + return capabilities.advertised + && capabilities.negentropy + && (capabilities.version == null || capabilities.version >= minVersion); +} + +export function chooseSyncMode( + capabilities: NegotiatedPeerCapabilities, + minVersion: number +): SyncMode | null { + if (!capabilities.advertised) { + return null; + } + + return supportsPeerNegentropy(capabilities, minVersion) + ? 'negentropy' + : 'legacy'; +} + +export function encodeNegentropyFrame(frame: string | Uint8Array): NegentropyFrame { + if (typeof frame === 'string') { + return { + encoding: 'utf8', + data: frame, + }; + } + + return { + encoding: 'base64', + data: Buffer.from(frame).toString('base64'), + }; +} + +export function decodeNegentropyFrame(frame: NegentropyFrame): string | Uint8Array { + if (frame.encoding === 'utf8') { + return frame.data; + } + + return Buffer.from(frame.data, 'base64'); +} + +export function normalizeNegentropyIds(ids: Array): string[] { + const unique = new Set(); + for (const id of ids) { + const hex = typeof id === 'string' + ? id.toLowerCase() + : Buffer.from(id).toString('hex').toLowerCase(); + + if (NEG_SYNC_ID_RE.test(hex)) { + unique.add(hex); + } + } + + return Array.from(unique); +} + +export function extractOperationHashes(operations: Operation[]): string[] { + const unique = new Set(); + for (const operation of operations) { + const hash = operation.signature?.hash?.toLowerCase(); + if (hash && NEG_SYNC_ID_RE.test(hash)) { + unique.add(hash); + } + } + + return Array.from(unique); +} diff --git a/services/mediators/hyperswarm/src/negentropy/transfer.ts b/services/mediators/hyperswarm/src/negentropy/transfer.ts new file mode 100644 index 000000000..be86a4d65 --- /dev/null +++ b/services/mediators/hyperswarm/src/negentropy/transfer.ts @@ -0,0 +1,70 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +export interface OpsPushBatchingOptions { + maxOpsPerPush: number; + maxBytesPerPush: number; +} + +export function chunkIds(ids: string[], maxPerChunk: number): string[][] { + if (!Number.isInteger(maxPerChunk) || maxPerChunk <= 0) { + throw new Error('maxPerChunk must be a positive integer'); + } + + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const unique = Array.from(new Set(ids)); + const chunks: string[][] = []; + for (let i = 0; i < unique.length; i += maxPerChunk) { + chunks.push(unique.slice(i, i + maxPerChunk)); + } + return chunks; +} + +export function chunkOperationsForPush( + operations: Operation[], + options: OpsPushBatchingOptions, +): Operation[][] { + const { maxOpsPerPush, maxBytesPerPush } = options; + if (!Number.isInteger(maxOpsPerPush) || maxOpsPerPush <= 0) { + throw new Error('maxOpsPerPush must be a positive integer'); + } + + if (!Number.isInteger(maxBytesPerPush) || maxBytesPerPush <= 0) { + throw new Error('maxBytesPerPush must be a positive integer'); + } + + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + const batches: Operation[][] = []; + let current: Operation[] = []; + let currentBytes = 0; + + for (const operation of operations) { + const operationBytes = estimateOperationBytes(operation); + const exceedsCount = current.length >= maxOpsPerPush; + const exceedsBytes = current.length > 0 && (currentBytes + operationBytes) > maxBytesPerPush; + + if (exceedsCount || exceedsBytes) { + batches.push(current); + current = []; + currentBytes = 0; + } + + current.push(operation); + currentBytes += operationBytes; + } + + if (current.length > 0) { + batches.push(current); + } + + return batches; +} + +export function estimateOperationBytes(operation: Operation): number { + return Buffer.byteLength(JSON.stringify(operation), 'utf8'); +} diff --git a/services/mediators/hyperswarm/src/sync-mapping.ts b/services/mediators/hyperswarm/src/sync-mapping.ts new file mode 100644 index 000000000..f52178853 --- /dev/null +++ b/services/mediators/hyperswarm/src/sync-mapping.ts @@ -0,0 +1,95 @@ +import { Operation } from '@mdip/gatekeeper/types'; + +// - id = operation.signature.hash (64 hex chars => 32 bytes) +// - timestamp = Date.parse(operation.signature.signed) in ms +export const SYNC_ID_HEX_LEN = 64; +export const SYNC_ID_BYTES_LEN = 32; + +export interface SyncMappedOperation { + idHex: string; + idBytes: Buffer; + tsMs: number; + operation: Operation; +} + +export type SyncMappingErrorCode = + | 'missing_signature' + | 'missing_signature_hash' + | 'invalid_signature_hash_type' + | 'invalid_signature_hash_format' + | 'missing_signature_signed' + | 'invalid_signature_signed_type' + | 'invalid_signature_signed_value'; + +export interface SyncMappingFailure { + ok: false; + code: SyncMappingErrorCode; + reason: string; +} + +export interface SyncMappingSuccess { + ok: true; + value: SyncMappedOperation; +} + +export type SyncMappingResult = SyncMappingFailure | SyncMappingSuccess; + +function fail(code: SyncMappingErrorCode, reason: string): SyncMappingFailure { + return { ok: false, code, reason }; +} + +export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { + const signature = operation.signature; + + if (!signature) { + return fail('missing_signature', 'operation.signature is required'); + } + + if (signature.hash == null || signature.hash === '') { + return fail('missing_signature_hash', 'operation.signature.hash is required'); + } + + if (typeof signature.hash !== 'string') { + return fail('invalid_signature_hash_type', 'operation.signature.hash must be a string'); + } + + const idHex = signature.hash.toLowerCase(); + if (!/^[a-f0-9]{64}$/.test(idHex)) { + return fail( + 'invalid_signature_hash_format', + `operation.signature.hash must be ${SYNC_ID_HEX_LEN} hex characters` + ); + } + + if (signature.signed == null || signature.signed === '') { + return fail('missing_signature_signed', 'operation.signature.signed is required'); + } + + if (typeof signature.signed !== 'string') { + return fail('invalid_signature_signed_type', 'operation.signature.signed must be a string'); + } + + const tsMs = Date.parse(signature.signed); + if (!Number.isFinite(tsMs)) { + return fail('invalid_signature_signed_value', 'operation.signature.signed must be parseable by Date.parse'); + } + + const idBytes = Buffer.from(idHex, 'hex'); + if (idBytes.length !== SYNC_ID_BYTES_LEN) { + return fail( + 'invalid_signature_hash_format', + `operation.signature.hash must decode to ${SYNC_ID_BYTES_LEN} bytes` + ); + } + + return { + ok: true, + value: { + idHex, + idBytes, + tsMs, + operation, + }, + }; +} + diff --git a/services/mediators/hyperswarm/src/sync-persistence.ts b/services/mediators/hyperswarm/src/sync-persistence.ts new file mode 100644 index 000000000..4375c60d6 --- /dev/null +++ b/services/mediators/hyperswarm/src/sync-persistence.ts @@ -0,0 +1,81 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { mapOperationToSyncKey } from './sync-mapping.js'; + +export interface SyncPersistenceRecord { + id: string; + ts: number; + operation: Operation; +} + +export interface MapAcceptedResult { + records: SyncPersistenceRecord[]; + invalid: number; +} + +export function filterIndexRejectedOperations(batch: Operation[], rejectedIndices: number[] = []): Operation[] { + if (!Array.isArray(batch) || batch.length === 0) { + return []; + } + + if (!Array.isArray(rejectedIndices) || rejectedIndices.length === 0) { + return [...batch]; + } + + const rejectedSet = new Set(); + for (const index of rejectedIndices) { + if (Number.isInteger(index) && index >= 0 && index < batch.length) { + rejectedSet.add(index); + } + } + + return batch.filter((_operation, index) => !rejectedSet.has(index)); +} + +export function filterOperationsByAcceptedHashes( + operations: Operation[], + acceptedHashes: string[] = [], +): Operation[] { + if (!Array.isArray(operations) || operations.length === 0) { + return []; + } + + if (!Array.isArray(acceptedHashes) || acceptedHashes.length === 0) { + return []; + } + + const acceptedSet = new Set( + acceptedHashes + .filter((hash): hash is string => hash !== '') + .map(hash => hash.toLowerCase()) + ); + + if (acceptedSet.size === 0) { + return []; + } + + return operations.filter(operation => { + const hash = operation.signature?.hash; + return typeof hash === 'string' && acceptedSet.has(hash.toLowerCase()); + }); +} + +export function mapAcceptedOperationsToSyncRecords(operations: Operation[]): MapAcceptedResult { + const records: SyncPersistenceRecord[] = []; + let invalid = 0; + + for (const operation of operations) { + const mapped = mapOperationToSyncKey(operation); + if (!mapped.ok) { + invalid += 1; + continue; + } + + records.push({ + id: mapped.value.idHex, + ts: mapped.value.tsMs, + operation, + }); + } + + return { records, invalid }; +} diff --git a/tests/gatekeeper/client.test.ts b/tests/gatekeeper/client.test.ts index 93a42d2b4..8719017a3 100644 --- a/tests/gatekeeper/client.test.ts +++ b/tests/gatekeeper/client.test.ts @@ -469,13 +469,19 @@ describe('importDIDs', () => { it('should return imported DID results', async () => { nock(GatekeeperURL) .post(Endpoints.dids.import) - .reply(200, { queued: 0, processed: 0 }); + .reply(200, { queued: 0, processed: 0, rejected: 0, total: 0, rejectedIndices: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); // @ts-expect-error Testing without arguments const results = await gatekeeper.importDIDs(); - expect(results).toStrictEqual({ queued: 0, processed: 0 }); + expect(results).toStrictEqual({ + queued: 0, + processed: 0, + rejected: 0, + total: 0, + rejectedIndices: [], + }); }); it('should throw exception on importDIDs server error', async () => { @@ -529,13 +535,19 @@ describe('importBatch', () => { it('should return imported batch results', async () => { nock(GatekeeperURL) .post(Endpoints.batch.import) - .reply(200, { queued: 0, processed: 0 }); + .reply(200, { queued: 0, processed: 0, rejected: 0, total: 0, rejectedIndices: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); // @ts-expect-error Testing without arguments const results = await gatekeeper.importBatch(); - expect(results).toStrictEqual({ queued: 0, processed: 0 }); + expect(results).toStrictEqual({ + queued: 0, + processed: 0, + rejected: 0, + total: 0, + rejectedIndices: [], + }); }); it('should throw exception on importBatch server error', async () => { @@ -628,12 +640,12 @@ describe('processEvents', () => { it('should return process status', async () => { nock(GatekeeperURL) .post(Endpoints.events.process) - .reply(200, { added: 0, merged: 0, pending: 0 }); + .reply(200, { added: 0, merged: 0, pending: 0, acceptedHashes: [] }); const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL }); const status = await gatekeeper.processEvents(); - expect(status).toStrictEqual({ added: 0, merged: 0, pending: 0 }); + expect(status).toStrictEqual({ added: 0, merged: 0, pending: 0, acceptedHashes: [] }); }); it('should throw exception on processEvents server error', async () => { diff --git a/tests/gatekeeper/sync.test.ts b/tests/gatekeeper/sync.test.ts index 44d50c64a..b63b92eb1 100644 --- a/tests/gatekeeper/sync.test.ts +++ b/tests/gatekeeper/sync.test.ts @@ -146,6 +146,25 @@ describe('importDIDs', () => { expect(response.merged).toBe(1); }); + + it('should report rejectedIndices using dids.flat() order', async () => { + const keypair = cipher.generateRandomJwk(); + const op1 = await helper.createAgentOp(keypair); + const did1 = await gatekeeper.createDID(op1); + const did1Events = await gatekeeper.exportDIDs([did1]); + + const op2 = await helper.createAgentOp(keypair); + const did2 = await gatekeeper.createDID(op2); + const did2Events = await gatekeeper.exportDIDs([did2]); + + delete did2Events[0][0].operation.signature; + + const response = await gatekeeper.importDIDs([did1Events[0], did2Events[0]]); + + expect(response.queued).toBe(1); + expect(response.rejected).toBe(1); + expect(response.rejectedIndices).toStrictEqual([1]); + }); }); describe('removeDIDs', () => { @@ -295,6 +314,7 @@ describe('importBatch', () => { const response = await gatekeeper.importBatch([1, 2, 3]); expect(response.rejected).toBe(3); + expect(response.rejectedIndices).toStrictEqual([0, 1, 2]); }); it('should report an error on invalid event time', async () => { @@ -498,6 +518,22 @@ describe('importBatch', () => { expect(response.queued).toBe(1); expect(response.rejected).toBe(1); + expect(response.rejectedIndices).toStrictEqual([1]); + }); + + it('should report sparse rejectedIndices in original submitted order', async () => { + const keypair = cipher.generateRandomJwk(); + const agentOp = await helper.createAgentOp(keypair); + const did = await gatekeeper.createDID(agentOp); + const ops = await gatekeeper.exportDID(did); + + const invalid1 = { ...ops[0], time: 'invalid' } as any; + const invalid2 = { ...ops[0], operation: { ...ops[0].operation, type: 'bad-type' } } as any; + const response = await gatekeeper.importBatch([invalid1, ops[0], invalid2]); + + expect(response.queued).toBe(1); + expect(response.rejected).toBe(2); + expect(response.rejectedIndices).toStrictEqual([0, 2]); }); it('should report an error on invalid update operation missing did', async () => { @@ -547,6 +583,7 @@ describe('processEvents', () => { const response = await gatekeeper.processEvents(); expect(response.merged).toBe(1); + expect(response.acceptedHashes).toContain(ops[0].operation.signature!.hash.toLowerCase()); }); it('should import a valid asset DID export', async () => { @@ -921,6 +958,7 @@ describe('processEvents', () => { const response = await gatekeeper.processEvents(); expect(response.added).toBe(1); expect(response.rejected).toBe(2); + expect(response.acceptedHashes).toStrictEqual([updateOp1.signature!.hash.toLowerCase()]); }); it('should handle a reorg event', async () => { diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts new file mode 100644 index 000000000..2ef48dd2e --- /dev/null +++ b/tests/hyperswarm/bootstrap.test.ts @@ -0,0 +1,81 @@ +import { jest } from '@jest/globals'; +import { GatekeeperEvent, Operation } from '@mdip/gatekeeper/types'; +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import { bootstrapSyncStoreIfEmpty } from '../../services/mediators/hyperswarm/src/bootstrap.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOperation(hashChar: string, signed: string): Operation { + return { + type: 'create', + signature: { + hash: h(hashChar), + signed, + value: `sig-${hashChar}`, + }, + }; +} + +function makeEvent(operation: Operation): GatekeeperEvent { + return { + registry: 'hyperswarm', + time: new Date().toISOString(), + operation, + }; +} + +describe('bootstrapSyncStoreIfEmpty', () => { + it('skips bootstrap when store is already populated', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + await store.upsertMany([{ + id: h('a'), + ts: Date.parse('2026-02-10T10:00:00.000Z'), + operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), + }]); + + const gatekeeper = { + exportBatch: jest.fn(async () => []), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(true); + expect(result.reason).toBe('store_not_empty'); + expect(gatekeeper.exportBatch).not.toHaveBeenCalled(); + expect(await store.count()).toBe(1); + }); + + it('bootstraps from gatekeeper exportBatch when store is empty', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); + const gatekeeper = { + exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(false); + expect(result.exported).toBe(2); + expect(result.mapped).toBe(2); + expect(result.invalid).toBe(0); + expect(result.inserted).toBe(2); + expect(result.countAfter).toBe(2); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(await store.count()).toBe(2); + }); + + it('throws when gatekeeper exportBatch fails', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + exportBatch: jest.fn(async () => { + throw new Error('boom'); + }), + }; + + await expect(bootstrapSyncStoreIfEmpty(store, gatekeeper)).rejects.toThrow('boom'); + }); +}); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts new file mode 100644 index 000000000..9e1d137f8 --- /dev/null +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -0,0 +1,273 @@ +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import NegentropyAdapter from '../../services/mediators/hyperswarm/src/negentropy/adapter.ts'; +import { Operation } from '@mdip/gatekeeper/types'; + +const DAY_MS = 24 * 60 * 60 * 1000; +const h = (c: string) => c.repeat(64); +const idFromNum = (n: number) => n.toString(16).padStart(64, '0'); + +function makeOp(hashChar: string, signed: string): Operation { + return { + type: 'create', + signature: { + signed, + hash: h(hashChar), + value: `sig-${hashChar}`, + }, + }; +} + +function makeOpFromHash(hash: string, signed: string): Operation { + return { + type: 'create', + signature: { + signed, + hash, + value: `sig-${hash.slice(0, 8)}`, + }, + }; +} + +async function seedStore( + store: InMemoryOperationSyncStore, + records: Array<{ id: string; ts: number; op: Operation }> +): Promise { + await store.start(); + await store.reset(); + await store.upsertMany(records.map(item => ({ + id: item.id, + ts: item.ts, + operation: item.op, + }))); +} + +async function seedNumericRange( + store: InMemoryOperationSyncStore, + start: number, + endExclusive: number, + baseTs: number, +): Promise { + const records = []; + for (let i = start; i < endExclusive; i++) { + const id = idFromNum(i); + records.push({ + id, + ts: baseTs + i, + op: makeOpFromHash(id, new Date(baseTs + i).toISOString()), + }); + } + await seedStore(store, records); +} + +describe('NegentropyAdapter', () => { + it('loads and builds from store', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + }); + + const stats = adapter.getStats(); + expect(stats.loaded).toBe(2); + expect(stats.skipped).toBe(0); + expect(stats.durationMs).toBeGreaterThanOrEqual(0); + }); + + it('reconciles two stores and reports have/need ids', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + await seedStore(storeA, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + ]); + + await seedStore(storeB, [ + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + { id: h('d'), ts: 4000, op: makeOp('d', '2026-02-09T13:00:00.000Z') }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + }); + + let msg: string | Uint8Array | null = await adapterA.initiate(); + const have = new Set(); + const need = new Set(); + + for (let i = 0; i < 20 && msg !== null; i++) { + const response = await adapterB.respond(msg); + if (response === null) { + break; + } + const round = await adapterA.reconcile(response); + + for (const id of round.haveIds) { + have.add(String(id)); + } + for (const id of round.needIds) { + need.add(String(id)); + } + + msg = round.nextMsg; + } + + expect(have.has(h('a'))).toBe(true); + expect(need.has(h('d'))).toBe(true); + }); + + it('throws when frameSizeLimit is invalid', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 1024, + })).rejects.toThrow('frameSizeLimit'); + }); + + it('plans recent window first then older windows in descending recency', async () => { + const store = new InMemoryOperationSyncStore(); + const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + await seedStore(store, [ + { id: h('a'), ts: nowMs - (10 * DAY_MS), op: makeOp('a', '2026-01-31T00:00:00.000Z') }, + { id: h('b'), ts: nowMs - (2 * DAY_MS), op: makeOp('b', '2026-02-08T00:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + recentWindowDays: 3, + olderWindowDays: 2, + deferInitialBuild: true, + }); + + const windows = await adapter.planWindows(nowMs); + expect(windows.length).toBeGreaterThanOrEqual(2); + expect(windows[0].name).toBe('recent'); + expect(windows[0].toTs).toBe(nowMs); + expect(windows[1].fromTs).toBeLessThanOrEqual(windows[1].toTs); + expect(windows[1].toTs).toBe(windows[0].fromTs - 1); + }); + + it('caps records per window and emits window stats', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + const stats = await adapter.rebuildForWindow({ + name: 'recent', + fromTs: 0, + toTs: 5000, + maxRecords: 2, + order: 0, + }); + + expect(stats.loaded).toBe(2); + expect(stats.skipped).toBe(0); + expect(stats.cappedByRecords).toBe(true); + expect(stats.windowName).toBe('recent'); + }); + + it('applies maxRoundsPerSession cap in windowed sessions', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + const baseTs = Date.parse('2026-02-01T00:00:00.000Z'); + const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + + await seedNumericRange(storeA, 0, 2000, baseTs); + await seedNumericRange(storeB, 1000, 3000, baseTs); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 4096, + recentWindowDays: 14, + olderWindowDays: 14, + maxRecordsPerWindow: 10000, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 4096, + recentWindowDays: 14, + olderWindowDays: 14, + maxRecordsPerWindow: 10000, + deferInitialBuild: true, + }); + + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { + nowMs, + maxRoundsPerSession: 1, + }); + + expect(session.windowCount).toBeGreaterThan(0); + expect(session.windows.some(window => window.cappedByRounds)).toBe(true); + expect(session.windows[0].rounds).toBe(1); + }); + + it('continues into older windows when a newer window hits record cap', async () => { + const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + const recentA = nowMs - (6 * 60 * 60 * 1000); + const recentB = nowMs - (8 * 60 * 60 * 1000); + const older = nowMs - (2 * DAY_MS); + + await seedStore(storeA, [ + { id: h('a'), ts: recentA, op: makeOp('a', new Date(recentA).toISOString()) }, + { id: h('b'), ts: recentB, op: makeOp('b', new Date(recentB).toISOString()) }, + { id: h('c'), ts: older, op: makeOp('c', new Date(older).toISOString()) }, + ]); + await seedStore(storeB, [ + { id: h('d'), ts: recentA, op: makeOp('d', new Date(recentA).toISOString()) }, + { id: h('e'), ts: older, op: makeOp('e', new Date(older).toISOString()) }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + recentWindowDays: 1, + olderWindowDays: 1, + maxRecordsPerWindow: 1, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + recentWindowDays: 1, + olderWindowDays: 1, + maxRecordsPerWindow: 1, + deferInitialBuild: true, + }); + + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { nowMs }); + const recent = session.windows.find(window => window.windowName === 'recent'); + const olderWindow = session.windows.find(window => window.windowName === 'older_1'); + + expect(session.windowCount).toBeGreaterThanOrEqual(2); + expect(recent?.cappedByRecords).toBe(true); + expect(olderWindow).toBeDefined(); + }); +}); diff --git a/tests/hyperswarm/negentropy-observability.test.ts b/tests/hyperswarm/negentropy-observability.test.ts new file mode 100644 index 000000000..165fc8db0 --- /dev/null +++ b/tests/hyperswarm/negentropy-observability.test.ts @@ -0,0 +1,61 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + addAggregateSample, + averageAggregate, + collectQueueDelaySamples, + createAggregateMetric, + messageBytes, + safeRate, +} from '../../services/mediators/hyperswarm/src/negentropy/observability.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(signed?: string): Operation { + return { + type: 'create', + signature: signed ? { + hash: h('a'), + signed, + value: 'sig', + } : undefined, + }; +} + +describe('negentropy observability helpers', () => { + it('aggregates samples and computes average', () => { + const metric = createAggregateMetric(); + addAggregateSample(metric, 10); + addAggregateSample(metric, 20); + addAggregateSample(metric, -5); + + expect(metric).toStrictEqual({ + count: 2, + total: 30, + max: 20, + }); + expect(averageAggregate(metric)).toBe(15); + }); + + it('collects queue delay samples from signed operations', () => { + const now = Date.parse('2026-02-13T10:00:00.000Z'); + const samples = collectQueueDelaySamples([ + makeOp('2026-02-13T09:59:00.000Z'), + makeOp('invalid-date'), + makeOp(), + ], now); + + expect(samples).toStrictEqual([60_000]); + }); + + it('computes message bytes for string, buffer, and object', () => { + expect(messageBytes('abc')).toBe(3); + expect(messageBytes(Buffer.from([1, 2, 3]))).toBe(3); + expect(messageBytes({ hello: 'world' })).toBeGreaterThan(0); + }); + + it('returns safe rate for invalid denominator', () => { + expect(safeRate(1, 0)).toBe(0); + expect(safeRate(1, -1)).toBe(0); + expect(safeRate(2, 4)).toBe(0.5); + }); +}); diff --git a/tests/hyperswarm/negentropy-policy.test.ts b/tests/hyperswarm/negentropy-policy.test.ts new file mode 100644 index 000000000..0c3b97006 --- /dev/null +++ b/tests/hyperswarm/negentropy-policy.test.ts @@ -0,0 +1,46 @@ +import { + shouldAcceptLegacySync, + shouldSchedulePeriodicRepair, + shouldStartConnectTimeNegentropy, +} from '../../services/mediators/hyperswarm/src/negentropy/policy.ts'; + +describe('negentropy sync policy', () => { + it('accepts legacy sync only when enabled and peer is not in negentropy mode', () => { + expect(shouldAcceptLegacySync('legacy', true)).toBe(true); + expect(shouldAcceptLegacySync('unknown', true)).toBe(true); + expect(shouldAcceptLegacySync('negentropy', true)).toBe(false); + expect(shouldAcceptLegacySync('legacy', false)).toBe(false); + }); + + it('starts connect-time negentropy only for initiator without active session', () => { + expect(shouldStartConnectTimeNegentropy('negentropy', false, true)).toBe(true); + expect(shouldStartConnectTimeNegentropy('negentropy', true, true)).toBe(false); + expect(shouldStartConnectTimeNegentropy('negentropy', false, false)).toBe(false); + expect(shouldStartConnectTimeNegentropy('legacy', false, true)).toBe(false); + }); + + it('schedules periodic repair only when all guards pass', () => { + const base = { + syncMode: 'negentropy' as const, + hasActiveSession: false, + importQueueLength: 0, + activeNegentropySessions: 0, + maxConcurrentNegentropySessions: 1, + lastRepairAtMs: 0, + nowMs: 1_000_000, + repairIntervalMs: 300_000, + isInitiator: true, + }; + + expect(shouldSchedulePeriodicRepair(base)).toBe(true); + expect(shouldSchedulePeriodicRepair({ ...base, isInitiator: false })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, hasActiveSession: true })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, importQueueLength: 1 })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, activeNegentropySessions: 1 })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, syncMode: 'legacy' })).toBe(false); + expect(shouldSchedulePeriodicRepair({ + ...base, + lastRepairAtMs: base.nowMs - 10_000, + })).toBe(false); + }); +}); diff --git a/tests/hyperswarm/negentropy-protocol.test.ts b/tests/hyperswarm/negentropy-protocol.test.ts new file mode 100644 index 000000000..eaa426cbf --- /dev/null +++ b/tests/hyperswarm/negentropy-protocol.test.ts @@ -0,0 +1,80 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + NEG_SYNC_ID_RE, + chooseSyncMode, + decodeNegentropyFrame, + encodeNegentropyFrame, + extractOperationHashes, + normalizeNegentropyIds, + normalizePeerCapabilities, +} from '../../services/mediators/hyperswarm/src/negentropy/protocol.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(hash: string): Operation { + return { + type: 'create', + signature: { + hash, + signed: '2026-02-11T00:00:00.000Z', + value: 'sig', + }, + }; +} + +describe('negentropy protocol helpers', () => { + it('encodes and decodes utf8 frame payloads', () => { + const payload = encodeNegentropyFrame('frame-message'); + expect(payload.encoding).toBe('utf8'); + expect(decodeNegentropyFrame(payload)).toBe('frame-message'); + }); + + it('encodes and decodes binary frame payloads', () => { + const bytes = Uint8Array.from([1, 2, 3, 250]); + const payload = encodeNegentropyFrame(bytes); + expect(payload.encoding).toBe('base64'); + const decoded = decodeNegentropyFrame(payload); + expect(Buffer.from(decoded as Uint8Array)).toStrictEqual(Buffer.from(bytes)); + }); + + it('normalizes peer capabilities and chooses sync mode', () => { + const unknown = normalizePeerCapabilities(); + expect(chooseSyncMode(unknown, 1)).toBeNull(); + + const legacy = normalizePeerCapabilities({ negentropy: false }); + expect(chooseSyncMode(legacy, 1)).toBe('legacy'); + + const negentropy = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 1 }); + expect(chooseSyncMode(negentropy, 1)).toBe('negentropy'); + + const oldVersion = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 0 }); + expect(chooseSyncMode(oldVersion, 1)).toBe('legacy'); + }); + + it('normalizes negentropy ids and filters invalid values', () => { + const a = h('a'); + const b = h('b'); + const ids = normalizeNegentropyIds([ + a.toUpperCase(), + Buffer.from(b, 'hex'), + 'not-a-sync-id', + ]); + + expect(ids).toStrictEqual([a, b]); + expect(ids.every(id => NEG_SYNC_ID_RE.test(id))).toBe(true); + }); + + it('extracts unique valid operation hashes', () => { + const a = h('a'); + const b = h('b'); + const operations: Operation[] = [ + makeOp(a.toUpperCase()), + makeOp(b), + makeOp('invalid'), + { type: 'create' }, + ]; + + const hashes = extractOperationHashes(operations); + expect(hashes).toStrictEqual([a, b]); + }); +}); diff --git a/tests/hyperswarm/negentropy-transfer.test.ts b/tests/hyperswarm/negentropy-transfer.test.ts new file mode 100644 index 000000000..7f289852a --- /dev/null +++ b/tests/hyperswarm/negentropy-transfer.test.ts @@ -0,0 +1,63 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + chunkIds, + chunkOperationsForPush, + estimateOperationBytes, +} from '../../services/mediators/hyperswarm/src/negentropy/transfer.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(hashChar: string, sizeTag: string = ''): Operation { + return { + type: 'create', + data: { note: `${sizeTag}${'x'.repeat(sizeTag.length * 20)}` }, + signature: { + hash: h(hashChar), + signed: '2026-02-12T00:00:00.000Z', + value: `sig-${hashChar}`, + }, + }; +} + +describe('negentropy transfer batching helpers', () => { + it('chunks id lists with de-duplication', () => { + const ids = ['a', 'b', 'c', 'c', 'd', 'e']; + const chunks = chunkIds(ids, 2); + expect(chunks).toStrictEqual([ + ['a', 'b'], + ['c', 'd'], + ['e'], + ]); + }); + + it('splits operations by count and bytes', () => { + const ops = [ + makeOp('a', 'small'), + makeOp('b', 'small'), + makeOp('c', 'this-is-a-larger-payload'), + makeOp('d', 'small'), + ]; + const bytesA = estimateOperationBytes(ops[0]); + const bytesB = estimateOperationBytes(ops[1]); + const bytesC = estimateOperationBytes(ops[2]); + + const batches = chunkOperationsForPush(ops, { + maxOpsPerPush: 2, + maxBytesPerPush: bytesA + bytesB + Math.floor(bytesC / 2), + }); + + expect(batches.length).toBeGreaterThanOrEqual(2); + expect(batches[0].length).toBeLessThanOrEqual(2); + expect(batches.flat().length).toBe(ops.length); + }); + + it('keeps a single oversized op in its own batch', () => { + const op = makeOp('a', 'x'.repeat(200)); + const batches = chunkOperationsForPush([op], { + maxOpsPerPush: 10, + maxBytesPerPush: 32, + }); + + expect(batches).toStrictEqual([[op]]); + }); +}); diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts new file mode 100644 index 000000000..f71b5576d --- /dev/null +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -0,0 +1,58 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + filterOperationsByAcceptedHashes, + filterIndexRejectedOperations, + mapAcceptedOperationsToSyncRecords, +} from '../../services/mediators/hyperswarm/src/sync-persistence.ts'; + +const h = (c: string) => c.repeat(64); + +function makeCreateOp(hashChar: string, signed: string): Operation { + return { + type: 'create', + created: signed, + mdip: { + version: 1, + type: 'agent', + registry: 'hyperswarm', + }, + signature: { + signed, + hash: h(hashChar), + value: `sig-${hashChar}`, + }, + }; +} + +describe('sync-persistence helpers', () => { + it('filters rejected indices in original submitted order', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); + + const filtered = filterIndexRejectedOperations([a, b, c], [1, 0, 999, -1, 1]); + + expect(filtered).toStrictEqual([c]); + }); + + it('maps accepted operations to sync records and counts invalid operations', () => { + const valid = makeCreateOp('A', '2026-02-10T10:00:00.000Z'); + const invalid = makeCreateOp('b', 'not-a-date'); + + const result = mapAcceptedOperationsToSyncRecords([valid, invalid]); + + expect(result.records.length).toBe(1); + expect(result.records[0].id).toBe(h('a')); + expect(result.records[0].ts).toBe(Date.parse(valid.signature!.signed)); + expect(result.invalid).toBe(1); + }); + + it('filters operations by accepted hashes', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); + + const accepted = filterOperationsByAcceptedHashes([a, b, c], [h('a').toUpperCase(), h('c')]); + expect(accepted).toStrictEqual([a, c]); + }); +}); diff --git a/tests/hyperswarm/sync-store.test.ts b/tests/hyperswarm/sync-store.test.ts new file mode 100644 index 000000000..9d5c19f38 --- /dev/null +++ b/tests/hyperswarm/sync-store.test.ts @@ -0,0 +1,112 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { Operation } from '@mdip/gatekeeper/types'; +import SqliteOperationSyncStore from '../../services/mediators/hyperswarm/src/db/sqlite.ts'; +import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; +import { OperationSyncStore } from '../../services/mediators/hyperswarm/src/db/types.ts'; + +const h = (c: string) => c.repeat(64); + +const opA: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T10:00:00.000Z', + hash: h('a'), + value: 'sig-a', + }, +}; + +const opB: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T11:00:00.000Z', + hash: h('b'), + value: 'sig-b', + }, +}; + +const opC: Operation = { + type: 'create', + signature: { + signed: '2026-02-09T11:30:00.000Z', + hash: h('c'), + value: 'sig-c', + }, +}; + +const recA = { id: h('a'), ts: 1000, operation: opA }; +const recB = { id: h('b'), ts: 2000, operation: opB }; +const recC = { id: h('c'), ts: 2000, operation: opC }; + +async function runStoreContractTests(store: OperationSyncStore): Promise { + await store.start(); + await store.reset(); + + const inserted1 = await store.upsertMany([recB, recA, recC]); + expect(inserted1).toBe(3); + + const inserted2 = await store.upsertMany([recA, recC]); + expect(inserted2).toBe(0); + + expect(await store.count()).toBe(3); + expect(await store.has(recA.id)).toBe(true); + expect(await store.has(h('d'))).toBe(false); + + const byIds = await store.getByIds([recC.id, recA.id, h('d')]); + expect(byIds.map(item => item.id)).toStrictEqual([recC.id, recA.id]); + + const sortedAll = await store.iterateSorted(); + expect(sortedAll.map(item => item.id)).toStrictEqual([recA.id, recB.id, recC.id]); + expect(sortedAll.every(item => Number.isFinite(item.insertedAt))).toBe(true); + + const firstTwo = await store.iterateSorted({ limit: 2 }); + expect(firstTwo.map(item => item.id)).toStrictEqual([recA.id, recB.id]); + + const rangeTs = await store.iterateSorted({ fromTs: 1500, toTs: 2000 }); + expect(rangeTs.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + const afterA = await store.iterateSorted({ after: { ts: 1000, id: recA.id } }); + expect(afterA.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + const afterB = await store.iterateSorted({ after: { ts: 2000, id: recB.id } }); + expect(afterB.map(item => item.id)).toStrictEqual([recC.id]); + + const afterRange = await store.iterateSorted({ + after: { ts: 1500, id: h('0') }, + fromTs: 1500, + toTs: 2000, + }); + expect(afterRange.map(item => item.id)).toStrictEqual([recB.id, recC.id]); + + await store.reset(); + expect(await store.count()).toBe(0); + + await store.stop(); +} + +describe('InMemoryOperationSyncStore', () => { + it('implements sync-store contract', async () => { + const store = new InMemoryOperationSyncStore(); + await runStoreContractTests(store); + }); +}); + +describe('SqliteOperationSyncStore', () => { + let tmpRoot = ''; + + beforeEach(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hypr-sync-store-')); + }); + + afterEach(async () => { + if (tmpRoot) { + await fs.rm(tmpRoot, { recursive: true, force: true }); + } + }); + + it('implements sync-store contract', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await runStoreContractTests(store); + }); +}); From 1a2f8face70f5597e99537bd7b2bee7da7fbe266 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 11:55:22 +0000 Subject: [PATCH 02/31] enable fallback to legacy on no capabilities --- .../hyperswarm/src/hyperswarm-mediator.ts | 61 ++++++++++++++++--- .../hyperswarm/src/negentropy/protocol.ts | 36 +++++++++++ tests/hyperswarm/negentropy-protocol.test.ts | 31 ++++++++++ 3 files changed, 119 insertions(+), 9 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index e2f6f6b2e..3755b33ec 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -17,12 +17,13 @@ import SqliteOperationSyncStore from './db/sqlite.js'; import NegentropyAdapter from './negentropy/adapter.js'; import { NEG_SYNC_ID_RE, - chooseSyncMode as chooseNegotiatedSyncMode, + chooseConnectSyncMode, decodeNegentropyFrame, encodeNegentropyFrame, extractOperationHashes, normalizeNegentropyIds, normalizePeerCapabilities, + type ConnectSyncModeReason, type NegentropyFrame, type NegotiatedPeerCapabilities, type PeerCapabilities, @@ -173,6 +174,10 @@ interface MediatorSyncStats { modeSelectionsTotal: number; modeSelectionsLegacy: number; modeSelectionsNegentropy: number; + modeSelectionsLegacyMissingCapabilities: number; + modeSelectionsLegacyNegentropyDisabled: number; + modeSelectionsLegacyVersionMismatch: number; + modeSelectionsNoModeLegacyDisabled: number; queueOpsRelayed: number; queueOpsImported: number; queueDelayMs: AggregateMetric; @@ -228,6 +233,10 @@ const syncStats: MediatorSyncStats = { modeSelectionsTotal: 0, modeSelectionsLegacy: 0, modeSelectionsNegentropy: 0, + modeSelectionsLegacyMissingCapabilities: 0, + modeSelectionsLegacyNegentropyDisabled: 0, + modeSelectionsLegacyVersionMismatch: 0, + modeSelectionsNoModeLegacyDisabled: 0, queueOpsRelayed: 0, queueOpsImported: 0, queueDelayMs: createAggregateMetric(), @@ -470,13 +479,17 @@ function expireIdlePeerSessions(): void { } } -function choosePeerSyncMode(peerKey: string): SyncMode | null { +function choosePeerSyncMode(peerKey: string): { mode: SyncMode | null; reason: ConnectSyncModeReason } | null { const conn = connectionInfo[peerKey]; if (!conn) { return null; } - return chooseNegotiatedSyncMode(conn.capabilities, NEGENTROPY_VERSION); + return chooseConnectSyncMode( + conn.capabilities, + NEGENTROPY_VERSION, + config.legacySyncEnabled, + ); } function getActiveNegentropySessions(): number { @@ -499,11 +512,24 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic return; } - const mode = source === 'connect' - ? choosePeerSyncMode(peerKey) - : conn.syncMode; + let mode: SyncMode | 'unknown' | null; + let modeReason: ConnectSyncModeReason | null = null; + + if (source === 'connect') { + const decision = choosePeerSyncMode(peerKey); + if (!decision) { + return; + } + mode = decision.mode; + modeReason = decision.reason; + } else { + mode = conn.syncMode; + } - if (!mode) { + if (!mode || mode === 'unknown') { + if (source === 'connect' && modeReason === 'legacy_disabled') { + syncStats.modeSelectionsNoModeLegacyDisabled += 1; + } return; } @@ -516,6 +542,15 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic syncStats.modeSelectionsTotal += 1; if (mode === 'legacy') { syncStats.modeSelectionsLegacy += 1; + if (modeReason === 'missing_capabilities') { + syncStats.modeSelectionsLegacyMissingCapabilities += 1; + } + if (modeReason === 'negentropy_disabled') { + syncStats.modeSelectionsLegacyNegentropyDisabled += 1; + } + if (modeReason === 'version_mismatch') { + syncStats.modeSelectionsLegacyVersionMismatch += 1; + } } else { syncStats.modeSelectionsNegentropy += 1; } @@ -532,7 +567,7 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic createPeerSession(peerKey, 'legacy', true, `legacy-${Date.now().toString(36)}`); syncQueue.push(conn.connection); - log.info({ peer: shortName(peerKey), mode }, 'peer sync mode selected'); + log.info({ peer: shortName(peerKey), mode, modeReason }, 'peer sync mode selected'); return; } @@ -571,7 +606,7 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic const session = createPeerSession(peerKey, 'negentropy', initiator); log.info( - { peer: shortName(peerKey), mode, initiator, sessionId: session.sessionId, source }, + { peer: shortName(peerKey), mode, modeReason, initiator, sessionId: session.sessionId, source }, 'peer sync mode selected' ); await startNegentropyOpen(peerKey, session); @@ -595,6 +630,14 @@ function buildSyncStatsSnapshot(): object { negentropy: syncStats.modeSelectionsNegentropy, fallbackCount: syncStats.modeSelectionsLegacy, fallbackRate: safeRate(syncStats.modeSelectionsLegacy, syncStats.modeSelectionsTotal), + legacyReasons: { + missingCapabilities: syncStats.modeSelectionsLegacyMissingCapabilities, + negentropyDisabled: syncStats.modeSelectionsLegacyNegentropyDisabled, + versionMismatch: syncStats.modeSelectionsLegacyVersionMismatch, + }, + noMode: { + legacyDisabled: syncStats.modeSelectionsNoModeLegacyDisabled, + }, }, queue: { relayed: syncStats.queueOpsRelayed, diff --git a/services/mediators/hyperswarm/src/negentropy/protocol.ts b/services/mediators/hyperswarm/src/negentropy/protocol.ts index 204eaf679..3ecbfcdf5 100644 --- a/services/mediators/hyperswarm/src/negentropy/protocol.ts +++ b/services/mediators/hyperswarm/src/negentropy/protocol.ts @@ -15,6 +15,18 @@ export interface NegotiatedPeerCapabilities { version: number | null; } +export type ConnectSyncModeReason = + | 'negentropy_supported' + | 'missing_capabilities' + | 'negentropy_disabled' + | 'version_mismatch' + | 'legacy_disabled'; + +export interface ConnectSyncModeDecision { + mode: SyncMode | null; + reason: ConnectSyncModeReason; +} + export interface NegentropyFrame { encoding: NegentropyFrameEncoding; data: string; @@ -60,6 +72,30 @@ export function chooseSyncMode( : 'legacy'; } +export function chooseConnectSyncMode( + capabilities: NegotiatedPeerCapabilities, + minVersion: number, + legacySyncEnabled: boolean, +): ConnectSyncModeDecision { + if (supportsPeerNegentropy(capabilities, minVersion)) { + return { mode: 'negentropy', reason: 'negentropy_supported' }; + } + + if (!legacySyncEnabled) { + return { mode: null, reason: 'legacy_disabled' }; + } + + if (!capabilities.advertised) { + return { mode: 'legacy', reason: 'missing_capabilities' }; + } + + if (!capabilities.negentropy) { + return { mode: 'legacy', reason: 'negentropy_disabled' }; + } + + return { mode: 'legacy', reason: 'version_mismatch' }; +} + export function encodeNegentropyFrame(frame: string | Uint8Array): NegentropyFrame { if (typeof frame === 'string') { return { diff --git a/tests/hyperswarm/negentropy-protocol.test.ts b/tests/hyperswarm/negentropy-protocol.test.ts index eaa426cbf..85e355f42 100644 --- a/tests/hyperswarm/negentropy-protocol.test.ts +++ b/tests/hyperswarm/negentropy-protocol.test.ts @@ -1,6 +1,7 @@ import { Operation } from '@mdip/gatekeeper/types'; import { NEG_SYNC_ID_RE, + chooseConnectSyncMode, chooseSyncMode, decodeNegentropyFrame, encodeNegentropyFrame, @@ -51,6 +52,36 @@ describe('negentropy protocol helpers', () => { expect(chooseSyncMode(oldVersion, 1)).toBe('legacy'); }); + it('chooses connect sync mode with explicit fallback reasons', () => { + const missing = normalizePeerCapabilities(); + expect(chooseConnectSyncMode(missing, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'missing_capabilities', + }); + expect(chooseConnectSyncMode(missing, 1, false)).toStrictEqual({ + mode: null, + reason: 'legacy_disabled', + }); + + const disabled = normalizePeerCapabilities({ negentropy: false, negentropyVersion: 1 }); + expect(chooseConnectSyncMode(disabled, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'negentropy_disabled', + }); + + const oldVersion = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 0 }); + expect(chooseConnectSyncMode(oldVersion, 1, true)).toStrictEqual({ + mode: 'legacy', + reason: 'version_mismatch', + }); + + const supported = normalizePeerCapabilities({ negentropy: true, negentropyVersion: 1 }); + expect(chooseConnectSyncMode(supported, 1, true)).toStrictEqual({ + mode: 'negentropy', + reason: 'negentropy_supported', + }); + }); + it('normalizes negentropy ids and filters invalid values', () => { const a = h('a'); const b = h('b'); From d2a6492c73f7d377e050e149130cc690135f918f Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 12:05:30 +0000 Subject: [PATCH 03/31] remove sync concurrency options --- services/mediators/hyperswarm/README.md | 3 ++- services/mediators/hyperswarm/src/config.js | 1 - services/mediators/hyperswarm/src/hyperswarm-mediator.ts | 7 +------ services/mediators/hyperswarm/src/negentropy/policy.ts | 3 +-- tests/hyperswarm/negentropy-policy.test.ts | 1 - 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/services/mediators/hyperswarm/README.md b/services/mediators/hyperswarm/README.md index 106787269..33e230e3a 100644 --- a/services/mediators/hyperswarm/README.md +++ b/services/mediators/hyperswarm/README.md @@ -52,10 +52,11 @@ The mediator emits periodic structured sync metrics in `connectionLoop` includin | `KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW` | 25000 | Maximum operations loaded into a single window adapter | | `KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION` | 64 | Maximum negentropy rounds per window session | | `KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS` | 300 | Seconds between periodic negentropy repair attempts per peer | -| `KC_HYPR_NEGENTROPY_MAX_CONCURRENT_SESSIONS` | 1 | Maximum concurrent negentropy sessions on this node | | `KC_HYPR_LEGACY_SYNC_ENABLE` | true | Allow legacy `sync`/`shareDb` compatibility path | | `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | +Negentropy session concurrency is currently fixed at one active session per node. + ## IPFS disabled mode Set `KC_IPFS_ENABLE=false` to run the mediator without IPFS or Keymaster integration. In this mode: diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index 3d4876e88..140c0bb04 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -62,7 +62,6 @@ const config = { negentropyMaxRecordsPerWindow: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW', 25000), negentropyMaxRoundsPerSession: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION', 64), negentropyRepairIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS', 300), - negentropyMaxConcurrentSessions: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_CONCURRENT_SESSIONS', 1), legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), }; diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 3755b33ec..f14dee28b 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -582,7 +582,6 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic hasActiveSession, importQueueLength: importQueue.length(), activeNegentropySessions, - maxConcurrentNegentropySessions: config.negentropyMaxConcurrentSessions, lastRepairAtMs: conn.lastNegentropyRepairAt, nowMs: Date.now(), repairIntervalMs: NEG_REPAIR_INTERVAL_MS, @@ -596,11 +595,7 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic return; } - if (activeNegentropySessions >= config.negentropyMaxConcurrentSessions) { - log.debug( - { peer: shortName(peerKey), activeNegentropySessions, max: config.negentropyMaxConcurrentSessions, source }, - 'negentropy session deferred by concurrency limit' - ); + if (activeNegentropySessions > 0) { return; } diff --git a/services/mediators/hyperswarm/src/negentropy/policy.ts b/services/mediators/hyperswarm/src/negentropy/policy.ts index cd4203b10..1c5ac0dda 100644 --- a/services/mediators/hyperswarm/src/negentropy/policy.ts +++ b/services/mediators/hyperswarm/src/negentropy/policy.ts @@ -5,7 +5,6 @@ export interface RepairSchedulingInput { hasActiveSession: boolean; importQueueLength: number; activeNegentropySessions: number; - maxConcurrentNegentropySessions: number; lastRepairAtMs: number; nowMs: number; repairIntervalMs: number; @@ -48,7 +47,7 @@ export function shouldSchedulePeriodicRepair(input: RepairSchedulingInput): bool return false; } - if (input.activeNegentropySessions >= input.maxConcurrentNegentropySessions) { + if (input.activeNegentropySessions > 0) { return false; } diff --git a/tests/hyperswarm/negentropy-policy.test.ts b/tests/hyperswarm/negentropy-policy.test.ts index 0c3b97006..2dd68208a 100644 --- a/tests/hyperswarm/negentropy-policy.test.ts +++ b/tests/hyperswarm/negentropy-policy.test.ts @@ -25,7 +25,6 @@ describe('negentropy sync policy', () => { hasActiveSession: false, importQueueLength: 0, activeNegentropySessions: 0, - maxConcurrentNegentropySessions: 1, lastRepairAtMs: 0, nowMs: 1_000_000, repairIntervalMs: 300_000, From 43934ae48fd5aab0c3a315ead55dcfe1e9f0a842 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 12:15:30 +0000 Subject: [PATCH 04/31] only rebuild on change --- .../hyperswarm/src/hyperswarm-mediator.ts | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index f14dee28b..7e7306b41 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -210,6 +210,10 @@ const ipfs = new KuboClient(); const cipher = new CipherNode(); let syncStore: OperationSyncStore = new SqliteOperationSyncStore(); let negentropyAdapter: NegentropyAdapter | null = null; +let adapterChangeSeq = 0; +let adapterBuiltSeq = -1; +let adapterBuiltAt = 0; +let adapterRebuildInFlight: Promise | null = null; EventEmitter.defaultMaxListeners = 100; @@ -222,6 +226,7 @@ const NEG_MAX_IDS_PER_LOOKUP = 1_000; const NEG_MAX_OPS_PER_PUSH = 256; const NEG_MAX_BYTES_PER_PUSH = 512 * 1024; const NEG_REPAIR_INTERVAL_MS = config.negentropyRepairIntervalSeconds * 1000; +const NEG_ADAPTER_MAX_AGE_MS = 60 * 1000; const connectionInfo: Record = {}; const knownNodes: Record = {}; @@ -672,12 +677,68 @@ function buildSyncStatsSnapshot(): object { }; } +function isNegentropyAdapterDirty(): boolean { + return adapterBuiltSeq < adapterChangeSeq; +} + +function markNegentropyAdapterDirty(): void { + adapterChangeSeq += 1; +} + +async function ensureAdapterFresh(reason: string): Promise { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + const now = Date.now(); + const recentlyBuilt = adapterBuiltAt > 0 && (now - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; + + if (!isNegentropyAdapterDirty() && recentlyBuilt) { + return; + } + + if (adapterRebuildInFlight) { + await adapterRebuildInFlight; + const recentAfterWait = adapterBuiltAt > 0 && (Date.now() - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; + if (!isNegentropyAdapterDirty() && recentAfterWait) { + return; + } + } + + const rebuildStartSeq = adapterChangeSeq; + const rebuildStartedAt = Date.now(); + const rebuildPromise = (async () => { + await negentropyAdapter!.rebuildFromStore(); + adapterBuiltSeq = rebuildStartSeq; + adapterBuiltAt = Date.now(); + log.debug( + { + reason, + durationMs: adapterBuiltAt - rebuildStartedAt, + adapterBuiltAt, + dirtyAfterRebuild: isNegentropyAdapterDirty(), + }, + 'negentropy adapter rebuilt from sync-store' + ); + })(); + + adapterRebuildInFlight = rebuildPromise; + try { + await rebuildPromise; + } + finally { + if (adapterRebuildInFlight === rebuildPromise) { + adapterRebuildInFlight = null; + } + } +} + async function startNegentropyOpen(peerKey: string, session: PeerSyncSession): Promise { if (!negentropyAdapter) { throw new Error('negentropy adapter unavailable'); } - await negentropyAdapter.rebuildFromStore(); + await ensureAdapterFresh('session_open_initiator'); const firstFrame = await negentropyAdapter.initiate(); const msg: NegOpenMessage = { ...createBaseMessage('neg_open'), @@ -1064,6 +1125,9 @@ async function persistAcceptedOperations(operations: Operation[], source: string } const inserted = await syncStore.upsertMany(records); + if (inserted > 0) { + markNegentropyAdapterDirty(); + } log.debug( { source, attempted: operations.length, mapped: records.length, invalid, inserted }, 'sync-store persist accepted ops' @@ -1309,7 +1373,7 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise closePeerSession(peerKey, 'replaced_by_remote_open'); } - await negentropyAdapter.rebuildFromStore(); + await ensureAdapterFresh('session_open_responder'); const session = createPeerSession(peerKey, 'negentropy', false, msg.sessionId); touchPeerSession(peerKey); await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); @@ -1596,6 +1660,10 @@ async function initNegentropyAdapter(): Promise { maxRoundsPerSession: config.negentropyMaxRoundsPerSession, deferInitialBuild: true, }); + adapterChangeSeq = 0; + adapterBuiltSeq = -1; + adapterBuiltAt = 0; + adapterRebuildInFlight = null; log.info( { stats: negentropyAdapter.getStats(), From e761f4e93a5bbce4fecd2fbf17fba0c2fc33e768 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 12:26:34 +0000 Subject: [PATCH 05/31] rebuild in background --- .../hyperswarm/src/hyperswarm-mediator.ts | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 7e7306b41..b34260184 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -213,7 +213,8 @@ let negentropyAdapter: NegentropyAdapter | null = null; let adapterChangeSeq = 0; let adapterBuiltSeq = -1; let adapterBuiltAt = 0; -let adapterRebuildInFlight: Promise | null = null; +let rebuildPromise: Promise | null = null; +let backgroundPrebuildQueued = false; EventEmitter.defaultMaxListeners = 100; @@ -464,6 +465,8 @@ function closePeerSession(peerKey: string, reason: string): void { } } + maybeStartBackgroundPrebuild('session_closed'); + log.debug({ peer: shortName(peerKey), mode: session.mode, @@ -685,6 +688,41 @@ function markNegentropyAdapterDirty(): void { adapterChangeSeq += 1; } +function maybeStartBackgroundPrebuild(reason: string): void { + if (!negentropyAdapter) { + return; + } + + if (!isNegentropyAdapterDirty()) { + return; + } + + if (getActiveNegentropySessions() > 0) { + return; + } + + if (rebuildPromise) { + backgroundPrebuildQueued = true; + return; + } + + backgroundPrebuildQueued = false; + void ensureAdapterFresh(`background_${reason}`) + .catch(error => { + log.error({ error, reason }, 'background negentropy prebuild failed'); + }) + .finally(() => { + if (!backgroundPrebuildQueued) { + return; + } + + backgroundPrebuildQueued = false; + if (isNegentropyAdapterDirty() && getActiveNegentropySessions() === 0) { + maybeStartBackgroundPrebuild('queued_followup'); + } + }); +} + async function ensureAdapterFresh(reason: string): Promise { if (!negentropyAdapter) { throw new Error('negentropy adapter unavailable'); @@ -697,8 +735,8 @@ async function ensureAdapterFresh(reason: string): Promise { return; } - if (adapterRebuildInFlight) { - await adapterRebuildInFlight; + if (rebuildPromise) { + await rebuildPromise; const recentAfterWait = adapterBuiltAt > 0 && (Date.now() - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; if (!isNegentropyAdapterDirty() && recentAfterWait) { return; @@ -707,7 +745,7 @@ async function ensureAdapterFresh(reason: string): Promise { const rebuildStartSeq = adapterChangeSeq; const rebuildStartedAt = Date.now(); - const rebuildPromise = (async () => { + const currentRebuildPromise = (async () => { await negentropyAdapter!.rebuildFromStore(); adapterBuiltSeq = rebuildStartSeq; adapterBuiltAt = Date.now(); @@ -722,13 +760,13 @@ async function ensureAdapterFresh(reason: string): Promise { ); })(); - adapterRebuildInFlight = rebuildPromise; + rebuildPromise = currentRebuildPromise; try { - await rebuildPromise; + await currentRebuildPromise; } finally { - if (adapterRebuildInFlight === rebuildPromise) { - adapterRebuildInFlight = null; + if (rebuildPromise === currentRebuildPromise) { + rebuildPromise = null; } } } @@ -1127,6 +1165,7 @@ async function persistAcceptedOperations(operations: Operation[], source: string const inserted = await syncStore.upsertMany(records); if (inserted > 0) { markNegentropyAdapterDirty(); + maybeStartBackgroundPrebuild(`persist_${source}`); } log.debug( { source, attempted: operations.length, mapped: records.length, invalid, inserted }, @@ -1663,7 +1702,8 @@ async function initNegentropyAdapter(): Promise { adapterChangeSeq = 0; adapterBuiltSeq = -1; adapterBuiltAt = 0; - adapterRebuildInFlight = null; + rebuildPromise = null; + backgroundPrebuildQueued = false; log.info( { stats: negentropyAdapter.getStats(), From 777a329f8f3486c93c0aed42996a7d2036b65adc Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 13:45:29 +0000 Subject: [PATCH 06/31] add windowed sync --- .../hyperswarm/src/hyperswarm-mediator.ts | 384 +++++++++++++++++- 1 file changed, 363 insertions(+), 21 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index b34260184..390b5d85a 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -14,7 +14,10 @@ import { childLogger } from '@mdip/common/logger'; import config from './config.js'; import type { OperationSyncStore } from './db/types.js'; import SqliteOperationSyncStore from './db/sqlite.js'; -import NegentropyAdapter from './negentropy/adapter.js'; +import NegentropyAdapter, { + type NegentropyWindowStats, + type ReconciliationWindow, +} from './negentropy/adapter.js'; import { NEG_SYNC_ID_RE, chooseConnectSyncMode, @@ -84,6 +87,15 @@ interface SyncMessage extends HyperMessageBase { interface NegOpenMessage extends HyperMessageBase { type: 'neg_open'; sessionId: string; + windowId: string; + window: { + name: string; + fromTs: number; + toTs: number; + maxRecords: number; + order: number; + }; + fullRepair: boolean; round: number; frame: NegentropyFrame; } @@ -91,6 +103,7 @@ interface NegOpenMessage extends HyperMessageBase { interface NegMsgMessage extends HyperMessageBase { type: 'neg_msg'; sessionId: string; + windowId: string; round: number; frame: NegentropyFrame; } @@ -98,6 +111,7 @@ interface NegMsgMessage extends HyperMessageBase { interface NegCloseMessage extends HyperMessageBase { type: 'neg_close'; sessionId: string; + windowId: string; round: number; reason?: string; } @@ -105,6 +119,7 @@ interface NegCloseMessage extends HyperMessageBase { interface OpsReqMessage extends HyperMessageBase { type: 'ops_req'; sessionId: string; + windowId: string; round: number; ids: string[]; } @@ -112,6 +127,7 @@ interface OpsReqMessage extends HyperMessageBase { interface OpsPushMessage extends HyperMessageBase { type: 'ops_push'; sessionId: string; + windowId: string; round: number; data: Operation[]; } @@ -160,6 +176,12 @@ interface PeerSyncSession { peerKey: string; mode: SyncMode; initiator: boolean; + fullRepair: boolean; + windows: ReconciliationWindow[]; + windowIndex: number; + windowId: string | null; + currentWindowStats: NegentropyWindowStats | null; + completedWindows: NegentropyWindowStats[]; startedAt: number; lastActivity: number; pendingHaveIds: string[]; @@ -213,6 +235,8 @@ let negentropyAdapter: NegentropyAdapter | null = null; let adapterChangeSeq = 0; let adapterBuiltSeq = -1; let adapterBuiltAt = 0; +let adapterBuiltWindowId: string | null = null; +let adapterBuiltWindowStats: NegentropyWindowStats | null = null; let rebuildPromise: Promise | null = null; let backgroundPrebuildQueued = false; @@ -419,6 +443,12 @@ function createPeerSession(peerKey: string, mode: SyncMode, initiator: boolean, peerKey, mode, initiator, + fullRepair: false, + windows: [], + windowIndex: 0, + windowId: null, + currentWindowStats: null, + completedWindows: [], startedAt: now, lastActivity: now, pendingHaveIds: [], @@ -608,11 +638,24 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic } const session = createPeerSession(peerKey, 'negentropy', initiator); + session.fullRepair = source === 'periodic'; + session.windows = await planRuntimeWindows(); + session.windowIndex = 0; + session.completedWindows = []; log.info( - { peer: shortName(peerKey), mode, modeReason, initiator, sessionId: session.sessionId, source }, + { + peer: shortName(peerKey), + mode, + modeReason, + initiator, + sessionId: session.sessionId, + source, + fullRepair: session.fullRepair, + plannedWindows: session.windows.length, + }, 'peer sync mode selected' ); - await startNegentropyOpen(peerKey, session); + await startNextNegentropyWindow(peerKey, session); } async function runPeriodicNegentropyRepair(): Promise { @@ -688,6 +731,136 @@ function markNegentropyAdapterDirty(): void { adapterChangeSeq += 1; } +function cloneWindowStats(stats: NegentropyWindowStats | null): NegentropyWindowStats | null { + return stats + ? { + ...stats, + } + : null; +} + +function buildFullHistoryWindow(): ReconciliationWindow { + return { + name: 'full_history', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: Number.MAX_SAFE_INTEGER, + order: 0, + }; +} + +function makeWindowId(window: ReconciliationWindow): string { + return `${window.order}:${window.name}:${window.fromTs}:${window.toTs}:${window.maxRecords}`; +} + +function windowLabel(window: ReconciliationWindow): string { + return `${window.name}[${window.fromTs},${window.toTs}]`; +} + +function getSessionWindow(session: PeerSyncSession): ReconciliationWindow | null { + if (session.windowIndex < 0 || session.windowIndex >= session.windows.length) { + return null; + } + return session.windows[session.windowIndex]; +} + +function parseRemoteWindow(raw: NegOpenMessage['window']): ReconciliationWindow | null { + if (!raw || typeof raw !== 'object') { + return null; + } + + const fromTs = Number(raw.fromTs); + const toTs = Number(raw.toTs); + const order = Number(raw.order); + const maxRecords = Number.isInteger(Number(raw.maxRecords)) && Number(raw.maxRecords) > 0 + ? Number(raw.maxRecords) + : config.negentropyMaxRecordsPerWindow; + + if (!Number.isFinite(fromTs) || !Number.isFinite(toTs) || fromTs > toTs) { + return null; + } + + if (!Number.isInteger(order) || order < 0) { + return null; + } + + return { + name: String(raw.name || `window_${order}`), + fromTs, + toTs, + order, + maxRecords, + }; +} + +function initializeSessionWindowState( + session: PeerSyncSession, + window: ReconciliationWindow, + windowId: string, + windowStats: NegentropyWindowStats, +): void { + session.windowId = windowId; + session.pendingHaveIds = []; + session.pendingNeedIds = []; + session.reconciliationComplete = false; + session.currentWindowStats = { + ...windowStats, + windowName: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + rounds: 0, + completed: false, + cappedByRounds: false, + }; +} + +function finalizeCurrentWindowStats( + session: PeerSyncSession, + options: { completed?: boolean; cappedByRounds?: boolean } = {}, +): NegentropyWindowStats | null { + if (!session.currentWindowStats) { + return null; + } + + const finished: NegentropyWindowStats = { + ...session.currentWindowStats, + completed: options.completed ?? true, + cappedByRounds: options.cappedByRounds ?? false, + }; + if (session.currentWindowStats.completed) { + return session.currentWindowStats; + } + session.currentWindowStats = finished; + session.completedWindows.push({ ...finished }); + return finished; +} + +function shouldAdvanceToOlderWindow(session: PeerSyncSession): boolean { + const hasMoreWindows = session.windowIndex + 1 < session.windows.length; + if (!hasMoreWindows) { + return false; + } + + if (session.fullRepair) { + return true; + } + + return session.currentWindowStats?.cappedByRecords === true; +} + +async function planRuntimeWindows(): Promise { + if (!negentropyAdapter) { + throw new Error('negentropy adapter unavailable'); + } + + const windows = await negentropyAdapter.planWindows(Date.now()); + if (windows.length > 0) { + return windows; + } + + return [buildFullHistoryWindow()]; +} + function maybeStartBackgroundPrebuild(reason: string): void { if (!negentropyAdapter) { return; @@ -707,7 +880,11 @@ function maybeStartBackgroundPrebuild(reason: string): void { } backgroundPrebuildQueued = false; - void ensureAdapterFresh(`background_${reason}`) + void (async () => { + const windows = await planRuntimeWindows(); + const recentWindow = windows[0]; + await ensureWindowAdapterFresh(recentWindow, `background_${reason}`); + })() .catch(error => { log.error({ error, reason }, 'background negentropy prebuild failed'); }) @@ -723,37 +900,50 @@ function maybeStartBackgroundPrebuild(reason: string): void { }); } -async function ensureAdapterFresh(reason: string): Promise { +async function ensureWindowAdapterFresh(window: ReconciliationWindow, reason: string): Promise { if (!negentropyAdapter) { throw new Error('negentropy adapter unavailable'); } + const targetWindowId = makeWindowId(window); const now = Date.now(); const recentlyBuilt = adapterBuiltAt > 0 && (now - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; + const sameWindow = adapterBuiltWindowId === targetWindowId; - if (!isNegentropyAdapterDirty() && recentlyBuilt) { - return; + if (!isNegentropyAdapterDirty() && recentlyBuilt && sameWindow) { + const cached = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (cached) { + return cached; + } } if (rebuildPromise) { await rebuildPromise; const recentAfterWait = adapterBuiltAt > 0 && (Date.now() - adapterBuiltAt) <= NEG_ADAPTER_MAX_AGE_MS; - if (!isNegentropyAdapterDirty() && recentAfterWait) { - return; + const sameWindowAfterWait = adapterBuiltWindowId === targetWindowId; + if (!isNegentropyAdapterDirty() && recentAfterWait && sameWindowAfterWait) { + const cached = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (cached) { + return cached; + } } } const rebuildStartSeq = adapterChangeSeq; const rebuildStartedAt = Date.now(); const currentRebuildPromise = (async () => { - await negentropyAdapter!.rebuildFromStore(); + const windowStats = await negentropyAdapter!.rebuildForWindow(window); adapterBuiltSeq = rebuildStartSeq; adapterBuiltAt = Date.now(); + adapterBuiltWindowId = targetWindowId; + adapterBuiltWindowStats = cloneWindowStats(windowStats); log.debug( { reason, durationMs: adapterBuiltAt - rebuildStartedAt, adapterBuiltAt, + windowId: targetWindowId, + window: windowLabel(window), dirtyAfterRebuild: isNegentropyAdapterDirty(), }, 'negentropy adapter rebuilt from sync-store' @@ -769,25 +959,93 @@ async function ensureAdapterFresh(reason: string): Promise { rebuildPromise = null; } } + + const refreshed = cloneWindowStats(adapterBuiltWindowStats ?? negentropyAdapter.getLastWindowStats()); + if (!refreshed) { + throw new Error(`negentropy window stats unavailable after rebuild (${targetWindowId})`); + } + return refreshed; } -async function startNegentropyOpen(peerKey: string, session: PeerSyncSession): Promise { +async function startNextNegentropyWindow(peerKey: string, session: PeerSyncSession): Promise { if (!negentropyAdapter) { throw new Error('negentropy adapter unavailable'); } - await ensureAdapterFresh('session_open_initiator'); + const window = getSessionWindow(session); + if (!window) { + throw new Error(`missing reconciliation window at index ${session.windowIndex}`); + } + + const windowId = makeWindowId(window); + const windowStats = await ensureWindowAdapterFresh(window, 'session_open_initiator'); + initializeSessionWindowState(session, window, windowId, windowStats); const firstFrame = await negentropyAdapter.initiate(); const msg: NegOpenMessage = { ...createBaseMessage('neg_open'), sessionId: session.sessionId, + windowId, + window: { + name: window.name, + fromTs: window.fromTs, + toTs: window.toTs, + maxRecords: window.maxRecords, + order: window.order, + }, + fullRepair: session.fullRepair, round: session.rounds, frame: encodeNegentropyFrame(firstFrame), }; if (!sendToPeer(peerKey, msg)) { closePeerSession(peerKey, 'send_neg_open_failed'); + return; } + + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId, + window: windowLabel(window), + fullRepair: session.fullRepair, + }, + 'negentropy window open sent' + ); +} + +async function maybeAdvanceToOlderWindow(peerKey: string, session: PeerSyncSession): Promise { + if (!shouldAdvanceToOlderWindow(session)) { + return false; + } + + session.windowIndex += 1; + await startNextNegentropyWindow(peerKey, session); + return true; +} + +function getExpectedWindowId(session: PeerSyncSession): string { + if (!session.windowId) { + throw new Error(`session ${session.sessionId} has no active window`); + } + return session.windowId; +} + +function isCurrentSessionWindow(peerKey: string, session: PeerSyncSession, windowId: string, msgType: string): boolean { + if (!session.windowId || windowId !== session.windowId) { + log.warn( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + msgType, + expectedWindowId: session.windowId, + receivedWindowId: windowId, + }, + 'ignoring negentropy message for non-current window' + ); + return false; + } + return true; } async function sendOpsReq(peerKey: string, session: PeerSyncSession, ids: string[]): Promise { @@ -798,6 +1056,7 @@ async function sendOpsReq(peerKey: string, session: PeerSyncSession, ids: string const msg: OpsReqMessage = { ...createBaseMessage('ops_req'), sessionId: session.sessionId, + windowId: getExpectedWindowId(session), round: session.rounds, ids: batch, }; @@ -830,6 +1089,7 @@ async function sendOpsPushForIds(peerKey: string, session: PeerSyncSession, ids: const msg: OpsPushMessage = { ...createBaseMessage('ops_push'), sessionId: session.sessionId, + windowId: getExpectedWindowId(session), round: session.rounds, data: opBatch, }; @@ -847,6 +1107,7 @@ function sendNegMsg(peerKey: string, session: PeerSyncSession, frame: string | U const msg: NegMsgMessage = { ...createBaseMessage('neg_msg'), sessionId: session.sessionId, + windowId: getExpectedWindowId(session), round: session.rounds, frame: encodeNegentropyFrame(frame), }; @@ -856,9 +1117,11 @@ function sendNegMsg(peerKey: string, session: PeerSyncSession, frame: string | U function sendNegClose(peerKey: string, session: PeerSyncSession, reason: string): boolean { session.localClosed = true; + const windowId = session.windowId ?? 'none'; const closeMsg: NegCloseMessage = { ...createBaseMessage('neg_close'), sessionId: session.sessionId, + windowId, round: session.rounds, reason, }; @@ -895,7 +1158,9 @@ async function reconcileNegentropyFrame( throw new Error('negentropy adapter unavailable'); } - if (session.rounds >= session.maxRounds) { + const windowRounds = session.currentWindowStats?.rounds ?? 0; + if (windowRounds >= session.maxRounds) { + finalizeCurrentWindowStats(session, { completed: false, cappedByRounds: true }); sendNegClose(peerKey, session, 'max_rounds_reached'); closePeerSession(peerKey, 'max_rounds_reached'); return null; @@ -903,6 +1168,9 @@ async function reconcileNegentropyFrame( const result = await negentropyAdapter.reconcile(frame); session.rounds += 1; + if (session.currentWindowStats) { + session.currentWindowStats.rounds += 1; + } touchPeerSession(peerKey); return { @@ -912,7 +1180,7 @@ async function reconcileNegentropyFrame( }; } -function maybeFinalizeInitiatorSession(peerKey: string, session: PeerSyncSession): void { +async function maybeFinalizeInitiatorSession(peerKey: string, session: PeerSyncSession): Promise { if (!session.initiator) { return; } @@ -925,6 +1193,11 @@ function maybeFinalizeInitiatorSession(peerKey: string, session: PeerSyncSession return; } + const advanced = await maybeAdvanceToOlderWindow(peerKey, session); + if (advanced) { + return; + } + if (!sendNegClose(peerKey, session, 'complete')) { closePeerSession(peerKey, 'send_neg_close_failed'); return; @@ -978,7 +1251,23 @@ async function handleNegentropyRoundAsInitiator( } session.reconciliationComplete = true; - maybeFinalizeInitiatorSession(peerKey, session); + const completedWindow = finalizeCurrentWindowStats(session, { completed: true, cappedByRounds: false }); + if (completedWindow) { + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId: session.windowId, + windowName: completedWindow.windowName, + loaded: completedWindow.loaded, + skipped: completedWindow.skipped, + rounds: completedWindow.rounds, + cappedByRecords: completedWindow.cappedByRecords, + }, + 'negentropy window complete (initiator)' + ); + } + await maybeFinalizeInitiatorSession(peerKey, session); } async function handleNegentropyRoundAsResponder( @@ -1013,6 +1302,22 @@ async function handleNegentropyRoundAsResponder( } session.reconciliationComplete = true; + const completedWindow = finalizeCurrentWindowStats(session, { completed: true, cappedByRounds: false }); + if (completedWindow) { + log.debug( + { + peer: shortName(peerKey), + sessionId: session.sessionId, + windowId: session.windowId, + windowName: completedWindow.windowName, + loaded: completedWindow.loaded, + skipped: completedWindow.skipped, + rounds: completedWindow.rounds, + cappedByRecords: completedWindow.cappedByRecords, + }, + 'negentropy window complete (responder)' + ); + } } function sendBatch(conn: HyperswarmConnection, batch: Operation[]): number { @@ -1407,13 +1712,39 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise return; } - const existing = peerSessions.get(peerKey); - if (existing && existing.sessionId !== msg.sessionId) { + const window = parseRemoteWindow(msg.window); + if (!window) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_open with invalid window'); + return; + } + if (typeof msg.windowId !== 'string' || msg.windowId.length === 0) { + log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_open with invalid windowId'); + return; + } + + let session = peerSessions.get(peerKey); + if (session && session.sessionId !== msg.sessionId) { closePeerSession(peerKey, 'replaced_by_remote_open'); + session = undefined; + } + + if (!session || session.mode !== 'negentropy' || session.sessionId !== msg.sessionId) { + session = createPeerSession(peerKey, 'negentropy', false, msg.sessionId); } - await ensureAdapterFresh('session_open_responder'); - const session = createPeerSession(peerKey, 'negentropy', false, msg.sessionId); + session.initiator = false; + session.fullRepair = msg.fullRepair === true; + session.maxRounds = config.negentropyMaxRoundsPerSession; + const existingIndex = session.windows.findIndex(existingWindow => makeWindowId(existingWindow) === msg.windowId); + if (existingIndex >= 0) { + session.windows[existingIndex] = window; + session.windowIndex = existingIndex; + } else { + session.windows.push(window); + session.windowIndex = session.windows.length - 1; + } + const windowStats = await ensureWindowAdapterFresh(window, 'session_open_responder'); + initializeSessionWindowState(session, window, msg.windowId, windowStats); touchPeerSession(peerKey); await handleNegentropyRoundAsResponder(peerKey, session, decodeNegentropyFrame(msg.frame)); return; @@ -1425,6 +1756,9 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring neg_msg for unknown session'); return; } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'neg_msg')) { + return; + } touchPeerSession(peerKey); if (session.initiator) { @@ -1441,6 +1775,9 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_req for unknown session'); return; } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'ops_req')) { + return; + } const requestedIds = Array.isArray(msg.ids) ? Array.from(new Set(msg.ids.map(id => String(id).toLowerCase()).filter(id => NEG_SYNC_ID_RE.test(id)))) @@ -1457,6 +1794,9 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise log.warn({ peer: shortName(peerKey), sessionId: msg.sessionId }, 'ignoring ops_push for unknown session'); return; } + if (!isCurrentSessionWindow(peerKey, session, msg.windowId, 'ops_push')) { + return; + } const batch = Array.isArray(msg.data) ? msg.data : []; if (batch.length > 0) { @@ -1474,7 +1814,7 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise }); } - maybeFinalizeInitiatorSession(peerKey, session); + await maybeFinalizeInitiatorSession(peerKey, session); } touchPeerSession(peerKey); @@ -1483,7 +1823,7 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise if (msg.type === 'neg_close') { const session = peerSessions.get(peerKey); - if (session && session.sessionId === msg.sessionId) { + if (session && session.sessionId === msg.sessionId && (!session.windowId || msg.windowId === session.windowId)) { closePeerSession(peerKey, msg.reason || 'remote_closed'); } return; @@ -1702,6 +2042,8 @@ async function initNegentropyAdapter(): Promise { adapterChangeSeq = 0; adapterBuiltSeq = -1; adapterBuiltAt = 0; + adapterBuiltWindowId = null; + adapterBuiltWindowStats = null; rebuildPromise = null; backgroundPrebuildQueued = false; log.info( From 626881a3186b08c96826fc2ed5ba995d09204711 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 13 Feb 2026 13:58:26 +0000 Subject: [PATCH 07/31] chore: update generated OpenAPI docs --- doc/gatekeeper-api.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/gatekeeper-api.json b/doc/gatekeeper-api.json index eb231757b..57b0d3939 100644 --- a/doc/gatekeeper-api.json +++ b/doc/gatekeeper-api.json @@ -1217,6 +1217,13 @@ "type": "integer", "description": "Number of events that failed validation (bad signature, size limit, etc.)." }, + "rejectedIndices": { + "type": "array", + "description": "Zero-based indexes of rejected events in the original submitted batch order (for importDIDs this is `dids.flat()` order).", + "items": { + "type": "integer" + } + }, "total": { "type": "integer", "description": "Total number of events in the queue after this import." @@ -1439,6 +1446,13 @@ "type": "integer", "description": "Number of events that failed validation." }, + "rejectedIndices": { + "type": "array", + "description": "Zero-based indexes of rejected events in the original submitted batch order.", + "items": { + "type": "integer" + } + }, "total": { "type": "integer", "description": "The total event queue size after this import." @@ -1786,6 +1800,13 @@ "pending": { "type": "integer", "description": "Number of events still left in the queue after processing." + }, + "acceptedHashes": { + "type": "array", + "description": "Lower-case signature hashes of events accepted during this processing run (added or merged).", + "items": { + "type": "string" + } } } } From c0609c804d97d23b26706a297e65bba219db40df Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 14:47:25 +0000 Subject: [PATCH 08/31] suppress lint warnings --- services/mediators/hyperswarm/src/hyperswarm-mediator.ts | 1 + tests/hyperswarm/bootstrap.test.ts | 1 + tests/hyperswarm/negentropy-adapter.test.ts | 4 ++++ tests/hyperswarm/sync-persistence.test.ts | 1 + 4 files changed, 7 insertions(+) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 390b5d85a..95201113b 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -850,6 +850,7 @@ function shouldAdvanceToOlderWindow(session: PeerSyncSession): boolean { async function planRuntimeWindows(): Promise { if (!negentropyAdapter) { + // eslint-disable-next-line sonarjs/no-duplicate-string throw new Error('negentropy adapter unavailable'); } diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 2ef48dd2e..4c83e3ca0 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -30,6 +30,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { await store.start(); await store.upsertMany([{ id: h('a'), + // eslint-disable-next-line sonarjs/no-duplicate-string ts: Date.parse('2026-02-10T10:00:00.000Z'), operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), }]); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts index 9e1d137f8..1e0a16009 100644 --- a/tests/hyperswarm/negentropy-adapter.test.ts +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -63,7 +63,9 @@ describe('NegentropyAdapter', () => { it('loads and builds from store', async () => { const store = new InMemoryOperationSyncStore(); await seedStore(store, [ + // eslint-disable-next-line sonarjs/no-duplicate-string { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + // eslint-disable-next-line sonarjs/no-duplicate-string { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, ]); @@ -85,6 +87,7 @@ describe('NegentropyAdapter', () => { await seedStore(storeA, [ { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, { id: h('b'), ts: 2000, op: makeOp('b', '2026-02-09T11:00:00.000Z') }, + // eslint-disable-next-line sonarjs/no-duplicate-string { id: h('c'), ts: 3000, op: makeOp('c', '2026-02-09T12:00:00.000Z') }, ]); @@ -140,6 +143,7 @@ describe('NegentropyAdapter', () => { it('plans recent window first then older windows in descending recency', async () => { const store = new InMemoryOperationSyncStore(); + // eslint-disable-next-line sonarjs/no-duplicate-string const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); await seedStore(store, [ { id: h('a'), ts: nowMs - (10 * DAY_MS), op: makeOp('a', '2026-01-31T00:00:00.000Z') }, diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index f71b5576d..9fd752954 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -26,6 +26,7 @@ function makeCreateOp(hashChar: string, signed: string): Operation { describe('sync-persistence helpers', () => { it('filters rejected indices in original submitted order', () => { + // eslint-disable-next-line sonarjs/no-duplicate-string const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); From 16cbacea08265b7ff1912a9abea329f7b6677c01 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 14:51:01 +0000 Subject: [PATCH 09/31] lint: single return statement --- services/mediators/hyperswarm/src/db/memory.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/services/mediators/hyperswarm/src/db/memory.ts b/services/mediators/hyperswarm/src/db/memory.ts index 9e68fd407..88358b8e3 100644 --- a/services/mediators/hyperswarm/src/db/memory.ts +++ b/services/mediators/hyperswarm/src/db/memory.ts @@ -78,11 +78,7 @@ export default class InMemoryOperationSyncStore implements OperationSyncStore { return false; } - if (typeof toTs === 'number' && item.ts > toTs) { - return false; - } - - return true; + return !(typeof toTs === 'number' && item.ts > toTs); }); return filtered.slice(0, limit); From 3ed6df8711a433423202992f107e9e80c992be06 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 16:02:51 +0000 Subject: [PATCH 10/31] improve coverage --- jest.config.js | 4 + tests/hyperswarm/bootstrap.test.ts | 18 +++ tests/hyperswarm/negentropy-adapter.test.ts | 138 ++++++++++++++++++ .../negentropy-observability.test.ts | 10 ++ tests/hyperswarm/negentropy-transfer.test.ts | 19 +++ tests/hyperswarm/sync-mapping.test.ts | 71 +++++++++ tests/hyperswarm/sync-persistence.test.ts | 27 ++++ tests/hyperswarm/sync-store.test.ts | 76 ++++++++++ 8 files changed, 363 insertions(+) create mode 100644 tests/hyperswarm/sync-mapping.test.ts diff --git a/jest.config.js b/jest.config.js index 83d4a6028..2052115d1 100644 --- a/jest.config.js +++ b/jest.config.js @@ -43,6 +43,10 @@ const config = { "/node_modules/", "/kc-app/", "/client/" + ], + coveragePathIgnorePatterns: [ + "/node_modules/", + "/services/mediators/hyperswarm/src/negentropy/Negentropy\\.cjs$", ] }; diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 4c83e3ca0..80f980f20 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -79,4 +79,22 @@ describe('bootstrapSyncStoreIfEmpty', () => { await expect(bootstrapSyncStoreIfEmpty(store, gatekeeper)).rejects.toThrow('boom'); }); + + it('handles empty/invalid export payload without upserting', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + exportBatch: jest.fn(async () => [{ registry: 'hyperswarm', time: new Date().toISOString() }]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper as any); + expect(result.skipped).toBe(false); + expect(result.exported).toBe(0); + expect(result.mapped).toBe(0); + expect(result.invalid).toBe(0); + expect(result.inserted).toBe(0); + expect(result.countAfter).toBe(0); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + }); }); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts index 1e0a16009..2ff171629 100644 --- a/tests/hyperswarm/negentropy-adapter.test.ts +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -1,6 +1,7 @@ import InMemoryOperationSyncStore from '../../services/mediators/hyperswarm/src/db/memory.ts'; import NegentropyAdapter from '../../services/mediators/hyperswarm/src/negentropy/adapter.ts'; import { Operation } from '@mdip/gatekeeper/types'; +import type { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from '../../services/mediators/hyperswarm/src/db/types.ts'; const DAY_MS = 24 * 60 * 60 * 1000; const h = (c: string) => c.repeat(64); @@ -60,6 +61,20 @@ async function seedNumericRange( } describe('NegentropyAdapter', () => { + it('returns null session/window stats before any rebuilds', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + expect(adapter.getLastWindowStats()).toBeNull(); + expect(adapter.getLastSessionStats()).toBeNull(); + }); + it('loads and builds from store', async () => { const store = new InMemoryOperationSyncStore(); await seedStore(store, [ @@ -141,6 +156,107 @@ describe('NegentropyAdapter', () => { })).rejects.toThrow('frameSizeLimit'); }); + it('throws when integer options are invalid', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + recentWindowDays: 0, + })).rejects.toThrow('recentWindowDays'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + olderWindowDays: 0, + })).rejects.toThrow('olderWindowDays'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + maxRecordsPerWindow: 0, + })).rejects.toThrow('maxRecordsPerWindow'); + + await expect(() => NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + maxRoundsPerSession: 0, + })).rejects.toThrow('maxRoundsPerSession'); + }); + + it('throws if initiate/reconcile called before adapter has been built', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapter.initiate()).rejects.toThrow('not initialized'); + await expect(adapter.reconcile('msg')).rejects.toThrow('not initialized'); + }); + + it('returns empty windows for empty store and rejects non-finite nowMs', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, []); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapter.planWindows(Number.NaN)).rejects.toThrow('nowMs must be a finite timestamp'); + await expect(adapter.planWindows(Date.now())).resolves.toStrictEqual([]); + }); + + it('skips invalid sync rows when rebuilding a window', async () => { + const validId = h('a'); + const validTs = Date.parse('2026-02-13T00:00:00.000Z'); + const validOp = makeOp('a', '2026-02-13T00:00:00.000Z'); + const rows: SyncOperationRecord[] = [ + { id: 'invalid-id', ts: validTs, operation: validOp, insertedAt: 1 }, + { id: validId, ts: Number.NaN, operation: validOp, insertedAt: 2 }, + { id: validId, ts: validTs, operation: validOp, insertedAt: 3 }, + ]; + + const stubStore: OperationSyncStore = { + start: async () => undefined, + stop: async () => undefined, + reset: async () => undefined, + upsertMany: async () => 0, + getByIds: async () => [], + has: async () => false, + count: async () => rows.length, + iterateSorted: async (options: SyncStoreListOptions = {}) => { + if (options.after) { + return []; + } + return rows; + }, + }; + + const adapter = await NegentropyAdapter.create({ + syncStore: stubStore, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + const stats = await adapter.rebuildForWindow({ + name: 'recent', + fromTs: Number.MIN_SAFE_INTEGER, + toTs: Number.MAX_SAFE_INTEGER, + maxRecords: 100, + order: 0, + }); + + expect(stats.loaded).toBe(1); + expect(stats.skipped).toBe(2); + }); + it('plans recent window first then older windows in descending recency', async () => { const store = new InMemoryOperationSyncStore(); // eslint-disable-next-line sonarjs/no-duplicate-string @@ -230,6 +346,28 @@ describe('NegentropyAdapter', () => { expect(session.windows[0].rounds).toBe(1); }); + it('throws for invalid runWindowedSessionWithPeer maxRoundsPerSession option', async () => { + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + await seedStore(storeA, []); + await seedStore(storeB, []); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await expect(adapterA.runWindowedSessionWithPeer(adapterB, { + maxRoundsPerSession: 0, + })).rejects.toThrow('maxRoundsPerSession'); + }); + it('continues into older windows when a newer window hits record cap', async () => { const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); const storeA = new InMemoryOperationSyncStore(); diff --git a/tests/hyperswarm/negentropy-observability.test.ts b/tests/hyperswarm/negentropy-observability.test.ts index 165fc8db0..945af92d2 100644 --- a/tests/hyperswarm/negentropy-observability.test.ts +++ b/tests/hyperswarm/negentropy-observability.test.ts @@ -22,6 +22,10 @@ function makeOp(signed?: string): Operation { } describe('negentropy observability helpers', () => { + it('returns zero average for an empty aggregate', () => { + expect(averageAggregate(createAggregateMetric())).toBe(0); + }); + it('aggregates samples and computes average', () => { const metric = createAggregateMetric(); addAggregateSample(metric, 10); @@ -47,6 +51,10 @@ describe('negentropy observability helpers', () => { expect(samples).toStrictEqual([60_000]); }); + it('returns empty samples for empty input', () => { + expect(collectQueueDelaySamples([])).toStrictEqual([]); + }); + it('computes message bytes for string, buffer, and object', () => { expect(messageBytes('abc')).toBe(3); expect(messageBytes(Buffer.from([1, 2, 3]))).toBe(3); @@ -54,6 +62,8 @@ describe('negentropy observability helpers', () => { }); it('returns safe rate for invalid denominator', () => { + expect(safeRate(Number.NaN, 10)).toBe(0); + expect(safeRate(10, Number.NaN)).toBe(0); expect(safeRate(1, 0)).toBe(0); expect(safeRate(1, -1)).toBe(0); expect(safeRate(2, 4)).toBe(0.5); diff --git a/tests/hyperswarm/negentropy-transfer.test.ts b/tests/hyperswarm/negentropy-transfer.test.ts index 7f289852a..03891f10f 100644 --- a/tests/hyperswarm/negentropy-transfer.test.ts +++ b/tests/hyperswarm/negentropy-transfer.test.ts @@ -20,6 +20,18 @@ function makeOp(hashChar: string, sizeTag: string = ''): Operation { } describe('negentropy transfer batching helpers', () => { + it('throws on invalid chunking options', () => { + expect(() => chunkIds(['a'], 0)).toThrow('maxPerChunk'); + expect(() => chunkOperationsForPush([makeOp('a')], { + maxOpsPerPush: 0, + maxBytesPerPush: 1024, + })).toThrow('maxOpsPerPush'); + expect(() => chunkOperationsForPush([makeOp('a')], { + maxOpsPerPush: 1, + maxBytesPerPush: 0, + })).toThrow('maxBytesPerPush'); + }); + it('chunks id lists with de-duplication', () => { const ids = ['a', 'b', 'c', 'c', 'd', 'e']; const chunks = chunkIds(ids, 2); @@ -60,4 +72,11 @@ describe('negentropy transfer batching helpers', () => { expect(batches).toStrictEqual([[op]]); }); + + it('returns empty when operations input is empty', () => { + expect(chunkOperationsForPush([], { + maxOpsPerPush: 10, + maxBytesPerPush: 1024, + })).toStrictEqual([]); + }); }); diff --git a/tests/hyperswarm/sync-mapping.test.ts b/tests/hyperswarm/sync-mapping.test.ts new file mode 100644 index 000000000..3f72f1c26 --- /dev/null +++ b/tests/hyperswarm/sync-mapping.test.ts @@ -0,0 +1,71 @@ +import { Operation } from '@mdip/gatekeeper/types'; +import { + SYNC_ID_BYTES_LEN, + mapOperationToSyncKey, + type SyncMappingErrorCode, +} from '../../services/mediators/hyperswarm/src/sync-mapping.ts'; + +const h = (c: string) => c.repeat(64); + +function makeOp(overrides: Partial> = {}): Operation { + return { + type: 'create', + signature: { + hash: h('a'), + signed: '2026-02-13T00:00:00.000Z', + value: 'sig-a', + ...overrides, + }, + }; +} + +function expectFailure(operation: Operation, code: SyncMappingErrorCode): void { + const result = mapOperationToSyncKey(operation); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(code); + } +} + +describe('sync-mapping', () => { + it('maps valid operation to sync key and normalizes hash case', () => { + const op = makeOp({ hash: h('A') }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.idHex).toBe(h('a')); + expect(result.value.idBytes.length).toBe(SYNC_ID_BYTES_LEN); + expect(result.value.tsMs).toBe(Date.parse('2026-02-13T00:00:00.000Z')); + expect(result.value.operation).toBe(op); + } + }); + + it('rejects missing signature', () => { + expectFailure({ type: 'create' }, 'missing_signature'); + }); + + it('rejects missing signature hash', () => { + expectFailure(makeOp({ hash: '' }), 'missing_signature_hash'); + }); + + it('rejects non-string signature hash', () => { + expectFailure(makeOp({ hash: 123 as unknown as string }), 'invalid_signature_hash_type'); + }); + + it('rejects invalid signature hash format', () => { + expectFailure(makeOp({ hash: 'xyz' }), 'invalid_signature_hash_format'); + }); + + it('rejects missing signature signed', () => { + expectFailure(makeOp({ signed: '' }), 'missing_signature_signed'); + }); + + it('rejects non-string signature signed', () => { + expectFailure(makeOp({ signed: 123 as unknown as string }), 'invalid_signature_signed_type'); + }); + + it('rejects unparseable signature signed value', () => { + expectFailure(makeOp({ signed: 'not-a-date' }), 'invalid_signature_signed_value'); + }); +}); + diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index 9fd752954..f3b005a24 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -25,6 +25,10 @@ function makeCreateOp(hashChar: string, signed: string): Operation { } describe('sync-persistence helpers', () => { + it('returns empty array when filterIndexRejectedOperations receives empty batch', () => { + expect(filterIndexRejectedOperations([], [0, 1])).toStrictEqual([]); + }); + it('filters rejected indices in original submitted order', () => { // eslint-disable-next-line sonarjs/no-duplicate-string const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); @@ -36,6 +40,14 @@ describe('sync-persistence helpers', () => { expect(filtered).toStrictEqual([c]); }); + it('returns original batch when rejected indices are missing/empty', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); + + expect(filterIndexRejectedOperations([a, b], undefined)).toStrictEqual([a, b]); + expect(filterIndexRejectedOperations([a, b], [])).toStrictEqual([a, b]); + }); + it('maps accepted operations to sync records and counts invalid operations', () => { const valid = makeCreateOp('A', '2026-02-10T10:00:00.000Z'); const invalid = makeCreateOp('b', 'not-a-date'); @@ -48,6 +60,15 @@ describe('sync-persistence helpers', () => { expect(result.invalid).toBe(1); }); + it('returns no records when all operations are invalid', () => { + const invalidA = makeCreateOp('a', 'not-a-date'); + const invalidB = { type: 'create' } as Operation; + + const result = mapAcceptedOperationsToSyncRecords([invalidA, invalidB]); + expect(result.records).toStrictEqual([]); + expect(result.invalid).toBe(2); + }); + it('filters operations by accepted hashes', () => { const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); @@ -56,4 +77,10 @@ describe('sync-persistence helpers', () => { const accepted = filterOperationsByAcceptedHashes([a, b, c], [h('a').toUpperCase(), h('c')]); expect(accepted).toStrictEqual([a, c]); }); + + it('returns empty when accepted hashes are empty or normalize to empty set', () => { + const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + expect(filterOperationsByAcceptedHashes([a], [])).toStrictEqual([]); + expect(filterOperationsByAcceptedHashes([a], ['', '' as unknown as string])).toStrictEqual([]); + }); }); diff --git a/tests/hyperswarm/sync-store.test.ts b/tests/hyperswarm/sync-store.test.ts index 9d5c19f38..bb67a6cb3 100644 --- a/tests/hyperswarm/sync-store.test.ts +++ b/tests/hyperswarm/sync-store.test.ts @@ -90,6 +90,23 @@ describe('InMemoryOperationSyncStore', () => { const store = new InMemoryOperationSyncStore(); await runStoreContractTests(store); }); + + it('preserves explicit insertedAt and handles empty getByIds input', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + await store.reset(); + + const insertedAt = 123456789; + const inserted = await store.upsertMany([{ + ...recA, + insertedAt, + }]); + + expect(inserted).toBe(1); + const rows = await store.getByIds([recA.id]); + expect(rows[0].insertedAt).toBe(insertedAt); + expect(await store.getByIds([])).toStrictEqual([]); + }); }); describe('SqliteOperationSyncStore', () => { @@ -109,4 +126,63 @@ describe('SqliteOperationSyncStore', () => { const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); await runStoreContractTests(store); }); + + it('is safe to stop before start and idempotent to start twice', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await expect(store.stop()).resolves.toBeUndefined(); + await store.start(); + await store.start(); + await expect(store.stop()).resolves.toBeUndefined(); + }); + + it('throws from data APIs when start was not called', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + + await expect(store.reset()).rejects.toThrow('Call start() first'); + await expect(store.upsertMany([recA])).rejects.toThrow('Call start() first'); + await expect(store.getByIds([recA.id])).rejects.toThrow('Call start() first'); + await expect(store.iterateSorted()).rejects.toThrow('Call start() first'); + await expect(store.has(recA.id)).rejects.toThrow('Call start() first'); + await expect(store.count()).rejects.toThrow('Call start() first'); + }); + + it('returns early for empty inputs', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + await store.reset(); + + expect(await store.upsertMany([])).toBe(0); + expect(await store.getByIds([])).toStrictEqual([]); + }); + + it('rolls back transaction when an upsert item fails serialization', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + await store.reset(); + + const circular: any = { type: 'create' }; + circular.self = circular; + + await expect(store.upsertMany([ + recA, + { + id: h('d'), + ts: 4000, + operation: circular, + } as any, + ])).rejects.toThrow(); + + expect(await store.count()).toBe(0); + }); + + it('returns zero count when sqlite get returns no row', async () => { + const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + await store.start(); + + const db = (store as any).db; + const originalGet = db.get.bind(db); + db.get = async () => undefined; + await expect(store.count()).resolves.toBe(0); + db.get = originalGet; + }); }); From 5a05aa80cdad82b4db3e9998573e48fbd9eea7c0 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 16:10:59 +0000 Subject: [PATCH 11/31] resolve lint errors --- tests/gatekeeper/client.test.ts | 1 - tests/hyperswarm/sync-persistence.test.ts | 1 + tests/hyperswarm/sync-store.test.ts | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/gatekeeper/client.test.ts b/tests/gatekeeper/client.test.ts index 8719017a3..317279aa6 100644 --- a/tests/gatekeeper/client.test.ts +++ b/tests/gatekeeper/client.test.ts @@ -1,7 +1,6 @@ import nock from 'nock'; import GatekeeperClient from '@mdip/gatekeeper/client'; import { ExpectedExceptionError } from '@mdip/common/errors'; -import {Operation} from "@mdip/gatekeeper/types"; const GatekeeperURL = 'http://gatekeeper.org'; const ServerError = { message: 'Server error' }; diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index f3b005a24..4dd69577c 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -32,6 +32,7 @@ describe('sync-persistence helpers', () => { it('filters rejected indices in original submitted order', () => { // eslint-disable-next-line sonarjs/no-duplicate-string const a = makeCreateOp('a', '2026-02-10T10:00:00.000Z'); + // eslint-disable-next-line sonarjs/no-duplicate-string const b = makeCreateOp('b', '2026-02-10T11:00:00.000Z'); const c = makeCreateOp('c', '2026-02-10T12:00:00.000Z'); diff --git a/tests/hyperswarm/sync-store.test.ts b/tests/hyperswarm/sync-store.test.ts index bb67a6cb3..4c7048526 100644 --- a/tests/hyperswarm/sync-store.test.ts +++ b/tests/hyperswarm/sync-store.test.ts @@ -123,6 +123,7 @@ describe('SqliteOperationSyncStore', () => { }); it('implements sync-store contract', async () => { + // eslint-disable-next-line sonarjs/no-duplicate-string const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); await runStoreContractTests(store); }); @@ -138,6 +139,7 @@ describe('SqliteOperationSyncStore', () => { it('throws from data APIs when start was not called', async () => { const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); + // eslint-disable-next-line sonarjs/no-duplicate-string await expect(store.reset()).rejects.toThrow('Call start() first'); await expect(store.upsertMany([recA])).rejects.toThrow('Call start() first'); await expect(store.getByIds([recA.id])).rejects.toThrow('Call start() first'); From 755e8a36bfb811288bc8044d27909fec683a2980 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 13 Feb 2026 16:24:07 +0000 Subject: [PATCH 12/31] more coverage --- tests/hyperswarm/negentropy-transfer.test.ts | 4 ++++ tests/hyperswarm/sync-persistence.test.ts | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/tests/hyperswarm/negentropy-transfer.test.ts b/tests/hyperswarm/negentropy-transfer.test.ts index 03891f10f..576dafdb5 100644 --- a/tests/hyperswarm/negentropy-transfer.test.ts +++ b/tests/hyperswarm/negentropy-transfer.test.ts @@ -42,6 +42,10 @@ describe('negentropy transfer batching helpers', () => { ]); }); + it('returns empty id chunks when input ids are empty', () => { + expect(chunkIds([], 2)).toStrictEqual([]); + }); + it('splits operations by count and bytes', () => { const ops = [ makeOp('a', 'small'), diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index 4dd69577c..39a1fdeaf 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -84,4 +84,9 @@ describe('sync-persistence helpers', () => { expect(filterOperationsByAcceptedHashes([a], [])).toStrictEqual([]); expect(filterOperationsByAcceptedHashes([a], ['', '' as unknown as string])).toStrictEqual([]); }); + + it('returns empty when operations input is empty or not an array', () => { + expect(filterOperationsByAcceptedHashes([], [h('a')])).toStrictEqual([]); + expect(filterOperationsByAcceptedHashes('not-an-array' as unknown as Operation[], [h('a')])).toStrictEqual([]); + }); }); From fbcaa03dce0e8dd2b6b8d50e97408340e28a95f9 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Thu, 19 Feb 2026 10:02:29 +0000 Subject: [PATCH 13/31] add private swarm env var --- sample.env | 2 ++ services/mediators/hyperswarm/src/config.js | 29 +++++++++++++++++++ .../hyperswarm/src/hyperswarm-mediator.ts | 3 +- services/mediators/hyperswarm/src/stubs.d.ts | 12 ++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/sample.env b/sample.env index c180ef219..b5c81e8ef 100644 --- a/sample.env +++ b/sample.env @@ -37,6 +37,8 @@ KC_SEARCH_URL=http://localhost:4002 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 KC_MDIP_PROTOCOL=/MDIP/v1.0-public +# Optional custom HyperDHT endpoints for private swarm. comma-separated host:port entries. +KC_HYPR_DHT_BOOTSTRAP= # Bitcoin mediator KC_BTC_HOST=localhost diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index 140c0bb04..be3a3bf57 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -46,6 +46,34 @@ function parseBooleanEnv(varName, defaultValue) { throw new Error(`Invalid ${varName}; expected true or false`); } +function parseBootstrapEnv(varName) { + const raw = process.env[varName]; + if (raw == null || raw.trim() === '') { + return []; + } + + const endpoints = raw + .split(',') + .map(value => value.trim()) + .filter(value => value.length > 0); + + for (const endpoint of endpoints) { + const splitIndex = endpoint.lastIndexOf(':'); + if (splitIndex <= 0 || splitIndex === endpoint.length - 1) { + throw new Error(`Invalid ${varName}; expected host:port entries`); + } + + const host = endpoint.slice(0, splitIndex).trim(); + const portRaw = endpoint.slice(splitIndex + 1).trim(); + const port = Number.parseInt(portRaw, 10); + if (!host || !Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error(`Invalid ${varName}; expected host:port entries`); + } + } + + return endpoints; +} + const config = { debug: process.env.KC_DEBUG ? process.env.KC_DEBUG === 'true' : false, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', @@ -55,6 +83,7 @@ const config = { nodeID: process.env.KC_NODE_ID || '', nodeName: process.env.KC_NODE_NAME || 'anon', protocol: process.env.KC_MDIP_PROTOCOL || '/MDIP/v1.0-public', + hyperswarmBootstrap: parseBootstrapEnv('KC_HYPR_DHT_BOOTSTRAP'), exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), negentropyFrameSizeLimit: parseFrameSizeLimit(), negentropyRecentWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS', 7), diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 95201113b..e09bf6791 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -307,7 +307,8 @@ async function createSwarm(): Promise { swarm.destroy(); } - swarm = new Hyperswarm(); + const hasCustomBootstrap = config.hyperswarmBootstrap.length > 0; + swarm = new Hyperswarm(hasCustomBootstrap ? { bootstrap: config.hyperswarmBootstrap } : {}); nodeKey = b4a.toString(swarm.keyPair.publicKey, 'hex'); swarm.on('connection', conn => addConnection(conn)); diff --git a/services/mediators/hyperswarm/src/stubs.d.ts b/services/mediators/hyperswarm/src/stubs.d.ts index 64a900ffa..143be4d27 100644 --- a/services/mediators/hyperswarm/src/stubs.d.ts +++ b/services/mediators/hyperswarm/src/stubs.d.ts @@ -12,6 +12,18 @@ declare module 'hyperswarm' { destroy?(): void; } + export interface HyperswarmOptions { + bootstrap?: string[]; + dht?: any; + keyPair?: { publicKey: Buffer; secretKey?: Buffer }; + seed?: Buffer; + maxPeers?: number; + firewall?: (remotePublicKey: Buffer) => boolean; + debug?: boolean; + backoffs?: number[]; + jitter?: number; + } + interface Discovery { flushed(): Promise; } From 297266fef60f357b33348334ae7a0dbf04ac3e54 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Thu, 19 Feb 2026 12:50:14 +0000 Subject: [PATCH 14/31] fix macos start script compatibility --- package-lock.json | 64 ++++++++++++++++++++++++++++++++--------------- start-node | 6 +++-- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index e86250720..66a1cbae9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7643,7 +7643,8 @@ "optional": true, "os": [ "android" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-android-arm64": { "version": "4.40.2", @@ -7657,7 +7658,8 @@ "optional": true, "os": [ "android" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-darwin-arm64": { "version": "4.40.2", @@ -7671,7 +7673,8 @@ "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-darwin-x64": { "version": "4.40.2", @@ -7685,7 +7688,8 @@ "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-freebsd-arm64": { "version": "4.40.2", @@ -7699,7 +7703,8 @@ "optional": true, "os": [ "freebsd" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-freebsd-x64": { "version": "4.40.2", @@ -7713,7 +7718,8 @@ "optional": true, "os": [ "freebsd" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { "version": "4.40.2", @@ -7727,7 +7733,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { "version": "4.40.2", @@ -7741,7 +7748,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-arm64-gnu": { "version": "4.40.2", @@ -7755,7 +7763,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-arm64-musl": { "version": "4.40.2", @@ -7769,7 +7778,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-loongarch64-gnu": { "version": "4.40.2", @@ -7783,7 +7793,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { "version": "4.40.2", @@ -7797,7 +7808,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { "version": "4.40.2", @@ -7811,7 +7823,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-riscv64-musl": { "version": "4.40.2", @@ -7825,7 +7838,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-s390x-gnu": { "version": "4.40.2", @@ -7839,7 +7853,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-x64-gnu": { "version": "4.40.2", @@ -7853,7 +7868,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-linux-x64-musl": { "version": "4.40.2", @@ -7867,7 +7883,8 @@ "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-win32-arm64-msvc": { "version": "4.40.2", @@ -7881,7 +7898,8 @@ "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-win32-ia32-msvc": { "version": "4.40.2", @@ -7895,7 +7913,8 @@ "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@rollup/rollup-win32-x64-msvc": { "version": "4.40.2", @@ -7909,7 +7928,8 @@ "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@rtsao/scc": { "version": "1.1.0", @@ -27247,6 +27267,7 @@ "multiformats": "^13.3.3" }, "devDependencies": { + "@rollup/plugin-json": "^6.1.0", "@types/ip": "^1.1.3" } }, @@ -27263,6 +27284,9 @@ "mongodb": "^6.5.0", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" + }, + "devDependencies": { + "@rollup/plugin-json": "^6.1.0" } }, "packages/keymaster/node_modules/image-size": { diff --git a/start-node b/start-node index 26dec4495..0f0a031b5 100755 --- a/start-node +++ b/start-node @@ -11,11 +11,13 @@ if [ -f .env ]; then set +a fi -if [ "${KC_IPFS_ENABLE,,}" != "false" ]; then +KC_IPFS_ENABLE_LC="$(printf '%s' "${KC_IPFS_ENABLE:-}" | tr '[:upper:]' '[:lower:]')" + +if [ "$KC_IPFS_ENABLE_LC" != "false" ]; then PROFILE_ARGS=(--profile ipfs) fi -if [ ${#SERVICES[@]} -gt 0 ] && [ "${KC_IPFS_ENABLE,,}" != "false" ]; then +if [ ${#SERVICES[@]} -gt 0 ] && [ "$KC_IPFS_ENABLE_LC" != "false" ]; then if ! printf '%s\n' "${SERVICES[@]}" | grep -qx "ipfs"; then SERVICES=("ipfs" "${SERVICES[@]}") fi From 033b31bf3ed15b0783e337e295d43ef58a1bbdef Mon Sep 17 00:00:00 2001 From: Bushstar Date: Thu, 19 Feb 2026 14:49:35 +0000 Subject: [PATCH 15/31] add new env vars to docker --- docker-compose.yml | 8 ++++++++ sample.env | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d7aa645df..8958c4642 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,15 @@ services: - KC_NODE_ID=${KC_NODE_ID} - KC_NODE_NAME=${KC_NODE_NAME} - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} + - KC_HYPR_DHT_BOOTSTRAP=${KC_HYPR_DHT_BOOTSTRAP} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} + - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} + - KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS} + - KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS} + - KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=${KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW} + - KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=${KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION} + - KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=${KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS} + - KC_HYPR_LEGACY_SYNC_ENABLE=${KC_HYPR_LEGACY_SYNC_ENABLE} - KC_LOG_LEVEL=${KC_LOG_LEVEL} volumes: - ./data:/app/hyperswarm/data diff --git a/sample.env b/sample.env index b5c81e8ef..e2cbb8b40 100644 --- a/sample.env +++ b/sample.env @@ -35,7 +35,14 @@ KC_KEYMASTER_URL=http://localhost:4226 KC_SEARCH_URL=http://localhost:4002 # Hyperswarm -KC_HYPR_EXPORT_INTERVAL=2 +KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks; integer >= 1. +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 # Negentropy frame byte limit; valid values are 0 (unlimited) or >= 4096. +KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days; integer >= 1. +KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days; integer >= 1. +KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window; integer >= 1. +KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session; integer >= 1. +KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes; integer >= 1. +KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync fallback for peers without negentropy capability; true|false. KC_MDIP_PROTOCOL=/MDIP/v1.0-public # Optional custom HyperDHT endpoints for private swarm. comma-separated host:port entries. KC_HYPR_DHT_BOOTSTRAP= From c23d1b1bd3d4980f37be04e7b1b2e1cd1b7d33ec Mon Sep 17 00:00:00 2001 From: Bushstar Date: Thu, 19 Feb 2026 15:18:05 +0000 Subject: [PATCH 16/31] omit acceptedHashes from logging --- services/mediators/hyperswarm/src/hyperswarm-mediator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index e09bf6791..7c5c317ea 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -1508,7 +1508,8 @@ async function mergeBatch(batch: Operation[]): Promise { const response = await gatekeeper.processEvents(); const processDurationMs = Date.now() - processStart; log.debug({ durationMs: processDurationMs }, 'processEvents'); - log.debug(`mergeBatch: ${JSON.stringify(response)}`); + const { acceptedHashes: _acceptedHashes, ...processSummary } = response; + log.debug(`mergeBatch: ${JSON.stringify(processSummary)}`); syncStats.opsApplied += (response.added ?? 0) + (response.merged ?? 0); syncStats.opsRejected += response.rejected ?? 0; From 929a8282fc8465f2e286bb55fe761ba8cb309236 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Thu, 19 Feb 2026 15:28:51 +0000 Subject: [PATCH 17/31] only sync when sync mode is known --- services/mediators/hyperswarm/src/hyperswarm-mediator.ts | 2 +- services/mediators/hyperswarm/src/negentropy/policy.ts | 2 +- tests/hyperswarm/negentropy-policy.test.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 7c5c317ea..907f9a859 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -596,7 +596,7 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic } if (mode === 'legacy') { - if (!shouldAcceptLegacySync(conn.syncMode, config.legacySyncEnabled)) { + if (!config.legacySyncEnabled) { return; } diff --git a/services/mediators/hyperswarm/src/negentropy/policy.ts b/services/mediators/hyperswarm/src/negentropy/policy.ts index 1c5ac0dda..1fab6be0f 100644 --- a/services/mediators/hyperswarm/src/negentropy/policy.ts +++ b/services/mediators/hyperswarm/src/negentropy/policy.ts @@ -19,7 +19,7 @@ export function shouldAcceptLegacySync( return false; } - return syncMode !== 'negentropy'; + return syncMode === 'legacy'; } export function shouldStartConnectTimeNegentropy( diff --git a/tests/hyperswarm/negentropy-policy.test.ts b/tests/hyperswarm/negentropy-policy.test.ts index 2dd68208a..46db85cfa 100644 --- a/tests/hyperswarm/negentropy-policy.test.ts +++ b/tests/hyperswarm/negentropy-policy.test.ts @@ -5,9 +5,9 @@ import { } from '../../services/mediators/hyperswarm/src/negentropy/policy.ts'; describe('negentropy sync policy', () => { - it('accepts legacy sync only when enabled and peer is not in negentropy mode', () => { + it('accepts legacy sync only when enabled and mode is explicitly legacy', () => { expect(shouldAcceptLegacySync('legacy', true)).toBe(true); - expect(shouldAcceptLegacySync('unknown', true)).toBe(true); + expect(shouldAcceptLegacySync('unknown', true)).toBe(false); expect(shouldAcceptLegacySync('negentropy', true)).toBe(false); expect(shouldAcceptLegacySync('legacy', false)).toBe(false); }); From 0a95944540ddf3befdcc74fe4cb76d3526e00935 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 20 Feb 2026 06:30:00 +0000 Subject: [PATCH 18/31] update test env script --- tests/cli-tests/generate_test_env.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index 4713d5651..76b9e3133 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -47,7 +47,15 @@ KC_KEYMASTER_URL=http://localhost:4226 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 +KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 +KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 +KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 +KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 +KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 +KC_HYPR_LEGACY_SYNC_ENABLE=true KC_MDIP_PROTOCOL=/MDIP/testing +KC_HYPR_DHT_BOOTSTRAP="" # Bitcoin mediator KC_BTC_HOST=localhost From fbba814b18b1d8720c8f6e23106e3cbf4d8d7f6f Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 20 Feb 2026 09:42:10 +0000 Subject: [PATCH 19/31] add frame default --- sample.env | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/sample.env b/sample.env index e2cbb8b40..13bd7d23d 100644 --- a/sample.env +++ b/sample.env @@ -35,17 +35,16 @@ KC_KEYMASTER_URL=http://localhost:4226 KC_SEARCH_URL=http://localhost:4002 # Hyperswarm -KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks; integer >= 1. -KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 # Negentropy frame byte limit; valid values are 0 (unlimited) or >= 4096. -KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days; integer >= 1. -KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days; integer >= 1. -KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window; integer >= 1. -KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session; integer >= 1. -KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes; integer >= 1. -KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync fallback for peers without negentropy capability; true|false. +KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public -# Optional custom HyperDHT endpoints for private swarm. comma-separated host:port entries. -KC_HYPR_DHT_BOOTSTRAP= +KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes. integer >= 1. +KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days. integer >= 1. +KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days. integer >= 1. +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=65536 # Negentropy frame byte limit. valid values are 0 (unlimited) or >= 4096. +KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window. integer >= 1. +KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session. integer >= 1. +KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync fallback for peers without negentropy capability. true|false. +KC_HYPR_DHT_BOOTSTRAP="" # Optional HyperDHT endpoint for private swarm. comma-separated host:port entries. # Bitcoin mediator KC_BTC_HOST=localhost From 4e43feef6d108e03ee7888a9d90d3d3f5441491f Mon Sep 17 00:00:00 2001 From: Bushstar Date: Fri, 20 Feb 2026 12:18:22 +0000 Subject: [PATCH 20/31] remove hyperdht env var --- docker-compose.yml | 1 - sample.env | 1 - services/mediators/hyperswarm/src/config.js | 29 ------------------- .../hyperswarm/src/hyperswarm-mediator.ts | 3 +- tests/cli-tests/generate_test_env.sh | 1 - 5 files changed, 1 insertion(+), 34 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8958c4642..8ec1133d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,6 @@ services: - KC_NODE_ID=${KC_NODE_ID} - KC_NODE_NAME=${KC_NODE_NAME} - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} - - KC_HYPR_DHT_BOOTSTRAP=${KC_HYPR_DHT_BOOTSTRAP} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} - KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS} diff --git a/sample.env b/sample.env index 13bd7d23d..a78a048df 100644 --- a/sample.env +++ b/sample.env @@ -44,7 +44,6 @@ KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=65536 # Negentropy frame byte limit. valid v KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window. integer >= 1. KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session. integer >= 1. KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync fallback for peers without negentropy capability. true|false. -KC_HYPR_DHT_BOOTSTRAP="" # Optional HyperDHT endpoint for private swarm. comma-separated host:port entries. # Bitcoin mediator KC_BTC_HOST=localhost diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index be3a3bf57..140c0bb04 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -46,34 +46,6 @@ function parseBooleanEnv(varName, defaultValue) { throw new Error(`Invalid ${varName}; expected true or false`); } -function parseBootstrapEnv(varName) { - const raw = process.env[varName]; - if (raw == null || raw.trim() === '') { - return []; - } - - const endpoints = raw - .split(',') - .map(value => value.trim()) - .filter(value => value.length > 0); - - for (const endpoint of endpoints) { - const splitIndex = endpoint.lastIndexOf(':'); - if (splitIndex <= 0 || splitIndex === endpoint.length - 1) { - throw new Error(`Invalid ${varName}; expected host:port entries`); - } - - const host = endpoint.slice(0, splitIndex).trim(); - const portRaw = endpoint.slice(splitIndex + 1).trim(); - const port = Number.parseInt(portRaw, 10); - if (!host || !Number.isInteger(port) || port <= 0 || port > 65535) { - throw new Error(`Invalid ${varName}; expected host:port entries`); - } - } - - return endpoints; -} - const config = { debug: process.env.KC_DEBUG ? process.env.KC_DEBUG === 'true' : false, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', @@ -83,7 +55,6 @@ const config = { nodeID: process.env.KC_NODE_ID || '', nodeName: process.env.KC_NODE_NAME || 'anon', protocol: process.env.KC_MDIP_PROTOCOL || '/MDIP/v1.0-public', - hyperswarmBootstrap: parseBootstrapEnv('KC_HYPR_DHT_BOOTSTRAP'), exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), negentropyFrameSizeLimit: parseFrameSizeLimit(), negentropyRecentWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS', 7), diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 907f9a859..58950b3d2 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -307,8 +307,7 @@ async function createSwarm(): Promise { swarm.destroy(); } - const hasCustomBootstrap = config.hyperswarmBootstrap.length > 0; - swarm = new Hyperswarm(hasCustomBootstrap ? { bootstrap: config.hyperswarmBootstrap } : {}); + swarm = new Hyperswarm(); nodeKey = b4a.toString(swarm.keyPair.publicKey, 'hex'); swarm.on('connection', conn => addConnection(conn)); diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index 76b9e3133..09d0aa600 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -55,7 +55,6 @@ KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 KC_HYPR_LEGACY_SYNC_ENABLE=true KC_MDIP_PROTOCOL=/MDIP/testing -KC_HYPR_DHT_BOOTSTRAP="" # Bitcoin mediator KC_BTC_HOST=localhost From ca702fdacd800d18e8b4fda04540df8721d3bc89 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 07:29:50 +0000 Subject: [PATCH 21/31] add negentrophy toggle --- docker-compose.yml | 1 + sample.env | 1 + services/mediators/hyperswarm/src/config.js | 7 ++++ .../hyperswarm/src/hyperswarm-mediator.ts | 28 +++++++++++-- .../hyperswarm/src/negentropy/protocol.ts | 7 +++- tests/cli-tests/generate_test_env.sh | 1 + tests/hyperswarm/config.test.ts | 42 +++++++++++++++++++ tests/hyperswarm/negentropy-protocol.test.ts | 4 ++ 8 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/hyperswarm/config.test.ts diff --git a/docker-compose.yml b/docker-compose.yml index 8ec1133d9..96d87e606 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,6 +100,7 @@ services: - KC_NODE_NAME=${KC_NODE_NAME} - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} + - KC_HYPR_NEGENTROPY_ENABLE=${KC_HYPR_NEGENTROPY_ENABLE} - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} - KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS} - KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS} diff --git a/sample.env b/sample.env index a78a048df..58f82f3c7 100644 --- a/sample.env +++ b/sample.env @@ -37,6 +37,7 @@ KC_SEARCH_URL=http://localhost:4002 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public +KC_HYPR_NEGENTROPY_ENABLE=true # Enables negentropy sync protocol. true|false. KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes. integer >= 1. KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days. integer >= 1. KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days. integer >= 1. diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index 140c0bb04..e8cb6332b 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -56,6 +56,7 @@ const config = { nodeName: process.env.KC_NODE_NAME || 'anon', protocol: process.env.KC_MDIP_PROTOCOL || '/MDIP/v1.0-public', exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), + negentropyEnabled: parseBooleanEnv('KC_HYPR_NEGENTROPY_ENABLE', true), negentropyFrameSizeLimit: parseFrameSizeLimit(), negentropyRecentWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS', 7), negentropyOlderWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS', 30), @@ -65,4 +66,10 @@ const config = { legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), }; +if (!config.negentropyEnabled && !config.legacySyncEnabled) { + throw new Error( + 'Invalid sync configuration; at least one of KC_HYPR_NEGENTROPY_ENABLE or KC_HYPR_LEGACY_SYNC_ENABLE must be true' + ); +} + export default config; diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 58950b3d2..e95b6feb2 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -402,13 +402,19 @@ function createBaseMessage(type: T): Omit { } async function initNegentropyAdapter(): Promise { + if (!config.negentropyEnabled) { + negentropyAdapter = null; + adapterChangeSeq = 0; + adapterBuiltSeq = -1; + adapterBuiltAt = 0; + adapterBuiltWindowId = null; + adapterBuiltWindowStats = null; + rebuildPromise = null; + backgroundPrebuildQueued = false; + log.info('negentropy disabled via KC_HYPR_NEGENTROPY_ENABLE; using legacy sync mode when available'); + return; + } + negentropyAdapter = await NegentropyAdapter.create({ syncStore, frameSizeLimit: config.negentropyFrameSizeLimit, diff --git a/services/mediators/hyperswarm/src/negentropy/protocol.ts b/services/mediators/hyperswarm/src/negentropy/protocol.ts index 3ecbfcdf5..43cd9d7b6 100644 --- a/services/mediators/hyperswarm/src/negentropy/protocol.ts +++ b/services/mediators/hyperswarm/src/negentropy/protocol.ts @@ -76,8 +76,9 @@ export function chooseConnectSyncMode( capabilities: NegotiatedPeerCapabilities, minVersion: number, legacySyncEnabled: boolean, + negentropyEnabled = true, ): ConnectSyncModeDecision { - if (supportsPeerNegentropy(capabilities, minVersion)) { + if (negentropyEnabled && supportsPeerNegentropy(capabilities, minVersion)) { return { mode: 'negentropy', reason: 'negentropy_supported' }; } @@ -85,6 +86,10 @@ export function chooseConnectSyncMode( return { mode: null, reason: 'legacy_disabled' }; } + if (!negentropyEnabled) { + return { mode: 'legacy', reason: 'negentropy_disabled' }; + } + if (!capabilities.advertised) { return { mode: 'legacy', reason: 'missing_capabilities' }; } diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index 09d0aa600..bc685720e 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -47,6 +47,7 @@ KC_KEYMASTER_URL=http://localhost:4226 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 +KC_HYPR_NEGENTROPY_ENABLE=true KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 diff --git a/tests/hyperswarm/config.test.ts b/tests/hyperswarm/config.test.ts new file mode 100644 index 000000000..bfd2ed806 --- /dev/null +++ b/tests/hyperswarm/config.test.ts @@ -0,0 +1,42 @@ +import { jest } from '@jest/globals'; + +const CONFIG_PATH = '../../services/mediators/hyperswarm/src/config.js'; +const ORIGINAL_ENV = { ...process.env }; + +async function importConfigIsolated() { + let loaded: any; + await jest.isolateModulesAsync(async () => { + loaded = (await import(CONFIG_PATH)).default; + }); + return loaded; +} + +describe('hyperswarm config', () => { + afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + jest.resetModules(); + }); + + it('throws when both negentropy and legacy sync are disabled', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'false'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'false'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow( + 'Invalid sync configuration; at least one of KC_HYPR_NEGENTROPY_ENABLE or KC_HYPR_LEGACY_SYNC_ENABLE must be true' + ); + }); + + it('allows startup when at least one sync mode is enabled', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'false'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'true'; + + const config = await importConfigIsolated(); + + expect(config.negentropyEnabled).toBe(false); + expect(config.legacySyncEnabled).toBe(true); + }); +}); diff --git a/tests/hyperswarm/negentropy-protocol.test.ts b/tests/hyperswarm/negentropy-protocol.test.ts index 85e355f42..2850b0ea3 100644 --- a/tests/hyperswarm/negentropy-protocol.test.ts +++ b/tests/hyperswarm/negentropy-protocol.test.ts @@ -80,6 +80,10 @@ describe('negentropy protocol helpers', () => { mode: 'negentropy', reason: 'negentropy_supported', }); + expect(chooseConnectSyncMode(supported, 1, true, false)).toStrictEqual({ + mode: 'legacy', + reason: 'negentropy_disabled', + }); }); it('normalizes negentropy ids and filters invalid values', () => { From c8161c41ede5cd7cae3bbf93c384546527620fe3 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 13:09:01 +0000 Subject: [PATCH 22/31] store time in seconds to match negentrophy --- sample.env | 4 +- .../hyperswarm/src/hyperswarm-mediator.ts | 6 ++- .../hyperswarm/src/negentropy/adapter.ts | 27 ++++++---- .../mediators/hyperswarm/src/sync-mapping.ts | 12 ++--- .../hyperswarm/src/sync-persistence.ts | 2 +- tests/hyperswarm/bootstrap.test.ts | 2 +- tests/hyperswarm/negentropy-adapter.test.ts | 50 ++++++++++--------- tests/hyperswarm/sync-mapping.test.ts | 3 +- tests/hyperswarm/sync-persistence.test.ts | 2 +- 9 files changed, 59 insertions(+), 49 deletions(-) diff --git a/sample.env b/sample.env index 58f82f3c7..74869b4e2 100644 --- a/sample.env +++ b/sample.env @@ -37,14 +37,14 @@ KC_SEARCH_URL=http://localhost:4002 # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public +KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync for peers without negentropy. true|false. KC_HYPR_NEGENTROPY_ENABLE=true # Enables negentropy sync protocol. true|false. KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes. integer >= 1. KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days. integer >= 1. KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days. integer >= 1. -KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=65536 # Negentropy frame byte limit. valid values are 0 (unlimited) or >= 4096. +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=131072 # Negentropy frame byte limit. valid values are 0 (unlimited) or >= 4096. KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window. integer >= 1. KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session. integer >= 1. -KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync fallback for peers without negentropy capability. true|false. # Bitcoin mediator KC_BTC_HOST=localhost diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index e95b6feb2..8d52ff37d 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -756,6 +756,10 @@ function buildFullHistoryWindow(): ReconciliationWindow { }; } +function currentSyncTimestampSec(): number { + return Math.floor(Date.now() / 1000); +} + function makeWindowId(window: ReconciliationWindow): string { return `${window.order}:${window.name}:${window.fromTs}:${window.toTs}:${window.maxRecords}`; } @@ -861,7 +865,7 @@ async function planRuntimeWindows(): Promise { throw new Error('negentropy adapter unavailable'); } - const windows = await negentropyAdapter.planWindows(Date.now()); + const windows = await negentropyAdapter.planWindows(currentSyncTimestampSec()); if (windows.length > 0) { return windows; } diff --git a/services/mediators/hyperswarm/src/negentropy/adapter.ts b/services/mediators/hyperswarm/src/negentropy/adapter.ts index 2e6f99d4b..f7b1cfe46 100644 --- a/services/mediators/hyperswarm/src/negentropy/adapter.ts +++ b/services/mediators/hyperswarm/src/negentropy/adapter.ts @@ -11,7 +11,7 @@ const DEFAULT_RECENT_WINDOW_DAYS = 7; const DEFAULT_OLDER_WINDOW_DAYS = 30; const DEFAULT_MAX_RECORDS_PER_WINDOW = 25_000; const DEFAULT_MAX_ROUNDS_PER_SESSION = 64; -const DAY_MS = 24 * 60 * 60 * 1000; +const DAY_SECONDS = 24 * 60 * 60; const require = createRequire(import.meta.url); interface NegentropyInstance { @@ -85,6 +85,7 @@ export interface NegentropySessionStats { } export interface NegentropyWindowSessionOptions { + nowTs?: number; nowMs?: number; maxRoundsPerSession?: number; } @@ -160,9 +161,9 @@ export default class NegentropyAdapter { return this.rebuildWindowAdapter(window); } - async planWindows(nowMs: number = Date.now(), earliestTsOverride?: number): Promise { - if (!Number.isFinite(nowMs)) { - throw new Error('nowMs must be a finite timestamp'); + async planWindows(nowTs: number = currentEpochSeconds(), earliestTsOverride?: number): Promise { + if (!Number.isFinite(nowTs)) { + throw new Error('nowTs must be a finite timestamp'); } const earliestTs = typeof earliestTsOverride === 'number' @@ -174,14 +175,14 @@ export default class NegentropyAdapter { } const windows: ReconciliationWindow[] = []; - const recentSpanMs = this.recentWindowDays * DAY_MS; - const olderSpanMs = this.olderWindowDays * DAY_MS; - const recentStart = Math.max(earliestTs, nowMs - recentSpanMs); + const recentSpanTs = this.recentWindowDays * DAY_SECONDS; + const olderSpanTs = this.olderWindowDays * DAY_SECONDS; + const recentStart = Math.max(earliestTs, nowTs - recentSpanTs); windows.push({ name: 'recent', fromTs: recentStart, - toTs: nowMs, + toTs: nowTs, maxRecords: this.maxRecordsPerWindow, order: 0, }); @@ -189,7 +190,7 @@ export default class NegentropyAdapter { let cursorTo = recentStart - 1; let order = 1; while (cursorTo >= earliestTs) { - const fromTs = Math.max(earliestTs, cursorTo - olderSpanMs + 1); + const fromTs = Math.max(earliestTs, cursorTo - olderSpanTs + 1); windows.push({ name: `older_${order}`, fromTs, @@ -209,7 +210,7 @@ export default class NegentropyAdapter { options: NegentropyWindowSessionOptions = {}, ): Promise { const startedAt = Date.now(); - const sessionNowMs = options.nowMs ?? Date.now(); + const sessionNowTs = options.nowTs ?? options.nowMs ?? currentEpochSeconds(); const maxRoundsPerSession = options.maxRoundsPerSession ?? this.maxRoundsPerSession; assertPositiveInteger(maxRoundsPerSession, 'maxRoundsPerSession'); @@ -222,7 +223,7 @@ export default class NegentropyAdapter { const windows = earliestCandidates.length === 0 ? [] - : await this.planWindows(sessionNowMs, Math.min(...earliestCandidates)); + : await this.planWindows(sessionNowTs, Math.min(...earliestCandidates)); const windowStats: NegentropyWindowStats[] = []; let totalLoaded = 0; @@ -452,6 +453,10 @@ export default class NegentropyAdapter { } } +function currentEpochSeconds(): number { + return Math.floor(Date.now() / 1000); +} + function isValidSyncId(id: string): boolean { return typeof id === 'string' && /^[a-f0-9]{64}$/i.test(id); } diff --git a/services/mediators/hyperswarm/src/sync-mapping.ts b/services/mediators/hyperswarm/src/sync-mapping.ts index f52178853..e0780b767 100644 --- a/services/mediators/hyperswarm/src/sync-mapping.ts +++ b/services/mediators/hyperswarm/src/sync-mapping.ts @@ -1,14 +1,14 @@ import { Operation } from '@mdip/gatekeeper/types'; // - id = operation.signature.hash (64 hex chars => 32 bytes) -// - timestamp = Date.parse(operation.signature.signed) in ms +// - timestamp = Math.floor(Date.parse(operation.signature.signed) / 1000) in epoch seconds export const SYNC_ID_HEX_LEN = 64; export const SYNC_ID_BYTES_LEN = 32; export interface SyncMappedOperation { idHex: string; idBytes: Buffer; - tsMs: number; + tsSec: number; operation: Operation; } @@ -69,10 +69,11 @@ export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { return fail('invalid_signature_signed_type', 'operation.signature.signed must be a string'); } - const tsMs = Date.parse(signature.signed); - if (!Number.isFinite(tsMs)) { + const parsedSigned = Date.parse(signature.signed); + if (!Number.isFinite(parsedSigned)) { return fail('invalid_signature_signed_value', 'operation.signature.signed must be parseable by Date.parse'); } + const tsSec = Math.floor(parsedSigned / 1000); const idBytes = Buffer.from(idHex, 'hex'); if (idBytes.length !== SYNC_ID_BYTES_LEN) { @@ -87,9 +88,8 @@ export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { value: { idHex, idBytes, - tsMs, + tsSec, operation, }, }; } - diff --git a/services/mediators/hyperswarm/src/sync-persistence.ts b/services/mediators/hyperswarm/src/sync-persistence.ts index 4375c60d6..53ca45ad7 100644 --- a/services/mediators/hyperswarm/src/sync-persistence.ts +++ b/services/mediators/hyperswarm/src/sync-persistence.ts @@ -72,7 +72,7 @@ export function mapAcceptedOperationsToSyncRecords(operations: Operation[]): Map records.push({ id: mapped.value.idHex, - ts: mapped.value.tsMs, + ts: mapped.value.tsSec, operation, }); } diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 80f980f20..bdc39971c 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -31,7 +31,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { await store.upsertMany([{ id: h('a'), // eslint-disable-next-line sonarjs/no-duplicate-string - ts: Date.parse('2026-02-10T10:00:00.000Z'), + ts: Math.floor(Date.parse('2026-02-10T10:00:00.000Z') / 1000), operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), }]); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts index 2ff171629..2b61f8407 100644 --- a/tests/hyperswarm/negentropy-adapter.test.ts +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -3,9 +3,11 @@ import NegentropyAdapter from '../../services/mediators/hyperswarm/src/negentrop import { Operation } from '@mdip/gatekeeper/types'; import type { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from '../../services/mediators/hyperswarm/src/db/types.ts'; -const DAY_MS = 24 * 60 * 60 * 1000; +const DAY_SECONDS = 24 * 60 * 60; const h = (c: string) => c.repeat(64); const idFromNum = (n: number) => n.toString(16).padStart(64, '0'); +const toEpochSeconds = (iso: string) => Math.floor(Date.parse(iso) / 1000); +const toISOFromEpochSeconds = (ts: number) => new Date(ts * 1000).toISOString(); function makeOp(hashChar: string, signed: string): Operation { return { @@ -54,7 +56,7 @@ async function seedNumericRange( records.push({ id, ts: baseTs + i, - op: makeOpFromHash(id, new Date(baseTs + i).toISOString()), + op: makeOpFromHash(id, toISOFromEpochSeconds(baseTs + i)), }); } await seedStore(store, records); @@ -199,7 +201,7 @@ describe('NegentropyAdapter', () => { await expect(adapter.reconcile('msg')).rejects.toThrow('not initialized'); }); - it('returns empty windows for empty store and rejects non-finite nowMs', async () => { + it('returns empty windows for empty store and rejects non-finite nowTs', async () => { const store = new InMemoryOperationSyncStore(); await seedStore(store, []); @@ -209,13 +211,13 @@ describe('NegentropyAdapter', () => { deferInitialBuild: true, }); - await expect(adapter.planWindows(Number.NaN)).rejects.toThrow('nowMs must be a finite timestamp'); - await expect(adapter.planWindows(Date.now())).resolves.toStrictEqual([]); + await expect(adapter.planWindows(Number.NaN)).rejects.toThrow('nowTs must be a finite timestamp'); + await expect(adapter.planWindows(Math.floor(Date.now() / 1000))).resolves.toStrictEqual([]); }); it('skips invalid sync rows when rebuilding a window', async () => { const validId = h('a'); - const validTs = Date.parse('2026-02-13T00:00:00.000Z'); + const validTs = toEpochSeconds('2026-02-13T00:00:00.000Z'); const validOp = makeOp('a', '2026-02-13T00:00:00.000Z'); const rows: SyncOperationRecord[] = [ { id: 'invalid-id', ts: validTs, operation: validOp, insertedAt: 1 }, @@ -260,10 +262,10 @@ describe('NegentropyAdapter', () => { it('plans recent window first then older windows in descending recency', async () => { const store = new InMemoryOperationSyncStore(); // eslint-disable-next-line sonarjs/no-duplicate-string - const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); await seedStore(store, [ - { id: h('a'), ts: nowMs - (10 * DAY_MS), op: makeOp('a', '2026-01-31T00:00:00.000Z') }, - { id: h('b'), ts: nowMs - (2 * DAY_MS), op: makeOp('b', '2026-02-08T00:00:00.000Z') }, + { id: h('a'), ts: nowTs - (10 * DAY_SECONDS), op: makeOp('a', '2026-01-31T00:00:00.000Z') }, + { id: h('b'), ts: nowTs - (2 * DAY_SECONDS), op: makeOp('b', '2026-02-08T00:00:00.000Z') }, ]); const adapter = await NegentropyAdapter.create({ @@ -274,10 +276,10 @@ describe('NegentropyAdapter', () => { deferInitialBuild: true, }); - const windows = await adapter.planWindows(nowMs); + const windows = await adapter.planWindows(nowTs); expect(windows.length).toBeGreaterThanOrEqual(2); expect(windows[0].name).toBe('recent'); - expect(windows[0].toTs).toBe(nowMs); + expect(windows[0].toTs).toBe(nowTs); expect(windows[1].fromTs).toBeLessThanOrEqual(windows[1].toTs); expect(windows[1].toTs).toBe(windows[0].fromTs - 1); }); @@ -313,8 +315,8 @@ describe('NegentropyAdapter', () => { it('applies maxRoundsPerSession cap in windowed sessions', async () => { const storeA = new InMemoryOperationSyncStore(); const storeB = new InMemoryOperationSyncStore(); - const baseTs = Date.parse('2026-02-01T00:00:00.000Z'); - const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + const baseTs = toEpochSeconds('2026-02-01T00:00:00.000Z'); + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); await seedNumericRange(storeA, 0, 2000, baseTs); await seedNumericRange(storeB, 1000, 3000, baseTs); @@ -337,7 +339,7 @@ describe('NegentropyAdapter', () => { }); const session = await adapterA.runWindowedSessionWithPeer(adapterB, { - nowMs, + nowTs, maxRoundsPerSession: 1, }); @@ -369,22 +371,22 @@ describe('NegentropyAdapter', () => { }); it('continues into older windows when a newer window hits record cap', async () => { - const nowMs = Date.parse('2026-02-10T00:00:00.000Z'); + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); const storeA = new InMemoryOperationSyncStore(); const storeB = new InMemoryOperationSyncStore(); - const recentA = nowMs - (6 * 60 * 60 * 1000); - const recentB = nowMs - (8 * 60 * 60 * 1000); - const older = nowMs - (2 * DAY_MS); + const recentA = nowTs - (6 * 60 * 60); + const recentB = nowTs - (8 * 60 * 60); + const older = nowTs - (2 * DAY_SECONDS); await seedStore(storeA, [ - { id: h('a'), ts: recentA, op: makeOp('a', new Date(recentA).toISOString()) }, - { id: h('b'), ts: recentB, op: makeOp('b', new Date(recentB).toISOString()) }, - { id: h('c'), ts: older, op: makeOp('c', new Date(older).toISOString()) }, + { id: h('a'), ts: recentA, op: makeOp('a', toISOFromEpochSeconds(recentA)) }, + { id: h('b'), ts: recentB, op: makeOp('b', toISOFromEpochSeconds(recentB)) }, + { id: h('c'), ts: older, op: makeOp('c', toISOFromEpochSeconds(older)) }, ]); await seedStore(storeB, [ - { id: h('d'), ts: recentA, op: makeOp('d', new Date(recentA).toISOString()) }, - { id: h('e'), ts: older, op: makeOp('e', new Date(older).toISOString()) }, + { id: h('d'), ts: recentA, op: makeOp('d', toISOFromEpochSeconds(recentA)) }, + { id: h('e'), ts: older, op: makeOp('e', toISOFromEpochSeconds(older)) }, ]); const adapterA = await NegentropyAdapter.create({ @@ -404,7 +406,7 @@ describe('NegentropyAdapter', () => { deferInitialBuild: true, }); - const session = await adapterA.runWindowedSessionWithPeer(adapterB, { nowMs }); + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { nowTs }); const recent = session.windows.find(window => window.windowName === 'recent'); const olderWindow = session.windows.find(window => window.windowName === 'older_1'); diff --git a/tests/hyperswarm/sync-mapping.test.ts b/tests/hyperswarm/sync-mapping.test.ts index 3f72f1c26..b939a4b1f 100644 --- a/tests/hyperswarm/sync-mapping.test.ts +++ b/tests/hyperswarm/sync-mapping.test.ts @@ -35,7 +35,7 @@ describe('sync-mapping', () => { if (result.ok) { expect(result.value.idHex).toBe(h('a')); expect(result.value.idBytes.length).toBe(SYNC_ID_BYTES_LEN); - expect(result.value.tsMs).toBe(Date.parse('2026-02-13T00:00:00.000Z')); + expect(result.value.tsSec).toBe(Math.floor(Date.parse('2026-02-13T00:00:00.000Z') / 1000)); expect(result.value.operation).toBe(op); } }); @@ -68,4 +68,3 @@ describe('sync-mapping', () => { expectFailure(makeOp({ signed: 'not-a-date' }), 'invalid_signature_signed_value'); }); }); - diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index 39a1fdeaf..73fb99086 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -57,7 +57,7 @@ describe('sync-persistence helpers', () => { expect(result.records.length).toBe(1); expect(result.records[0].id).toBe(h('a')); - expect(result.records[0].ts).toBe(Date.parse(valid.signature!.signed)); + expect(result.records[0].ts).toBe(Math.floor(Date.parse(valid.signature!.signed) / 1000)); expect(result.invalid).toBe(1); }); From 50f5165e100567a0587dbc9934cac0798667543c Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 13:16:24 +0000 Subject: [PATCH 23/31] wait for shutdown to complete --- .../hyperswarm/src/hyperswarm-mediator.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 8d52ff37d..e6e3c39d4 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -292,14 +292,22 @@ let swarm: Hyperswarm | null = null; let nodeKey = ''; let nodeInfo: NodeInfo; -goodbye(() => { +goodbye(async () => { if (swarm) { - swarm.destroy(); + try { + await Promise.resolve(swarm.destroy()); + } catch (error) { + log.error({ error }, 'swarm destroy error'); + } finally { + swarm = null; + } } - void syncStore.stop().catch(error => { + try { + await syncStore.stop(); + } catch (error) { log.error({ error }, 'syncStore stop error'); - }); + } }); async function createSwarm(): Promise { From a5670e3c7414c6c7b289a88e4f721ec575124a6c Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 14:43:45 +0000 Subject: [PATCH 24/31] replace unix epoch with mdip epoch --- services/mediators/hyperswarm/src/sync-mapping.ts | 5 ++++- tests/hyperswarm/sync-mapping.test.ts | 10 ++++++++++ tests/hyperswarm/sync-persistence.test.ts | 11 +++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/services/mediators/hyperswarm/src/sync-mapping.ts b/services/mediators/hyperswarm/src/sync-mapping.ts index e0780b767..b5b6cfb0a 100644 --- a/services/mediators/hyperswarm/src/sync-mapping.ts +++ b/services/mediators/hyperswarm/src/sync-mapping.ts @@ -2,8 +2,10 @@ import { Operation } from '@mdip/gatekeeper/types'; // - id = operation.signature.hash (64 hex chars => 32 bytes) // - timestamp = Math.floor(Date.parse(operation.signature.signed) / 1000) in epoch seconds +// - if timestamp is Unix epoch (0), remap to MDIP epoch (2024-01-01T00:00:00Z) export const SYNC_ID_HEX_LEN = 64; export const SYNC_ID_BYTES_LEN = 32; +export const MDIP_EPOCH_SECONDS = 1_704_067_200; // 2024-01-01T00:00:00Z export interface SyncMappedOperation { idHex: string; @@ -74,6 +76,7 @@ export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { return fail('invalid_signature_signed_value', 'operation.signature.signed must be parseable by Date.parse'); } const tsSec = Math.floor(parsedSigned / 1000); + const normalizedTsSec = tsSec === 0 ? MDIP_EPOCH_SECONDS : tsSec; const idBytes = Buffer.from(idHex, 'hex'); if (idBytes.length !== SYNC_ID_BYTES_LEN) { @@ -88,7 +91,7 @@ export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { value: { idHex, idBytes, - tsSec, + tsSec: normalizedTsSec, operation, }, }; diff --git a/tests/hyperswarm/sync-mapping.test.ts b/tests/hyperswarm/sync-mapping.test.ts index b939a4b1f..1f3d9b74e 100644 --- a/tests/hyperswarm/sync-mapping.test.ts +++ b/tests/hyperswarm/sync-mapping.test.ts @@ -1,5 +1,6 @@ import { Operation } from '@mdip/gatekeeper/types'; import { + MDIP_EPOCH_SECONDS, SYNC_ID_BYTES_LEN, mapOperationToSyncKey, type SyncMappingErrorCode, @@ -40,6 +41,15 @@ describe('sync-mapping', () => { } }); + it('remaps unix epoch signed timestamp to MDIP epoch', () => { + const op = makeOp({ signed: '1970-01-01T00:00:00.000Z' }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.tsSec).toBe(MDIP_EPOCH_SECONDS); + } + }); + it('rejects missing signature', () => { expectFailure({ type: 'create' }, 'missing_signature'); }); diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index 73fb99086..4f342a3c5 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -4,6 +4,9 @@ import { filterIndexRejectedOperations, mapAcceptedOperationsToSyncRecords, } from '../../services/mediators/hyperswarm/src/sync-persistence.ts'; +import { + MDIP_EPOCH_SECONDS, +} from '../../services/mediators/hyperswarm/src/sync-mapping.ts'; const h = (c: string) => c.repeat(64); @@ -61,6 +64,14 @@ describe('sync-persistence helpers', () => { expect(result.invalid).toBe(1); }); + it('maps unix epoch signed timestamp to MDIP epoch seconds', () => { + const legacyEpoch = makeCreateOp('a', '1970-01-01T00:00:00.000Z'); + const result = mapAcceptedOperationsToSyncRecords([legacyEpoch]); + expect(result.records.length).toBe(1); + expect(result.records[0].ts).toBe(MDIP_EPOCH_SECONDS); + expect(result.invalid).toBe(0); + }); + it('returns no records when all operations are invalid', () => { const invalidA = makeCreateOp('a', 'not-a-date'); const invalidB = { type: 'create' } as Operation; From 1f7bb672bf164bdb11aefe4ea8f8243a672c9830 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 14:55:50 +0000 Subject: [PATCH 25/31] clamp older dates to the mdip epoch --- services/mediators/hyperswarm/src/sync-mapping.ts | 4 ++-- tests/hyperswarm/sync-mapping.test.ts | 11 ++++++++++- tests/hyperswarm/sync-persistence.test.ts | 8 ++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/services/mediators/hyperswarm/src/sync-mapping.ts b/services/mediators/hyperswarm/src/sync-mapping.ts index b5b6cfb0a..ed1fb400a 100644 --- a/services/mediators/hyperswarm/src/sync-mapping.ts +++ b/services/mediators/hyperswarm/src/sync-mapping.ts @@ -2,7 +2,7 @@ import { Operation } from '@mdip/gatekeeper/types'; // - id = operation.signature.hash (64 hex chars => 32 bytes) // - timestamp = Math.floor(Date.parse(operation.signature.signed) / 1000) in epoch seconds -// - if timestamp is Unix epoch (0), remap to MDIP epoch (2024-01-01T00:00:00Z) +// - if timestamp is before MDIP epoch, clamp it to MDIP epoch (2024-01-01T00:00:00Z) export const SYNC_ID_HEX_LEN = 64; export const SYNC_ID_BYTES_LEN = 32; export const MDIP_EPOCH_SECONDS = 1_704_067_200; // 2024-01-01T00:00:00Z @@ -76,7 +76,7 @@ export function mapOperationToSyncKey(operation: Operation): SyncMappingResult { return fail('invalid_signature_signed_value', 'operation.signature.signed must be parseable by Date.parse'); } const tsSec = Math.floor(parsedSigned / 1000); - const normalizedTsSec = tsSec === 0 ? MDIP_EPOCH_SECONDS : tsSec; + const normalizedTsSec = tsSec < MDIP_EPOCH_SECONDS ? MDIP_EPOCH_SECONDS : tsSec; const idBytes = Buffer.from(idHex, 'hex'); if (idBytes.length !== SYNC_ID_BYTES_LEN) { diff --git a/tests/hyperswarm/sync-mapping.test.ts b/tests/hyperswarm/sync-mapping.test.ts index 1f3d9b74e..974fb67a3 100644 --- a/tests/hyperswarm/sync-mapping.test.ts +++ b/tests/hyperswarm/sync-mapping.test.ts @@ -41,7 +41,7 @@ describe('sync-mapping', () => { } }); - it('remaps unix epoch signed timestamp to MDIP epoch', () => { + it('clamps unix epoch signed timestamp to MDIP epoch', () => { const op = makeOp({ signed: '1970-01-01T00:00:00.000Z' }); const result = mapOperationToSyncKey(op); expect(result.ok).toBe(true); @@ -50,6 +50,15 @@ describe('sync-mapping', () => { } }); + it('clamps pre-MDIP signed timestamps to MDIP epoch', () => { + const op = makeOp({ signed: '1971-01-01T00:00:00.000Z' }); + const result = mapOperationToSyncKey(op); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value.tsSec).toBe(MDIP_EPOCH_SECONDS); + } + }); + it('rejects missing signature', () => { expectFailure({ type: 'create' }, 'missing_signature'); }); diff --git a/tests/hyperswarm/sync-persistence.test.ts b/tests/hyperswarm/sync-persistence.test.ts index 4f342a3c5..0085d5e0f 100644 --- a/tests/hyperswarm/sync-persistence.test.ts +++ b/tests/hyperswarm/sync-persistence.test.ts @@ -64,6 +64,14 @@ describe('sync-persistence helpers', () => { expect(result.invalid).toBe(1); }); + it('maps pre-MDIP signed timestamp to MDIP epoch seconds', () => { + const legacyEpoch = makeCreateOp('a', '1971-01-01T00:00:00.000Z'); + const result = mapAcceptedOperationsToSyncRecords([legacyEpoch]); + expect(result.records.length).toBe(1); + expect(result.records[0].ts).toBe(MDIP_EPOCH_SECONDS); + expect(result.invalid).toBe(0); + }); + it('maps unix epoch signed timestamp to MDIP epoch seconds', () => { const legacyEpoch = makeCreateOp('a', '1970-01-01T00:00:00.000Z'); const result = mapAcceptedOperationsToSyncRecords([legacyEpoch]); From 301577f8021da6c0f99e90e934a80fc87c7529ee Mon Sep 17 00:00:00 2001 From: Bushstar Date: Mon, 23 Feb 2026 15:12:01 +0000 Subject: [PATCH 26/31] add hypr config tests --- tests/hyperswarm/config.test.ts | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/hyperswarm/config.test.ts b/tests/hyperswarm/config.test.ts index bfd2ed806..8747a9309 100644 --- a/tests/hyperswarm/config.test.ts +++ b/tests/hyperswarm/config.test.ts @@ -39,4 +39,47 @@ describe('hyperswarm config', () => { expect(config.negentropyEnabled).toBe(false); expect(config.legacySyncEnabled).toBe(true); }); + + it('uses defaults when optional env vars are empty', async () => { + process.env.KC_HYPR_EXPORT_INTERVAL = ''; + process.env.KC_HYPR_NEGENTROPY_ENABLE = ''; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = ''; + + const config = await importConfigIsolated(); + + expect(config.exportInterval).toBe(2); + expect(config.negentropyEnabled).toBe(true); + expect(config.legacySyncEnabled).toBe(true); + }); + + it('throws on invalid positive integer env values', async () => { + process.env.KC_HYPR_EXPORT_INTERVAL = '0'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('Invalid KC_HYPR_EXPORT_INTERVAL; expected a positive integer'); + }); + + it('throws when frame size limit is non-zero and below minimum', async () => { + process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '1024'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4096'); + }); + + it('throws on invalid boolean env values', async () => { + process.env.KC_HYPR_NEGENTROPY_ENABLE = 'maybe'; + process.env.KC_HYPR_LEGACY_SYNC_ENABLE = 'true'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('Invalid KC_HYPR_NEGENTROPY_ENABLE; expected true or false'); + }); }); From e7b09e17d89b738a4ae2025b93c5685df40d8815 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Tue, 24 Feb 2026 09:51:31 +0000 Subject: [PATCH 27/31] full sync on connect --- docker-compose.yml | 5 +- sample.env | 7 +-- services/mediators/hyperswarm/README.md | 13 ++--- services/mediators/hyperswarm/src/config.js | 13 ++--- .../hyperswarm/src/hyperswarm-mediator.ts | 55 ++++++++----------- .../hyperswarm/src/negentropy/policy.ts | 11 +++- tests/cli-tests/generate_test_env.sh | 5 +- tests/hyperswarm/config.test.ts | 11 +++- tests/hyperswarm/negentropy-policy.test.ts | 6 +- 9 files changed, 63 insertions(+), 63 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 96d87e606..633eb6469 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,11 +102,10 @@ services: - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} - KC_HYPR_NEGENTROPY_ENABLE=${KC_HYPR_NEGENTROPY_ENABLE} - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} - - KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS} - - KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS} + - KC_HYPR_NEGENTROPY_WINDOW_DAYS=${KC_HYPR_NEGENTROPY_WINDOW_DAYS} - KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=${KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW} - KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=${KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION} - - KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=${KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS} + - KC_HYPR_NEGENTROPY_INTERVAL=${KC_HYPR_NEGENTROPY_INTERVAL} - KC_HYPR_LEGACY_SYNC_ENABLE=${KC_HYPR_LEGACY_SYNC_ENABLE} - KC_LOG_LEVEL=${KC_LOG_LEVEL} volumes: diff --git a/sample.env b/sample.env index 74869b4e2..c68ce785f 100644 --- a/sample.env +++ b/sample.env @@ -39,10 +39,9 @@ KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync for peers without negentropy. true|false. KC_HYPR_NEGENTROPY_ENABLE=true # Enables negentropy sync protocol. true|false. -KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 # Seconds between periodic anti-entropy repair passes. integer >= 1. -KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 # Recent-first reconciliation window size in days. integer >= 1. -KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 # Older reconciliation window size in days. integer >= 1. -KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=131072 # Negentropy frame byte limit. valid values are 0 (unlimited) or >= 4096. +KC_HYPR_NEGENTROPY_INTERVAL=300 # Seconds between retry attempts for peers not yet fully synced. integer >= 1. +KC_HYPR_NEGENTROPY_WINDOW_DAYS=30 # Reconciliation window size in days for full-sync chunking. integer >= 1. +KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=128 # Negentropy frame limit in KB. valid values are 0 (unlimited) or >= 4. KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 # Maximum records loaded per reconciliation window. integer >= 1. KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 # Maximum negentropy rounds per sync session. integer >= 1. diff --git a/services/mediators/hyperswarm/README.md b/services/mediators/hyperswarm/README.md index 33e230e3a..0e187c106 100644 --- a/services/mediators/hyperswarm/README.md +++ b/services/mediators/hyperswarm/README.md @@ -4,7 +4,7 @@ The Hyperswarm mediator is responsible for distributing unconfirmed MDIP operati The mediator supports two synchronization modes: -- `negentropy` mode (preferred): connect-time catch-up and periodic anti-entropy repair using `neg_open`/`neg_msg`/`ops_req`/`ops_push`/`neg_close`. +- `negentropy` mode (preferred): full-history windowed sync on connect, with periodic retry only until the peer reaches a completed sync, using `neg_open`/`neg_msg`/`ops_req`/`ops_push`/`neg_close`. - `legacy` mode (compatibility): classic `sync` -> full-history `batch` transfer (`shareDb`). Realtime propagation is always handled by the Gatekeeper queue gossip path: @@ -12,13 +12,13 @@ Realtime propagation is always handled by the Gatekeeper queue gossip path: - relays queue operations with a `queue` message - peers import and further relay `queue` messages -This keeps low latency for new operations while negentropy handles catch-up/repair. +This keeps low latency for new operations while negentropy handles catch-up. ## Sync mode behavior | peer mode | connect-time behavior | periodic behavior | queue gossip | | --- | --- | --- | --- | -| `negentropy` | negotiate + run negentropy session | periodic anti-entropy repair sessions | enabled | +| `negentropy` | negotiate + run full-history windowed session | periodic retry until sync completes, then stop | enabled | | `legacy` | `sync` + `shareDb` full-history export | n/a | enabled | `shareDb` is intentionally retained for backward compatibility and can be disabled once compatibility validation is complete. @@ -46,12 +46,11 @@ The mediator emits periodic structured sync metrics in `connectionLoop` includin | `KC_NODE_NAME` | anon | Human-readable name for the node | | `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | | `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | -| `KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT` | 0 | Negentropy frame-size limit (0 or >= 4096) | -| `KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS` | 7 | First reconciliation window size in days (recent-first) | -| `KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS` | 30 | Older reconciliation window size in days | +| `KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT` | 0 | Negentropy frame-size limit in KB (0 or >= 4) | +| `KC_HYPR_NEGENTROPY_WINDOW_DAYS` | 30 | Reconciliation window size in days for full-sync chunking | | `KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW` | 25000 | Maximum operations loaded into a single window adapter | | `KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION` | 64 | Maximum negentropy rounds per window session | -| `KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS` | 300 | Seconds between periodic negentropy repair attempts per peer | +| `KC_HYPR_NEGENTROPY_INTERVAL` | 300 | Seconds between retry attempts for peers not yet fully synced | | `KC_HYPR_LEGACY_SYNC_ENABLE` | true | Allow legacy `sync`/`shareDb` compatibility path | | `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index e8cb6332b..f7659a22e 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -19,13 +19,13 @@ function parsePositiveIntEnv(varName, defaultValue, options = {}) { } function parseFrameSizeLimit() { - const value = parsePositiveIntEnv('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT', 0, { allowZero: true }); + const valueKb = parsePositiveIntEnv('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT', 0, { allowZero: true }); - if (value > 0 && value < 4096) { - throw new Error('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4096'); + if (valueKb > 0 && valueKb < 4) { + throw new Error('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4 (KB)'); } - return value; + return valueKb * 1024; } function parseBooleanEnv(varName, defaultValue) { @@ -58,11 +58,10 @@ const config = { exportInterval: parsePositiveIntEnv('KC_HYPR_EXPORT_INTERVAL', 2), negentropyEnabled: parseBooleanEnv('KC_HYPR_NEGENTROPY_ENABLE', true), negentropyFrameSizeLimit: parseFrameSizeLimit(), - negentropyRecentWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS', 7), - negentropyOlderWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS', 30), + negentropyWindowDays: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_WINDOW_DAYS', 30), negentropyMaxRecordsPerWindow: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW', 25000), negentropyMaxRoundsPerSession: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION', 64), - negentropyRepairIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS', 300), + negentropyIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_INTERVAL', 300), legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), }; diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index e6e3c39d4..e8457e3cc 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -95,7 +95,6 @@ interface NegOpenMessage extends HyperMessageBase { maxRecords: number; order: number; }; - fullRepair: boolean; round: number; frame: NegentropyFrame; } @@ -168,7 +167,8 @@ interface ConnectionInfo { capabilities: NegotiatedPeerCapabilities; syncMode: SyncMode | 'unknown'; syncStarted: boolean; - lastNegentropyRepairAt: number; + lastNegentropyAttemptAt: number; + negentropySynced: boolean; } interface PeerSyncSession { @@ -176,7 +176,6 @@ interface PeerSyncSession { peerKey: string; mode: SyncMode; initiator: boolean; - fullRepair: boolean; windows: ReconciliationWindow[]; windowIndex: number; windowId: string | null; @@ -250,7 +249,7 @@ const NEG_MAX_IDS_PER_OPS_REQ = 1_000; const NEG_MAX_IDS_PER_LOOKUP = 1_000; const NEG_MAX_OPS_PER_PUSH = 256; const NEG_MAX_BYTES_PER_PUSH = 512 * 1024; -const NEG_REPAIR_INTERVAL_MS = config.negentropyRepairIntervalSeconds * 1000; +const NEG_REPAIR_INTERVAL_MS = config.negentropyIntervalSeconds * 1000; const NEG_ADAPTER_MAX_AGE_MS = 60 * 1000; const connectionInfo: Record = {}; @@ -376,7 +375,8 @@ function addConnection(conn: HyperswarmConnection): void { }, syncMode: 'unknown', syncStarted: false, - lastNegentropyRepairAt: 0, + lastNegentropyAttemptAt: 0, + negentropySynced: false, }; const peerNames = Object.values(connectionInfo).map(info => info.peerName); @@ -457,7 +457,6 @@ function createPeerSession(peerKey: string, mode: SyncMode, initiator: boolean, peerKey, mode, initiator, - fullRepair: false, windows: [], windowIndex: 0, windowId: null, @@ -477,7 +476,7 @@ function createPeerSession(peerKey: string, mode: SyncMode, initiator: boolean, connectionInfo[peerKey].syncStarted = true; if (mode === 'negentropy') { syncStats.negentropySessionsStarted += 1; - connectionInfo[peerKey].lastNegentropyRepairAt = now; + connectionInfo[peerKey].lastNegentropyAttemptAt = now; } return session; } @@ -500,11 +499,13 @@ function closePeerSession(peerKey: string, reason: string): void { addAggregateSample(syncStats.syncDurationMs, Date.now() - session.startedAt); const conn = connectionInfo[peerKey]; if (conn && session.mode === 'negentropy') { - conn.lastNegentropyRepairAt = Date.now(); + conn.lastNegentropyAttemptAt = Date.now(); syncStats.negentropySessionsClosed += 1; - if (reason === 'complete' || reason === 'remote_closed') { + if (reason === 'complete') { + conn.negentropySynced = true; syncStats.negentropySessionsCompleted += 1; } else { + conn.negentropySynced = false; syncStats.negentropySessionsFailed += 1; } } @@ -628,6 +629,13 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic const hasActiveSession = peerSessions.has(peerKey); const activeNegentropySessions = getActiveNegentropySessions(); + conn.syncMode = 'negentropy'; + conn.syncStarted = true; + + if (conn.negentropySynced) { + return; + } + const shouldStart = source === 'connect' ? shouldStartConnectTimeNegentropy(mode, hasActiveSession, initiator) : shouldSchedulePeriodicRepair({ @@ -635,15 +643,13 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic hasActiveSession, importQueueLength: importQueue.length(), activeNegentropySessions, - lastRepairAtMs: conn.lastNegentropyRepairAt, + lastAttemptAtMs: conn.lastNegentropyAttemptAt, nowMs: Date.now(), repairIntervalMs: NEG_REPAIR_INTERVAL_MS, isInitiator: initiator, + syncCompleted: conn.negentropySynced, }); - conn.syncMode = 'negentropy'; - conn.syncStarted = true; - if (!shouldStart) { return; } @@ -653,7 +659,6 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic } const session = createPeerSession(peerKey, 'negentropy', initiator); - session.fullRepair = source === 'periodic'; session.windows = await planRuntimeWindows(); session.windowIndex = 0; session.completedWindows = []; @@ -665,7 +670,6 @@ async function maybeStartPeerSync(peerKey: string, source: 'connect' | 'periodic initiator, sessionId: session.sessionId, source, - fullRepair: session.fullRepair, plannedWindows: session.windows.length, }, 'peer sync mode selected' @@ -855,16 +859,7 @@ function finalizeCurrentWindowStats( } function shouldAdvanceToOlderWindow(session: PeerSyncSession): boolean { - const hasMoreWindows = session.windowIndex + 1 < session.windows.length; - if (!hasMoreWindows) { - return false; - } - - if (session.fullRepair) { - return true; - } - - return session.currentWindowStats?.cappedByRecords === true; + return session.windowIndex + 1 < session.windows.length; } async function planRuntimeWindows(): Promise { @@ -1012,7 +1007,6 @@ async function startNextNegentropyWindow(peerKey: string, session: PeerSyncSessi maxRecords: window.maxRecords, order: window.order, }, - fullRepair: session.fullRepair, round: session.rounds, frame: encodeNegentropyFrame(firstFrame), }; @@ -1028,7 +1022,6 @@ async function startNextNegentropyWindow(peerKey: string, session: PeerSyncSessi sessionId: session.sessionId, windowId, window: windowLabel(window), - fullRepair: session.fullRepair, }, 'negentropy window open sent' ); @@ -1754,7 +1747,6 @@ async function receiveMsg(peerKey: string, json: Buffer | string): Promise } session.initiator = false; - session.fullRepair = msg.fullRepair === true; session.maxRounds = config.negentropyMaxRoundsPerSession; const existingIndex = session.windows.findIndex(existingWindow => makeWindowId(existingWindow) === msg.windowId); if (existingIndex >= 0) { @@ -2067,8 +2059,8 @@ async function initNegentropyAdapter(): Promise { negentropyAdapter = await NegentropyAdapter.create({ syncStore, frameSizeLimit: config.negentropyFrameSizeLimit, - recentWindowDays: config.negentropyRecentWindowDays, - olderWindowDays: config.negentropyOlderWindowDays, + recentWindowDays: config.negentropyWindowDays, + olderWindowDays: config.negentropyWindowDays, maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, maxRoundsPerSession: config.negentropyMaxRoundsPerSession, deferInitialBuild: true, @@ -2083,8 +2075,7 @@ async function initNegentropyAdapter(): Promise { log.info( { stats: negentropyAdapter.getStats(), - recentWindowDays: config.negentropyRecentWindowDays, - olderWindowDays: config.negentropyOlderWindowDays, + windowDays: config.negentropyWindowDays, maxRecordsPerWindow: config.negentropyMaxRecordsPerWindow, maxRoundsPerSession: config.negentropyMaxRoundsPerSession, frameSizeLimit: config.negentropyFrameSizeLimit, diff --git a/services/mediators/hyperswarm/src/negentropy/policy.ts b/services/mediators/hyperswarm/src/negentropy/policy.ts index 1fab6be0f..f9d7a1f72 100644 --- a/services/mediators/hyperswarm/src/negentropy/policy.ts +++ b/services/mediators/hyperswarm/src/negentropy/policy.ts @@ -5,10 +5,11 @@ export interface RepairSchedulingInput { hasActiveSession: boolean; importQueueLength: number; activeNegentropySessions: number; - lastRepairAtMs: number; + lastAttemptAtMs: number; nowMs: number; repairIntervalMs: number; isInitiator: boolean; + syncCompleted: boolean; } export function shouldAcceptLegacySync( @@ -51,9 +52,13 @@ export function shouldSchedulePeriodicRepair(input: RepairSchedulingInput): bool return false; } - if (input.lastRepairAtMs <= 0) { + if (input.syncCompleted) { + return false; + } + + if (input.lastAttemptAtMs <= 0) { return true; } - return (input.nowMs - input.lastRepairAtMs) >= input.repairIntervalMs; + return (input.nowMs - input.lastAttemptAtMs) >= input.repairIntervalMs; } diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index bc685720e..1b3e3ae12 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -49,11 +49,10 @@ KC_KEYMASTER_URL=http://localhost:4226 KC_HYPR_EXPORT_INTERVAL=2 KC_HYPR_NEGENTROPY_ENABLE=true KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=0 -KC_HYPR_NEGENTROPY_RECENT_WINDOW_DAYS=7 -KC_HYPR_NEGENTROPY_OLDER_WINDOW_DAYS=30 +KC_HYPR_NEGENTROPY_WINDOW_DAYS=30 KC_HYPR_NEGENTROPY_MAX_RECORDS_PER_WINDOW=25000 KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION=64 -KC_HYPR_NEGENTROPY_REPAIR_INTERVAL_SECONDS=300 +KC_HYPR_NEGENTROPY_INTERVAL=300 KC_HYPR_LEGACY_SYNC_ENABLE=true KC_MDIP_PROTOCOL=/MDIP/testing diff --git a/tests/hyperswarm/config.test.ts b/tests/hyperswarm/config.test.ts index 8747a9309..52aa8769b 100644 --- a/tests/hyperswarm/config.test.ts +++ b/tests/hyperswarm/config.test.ts @@ -63,13 +63,20 @@ describe('hyperswarm config', () => { }); it('throws when frame size limit is non-zero and below minimum', async () => { - process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '1024'; + process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '2'; await expect( jest.isolateModulesAsync(async () => { await import(CONFIG_PATH); }) - ).rejects.toThrow('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4096'); + ).rejects.toThrow('KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT must be 0 or >= 4 (KB)'); + }); + + it('interprets frame size limit env value as KB', async () => { + process.env.KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT = '4'; + + const config = await importConfigIsolated(); + expect(config.negentropyFrameSizeLimit).toBe(4096); }); it('throws on invalid boolean env values', async () => { diff --git a/tests/hyperswarm/negentropy-policy.test.ts b/tests/hyperswarm/negentropy-policy.test.ts index 46db85cfa..8e8d85a9f 100644 --- a/tests/hyperswarm/negentropy-policy.test.ts +++ b/tests/hyperswarm/negentropy-policy.test.ts @@ -25,10 +25,11 @@ describe('negentropy sync policy', () => { hasActiveSession: false, importQueueLength: 0, activeNegentropySessions: 0, - lastRepairAtMs: 0, + lastAttemptAtMs: 0, nowMs: 1_000_000, repairIntervalMs: 300_000, isInitiator: true, + syncCompleted: false, }; expect(shouldSchedulePeriodicRepair(base)).toBe(true); @@ -37,9 +38,10 @@ describe('negentropy sync policy', () => { expect(shouldSchedulePeriodicRepair({ ...base, importQueueLength: 1 })).toBe(false); expect(shouldSchedulePeriodicRepair({ ...base, activeNegentropySessions: 1 })).toBe(false); expect(shouldSchedulePeriodicRepair({ ...base, syncMode: 'legacy' })).toBe(false); + expect(shouldSchedulePeriodicRepair({ ...base, syncCompleted: true })).toBe(false); expect(shouldSchedulePeriodicRepair({ ...base, - lastRepairAtMs: base.nowMs - 10_000, + lastAttemptAtMs: base.nowMs - 10_000, })).toBe(false); }); }); From 271a538a285a6f5ef3b2a86751d4c355cea3e572 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Tue, 24 Feb 2026 10:16:50 +0000 Subject: [PATCH 28/31] resync from gatekeeper when drift over one percent --- .../mediators/hyperswarm/src/bootstrap.ts | 55 ++++++++++++++----- tests/hyperswarm/bootstrap.test.ts | 38 +++++++++++-- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/services/mediators/hyperswarm/src/bootstrap.ts b/services/mediators/hyperswarm/src/bootstrap.ts index 7f78c0c87..8b91c063a 100644 --- a/services/mediators/hyperswarm/src/bootstrap.ts +++ b/services/mediators/hyperswarm/src/bootstrap.ts @@ -2,50 +2,77 @@ import type { GatekeeperEvent } from '@mdip/gatekeeper/types'; import type { OperationSyncStore } from './db/types.js'; import { mapAcceptedOperationsToSyncRecords } from './sync-persistence.js'; +const DEFAULT_DRIFT_THRESHOLD_PCT = 0.01; // 1% + export interface BootstrapGatekeeper { exportBatch(dids?: string[]): Promise; } export interface BootstrapResult { skipped: boolean; - reason?: 'store_not_empty'; + reason?: 'store_within_drift_tolerance'; countBefore: number; countAfter: number; exported: number; mapped: number; invalid: number; inserted: number; + driftPct: number; + driftThresholdPct: number; durationMs: number; } +export interface BootstrapOptions { + driftThresholdPct?: number; +} + +function assertValidDriftThreshold(value: number): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error('Invalid driftThresholdPct; expected a number between 0 and 1'); + } +} + export async function bootstrapSyncStoreIfEmpty( syncStore: OperationSyncStore, gatekeeper: BootstrapGatekeeper, + options: BootstrapOptions = {}, ): Promise { + const driftThresholdPct = options.driftThresholdPct ?? DEFAULT_DRIFT_THRESHOLD_PCT; + assertValidDriftThreshold(driftThresholdPct); + const startedAt = Date.now(); const countBefore = await syncStore.count(); - if (countBefore > 0) { + const exportedEvents = await gatekeeper.exportBatch(); + const operations = exportedEvents + .map(event => event.operation) + .filter((operation): operation is NonNullable => !!operation); + + const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); + const canonicalCount = records.length; + const driftPct = Math.abs(countBefore - canonicalCount) / Math.max(canonicalCount, 1); + + if (countBefore > 0 && driftPct < driftThresholdPct) { return { skipped: true, - reason: 'store_not_empty', + reason: 'store_within_drift_tolerance', countBefore, countAfter: countBefore, - exported: 0, - mapped: 0, - invalid: 0, + exported: operations.length, + mapped: canonicalCount, + invalid, inserted: 0, + driftPct, + driftThresholdPct, durationMs: Date.now() - startedAt, }; } - const exportedEvents = await gatekeeper.exportBatch(); - const operations = exportedEvents - .map(event => event.operation) - .filter((operation): operation is NonNullable => !!operation); + if (countBefore > 0) { + await syncStore.reset(); + } - const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); - const inserted = records.length > 0 ? await syncStore.upsertMany(records) : 0; + const inserted = canonicalCount > 0 ? await syncStore.upsertMany(records) : 0; const countAfter = await syncStore.count(); return { @@ -53,9 +80,11 @@ export async function bootstrapSyncStoreIfEmpty( countBefore, countAfter, exported: operations.length, - mapped: records.length, + mapped: canonicalCount, invalid, inserted, + driftPct, + driftThresholdPct, durationMs: Date.now() - startedAt, }; } diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index bdc39971c..9131b1fea 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -25,27 +25,53 @@ function makeEvent(operation: Operation): GatekeeperEvent { } describe('bootstrapSyncStoreIfEmpty', () => { - it('skips bootstrap when store is already populated', async () => { + it('skips rebuild when store drift is within configured percentage tolerance', async () => { const store = new InMemoryOperationSyncStore(); await store.start(); + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); await store.upsertMany([{ id: h('a'), // eslint-disable-next-line sonarjs/no-duplicate-string ts: Math.floor(Date.parse('2026-02-10T10:00:00.000Z') / 1000), - operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), + operation: opA, }]); const gatekeeper = { - exportBatch: jest.fn(async () => []), + exportBatch: jest.fn(async () => [makeEvent(opA)]), }; const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); expect(result.skipped).toBe(true); - expect(result.reason).toBe('store_not_empty'); - expect(gatekeeper.exportBatch).not.toHaveBeenCalled(); + expect(result.reason).toBe('store_within_drift_tolerance'); + expect(result.driftPct).toBe(0); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); expect(await store.count()).toBe(1); }); + it('rebuilds populated store when drift percentage exceeds threshold', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + await store.upsertMany([{ + id: h('a'), + ts: Math.floor(Date.parse('2026-02-10T10:00:00.000Z') / 1000), + operation: makeOperation('a', '2026-02-10T10:00:00.000Z'), + }]); + + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); + const gatekeeper = { + exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper); + expect(result.skipped).toBe(false); + expect(result.countBefore).toBe(1); + expect(result.countAfter).toBe(2); + expect(result.driftPct).toBeGreaterThan(result.driftThresholdPct); + expect(result.inserted).toBe(2); + expect(await store.count()).toBe(2); + }); + it('bootstraps from gatekeeper exportBatch when store is empty', async () => { const store = new InMemoryOperationSyncStore(); await store.start(); @@ -63,6 +89,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.invalid).toBe(0); expect(result.inserted).toBe(2); expect(result.countAfter).toBe(2); + expect(result.driftPct).toBe(1); expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); expect(await store.count()).toBe(2); }); @@ -95,6 +122,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.invalid).toBe(0); expect(result.inserted).toBe(0); expect(result.countAfter).toBe(0); + expect(result.driftPct).toBe(0); expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); }); }); From 003bf81c50bf33f747aeb309608f3c7a3b26cfd0 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Tue, 24 Feb 2026 10:59:03 +0000 Subject: [PATCH 29/31] fix lint error --- tests/hyperswarm/bootstrap.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 9131b1fea..25b002319 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -28,6 +28,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { it('skips rebuild when store drift is within configured percentage tolerance', async () => { const store = new InMemoryOperationSyncStore(); await store.start(); + // eslint-disable-next-line sonarjs/no-duplicate-string const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); await store.upsertMany([{ id: h('a'), From 9a000a3bf84355def6834c2387bce2b9b3036b62 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 25 Feb 2026 10:24:56 +0000 Subject: [PATCH 30/31] chunk bootstrap --- .../mediators/hyperswarm/src/bootstrap.ts | 139 ++++++++++++++++-- test.sh | 12 ++ tests/hyperswarm/bootstrap.test.ts | 38 +++++ 3 files changed, 177 insertions(+), 12 deletions(-) create mode 100755 test.sh diff --git a/services/mediators/hyperswarm/src/bootstrap.ts b/services/mediators/hyperswarm/src/bootstrap.ts index 8b91c063a..823f95c82 100644 --- a/services/mediators/hyperswarm/src/bootstrap.ts +++ b/services/mediators/hyperswarm/src/bootstrap.ts @@ -3,8 +3,10 @@ import type { OperationSyncStore } from './db/types.js'; import { mapAcceptedOperationsToSyncRecords } from './sync-persistence.js'; const DEFAULT_DRIFT_THRESHOLD_PCT = 0.01; // 1% +const BOOTSTRAP_DID_BATCH_SIZE = 500; export interface BootstrapGatekeeper { + getDIDs(): Promise; exportBatch(dids?: string[]): Promise; } @@ -32,6 +34,101 @@ function assertValidDriftThreshold(value: number): void { } } +interface BootstrapBatchTotals { + exported: number; + mapped: number; + invalid: number; + canonicalCount: number; +} + +interface BootstrapImportTotals extends BootstrapBatchTotals { + inserted: number; +} + +function toOperations(events: GatekeeperEvent[]): NonNullable[] { + return events + .map(event => event.operation) + .filter((operation): operation is NonNullable => !!operation); +} + +async function exportBatchForDids( + gatekeeper: BootstrapGatekeeper, + dids: string[], + batchIndex: number, + batchCount: number, +): Promise[]> { + try { + const events = await gatekeeper.exportBatch(dids); + return toOperations(events); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `bootstrap exportBatch failed for DID batch ${batchIndex + 1}/${batchCount} (${dids.length} dids): ${message}` + ); + } +} + +async function scanGatekeeperBatches( + gatekeeper: BootstrapGatekeeper, + dids: string[], +): Promise { + let exported = 0; + let mapped = 0; + let invalid = 0; + const batchCount = dids.length === 0 ? 0 : Math.ceil(dids.length / BOOTSTRAP_DID_BATCH_SIZE); + + for (let offset = 0, batchIndex = 0; offset < dids.length; offset += BOOTSTRAP_DID_BATCH_SIZE, batchIndex += 1) { + const didBatch = dids.slice(offset, offset + BOOTSTRAP_DID_BATCH_SIZE); + const operations = await exportBatchForDids(gatekeeper, didBatch, batchIndex, batchCount); + exported += operations.length; + + const { records, invalid: invalidBatch } = mapAcceptedOperationsToSyncRecords(operations); + mapped += records.length; + invalid += invalidBatch; + } + + return { + exported, + mapped, + invalid, + canonicalCount: mapped, + }; +} + +async function importGatekeeperBatches( + gatekeeper: BootstrapGatekeeper, + syncStore: OperationSyncStore, + dids: string[], +): Promise { + let exported = 0; + let mapped = 0; + let invalid = 0; + let inserted = 0; + const batchCount = dids.length === 0 ? 0 : Math.ceil(dids.length / BOOTSTRAP_DID_BATCH_SIZE); + + for (let offset = 0, batchIndex = 0; offset < dids.length; offset += BOOTSTRAP_DID_BATCH_SIZE, batchIndex += 1) { + const didBatch = dids.slice(offset, offset + BOOTSTRAP_DID_BATCH_SIZE); + const operations = await exportBatchForDids(gatekeeper, didBatch, batchIndex, batchCount); + exported += operations.length; + + const { records, invalid: invalidBatch } = mapAcceptedOperationsToSyncRecords(operations); + mapped += records.length; + invalid += invalidBatch; + + if (records.length > 0) { + inserted += await syncStore.upsertMany(records); + } + } + + return { + exported, + mapped, + invalid, + inserted, + canonicalCount: mapped, + }; +} + export async function bootstrapSyncStoreIfEmpty( syncStore: OperationSyncStore, gatekeeper: BootstrapGatekeeper, @@ -42,14 +139,29 @@ export async function bootstrapSyncStoreIfEmpty( const startedAt = Date.now(); const countBefore = await syncStore.count(); + const dids = await gatekeeper.getDIDs(); - const exportedEvents = await gatekeeper.exportBatch(); - const operations = exportedEvents - .map(event => event.operation) - .filter((operation): operation is NonNullable => !!operation); + let exported = 0; + let mapped = 0; + let invalid = 0; + let inserted = 0; + let canonicalCount = 0; + + if (countBefore > 0) { + const scanned = await scanGatekeeperBatches(gatekeeper, dids); + exported = scanned.exported; + mapped = scanned.mapped; + invalid = scanned.invalid; + canonicalCount = scanned.canonicalCount; + } else { + const imported = await importGatekeeperBatches(gatekeeper, syncStore, dids); + exported = imported.exported; + mapped = imported.mapped; + invalid = imported.invalid; + inserted = imported.inserted; + canonicalCount = imported.canonicalCount; + } - const { records, invalid } = mapAcceptedOperationsToSyncRecords(operations); - const canonicalCount = records.length; const driftPct = Math.abs(countBefore - canonicalCount) / Math.max(canonicalCount, 1); if (countBefore > 0 && driftPct < driftThresholdPct) { @@ -58,8 +170,8 @@ export async function bootstrapSyncStoreIfEmpty( reason: 'store_within_drift_tolerance', countBefore, countAfter: countBefore, - exported: operations.length, - mapped: canonicalCount, + exported, + mapped, invalid, inserted: 0, driftPct, @@ -70,17 +182,20 @@ export async function bootstrapSyncStoreIfEmpty( if (countBefore > 0) { await syncStore.reset(); + const imported = await importGatekeeperBatches(gatekeeper, syncStore, dids); + exported = imported.exported; + mapped = imported.mapped; + invalid = imported.invalid; + inserted = imported.inserted; } - - const inserted = canonicalCount > 0 ? await syncStore.upsertMany(records) : 0; const countAfter = await syncStore.count(); return { skipped: false, countBefore, countAfter, - exported: operations.length, - mapped: canonicalCount, + exported, + mapped, invalid, inserted, driftPct, diff --git a/test.sh b/test.sh new file mode 100755 index 000000000..b06ab961f --- /dev/null +++ b/test.sh @@ -0,0 +1,12 @@ +SERVICE=hypr-mediator +CID=$(docker compose ps -q "$SERVICE") +if [ -z "$CID" ]; then + echo "No running container for service: $SERVICE" + docker compose ps + exit 1 +fi + +while :; do + echo "$(date -Is),$(docker stats --no-stream --format '{{.MemUsage}}' "$CID")" + sleep 1 +done | tee /tmp/${SERVICE}-mem.csv diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 25b002319..8d4406bdc 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -24,6 +24,10 @@ function makeEvent(operation: Operation): GatekeeperEvent { }; } +function makeDid(index: number): string { + return `did:test:${index}`; +} + describe('bootstrapSyncStoreIfEmpty', () => { it('skips rebuild when store drift is within configured percentage tolerance', async () => { const store = new InMemoryOperationSyncStore(); @@ -38,6 +42,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { }]); const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), exportBatch: jest.fn(async () => [makeEvent(opA)]), }; @@ -45,7 +50,9 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.skipped).toBe(true); expect(result.reason).toBe('store_within_drift_tolerance'); expect(result.driftPct).toBe(0); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledWith([makeDid(1)]); expect(await store.count()).toBe(1); }); @@ -61,6 +68,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1), makeDid(2)]), exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), }; @@ -70,6 +78,8 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.countAfter).toBe(2); expect(result.driftPct).toBeGreaterThan(result.driftThresholdPct); expect(result.inserted).toBe(2); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(2); expect(await store.count()).toBe(2); }); @@ -80,6 +90,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1), makeDid(2)]), exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), }; @@ -91,7 +102,9 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.inserted).toBe(2); expect(result.countAfter).toBe(2); expect(result.driftPct).toBe(1); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledWith([makeDid(1), makeDid(2)]); expect(await store.count()).toBe(2); }); @@ -100,6 +113,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { await store.start(); const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), exportBatch: jest.fn(async () => { throw new Error('boom'); }), @@ -113,6 +127,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { await store.start(); const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), exportBatch: jest.fn(async () => [{ registry: 'hyperswarm', time: new Date().toISOString() }]), }; @@ -124,6 +139,29 @@ describe('bootstrapSyncStoreIfEmpty', () => { expect(result.inserted).toBe(0); expect(result.countAfter).toBe(0); expect(result.driftPct).toBe(0); + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); }); + + it('exports in DID batches to avoid loading the full export in one payload', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const dids = Array.from({ length: 501 }, (_, index) => makeDid(index + 1)); + const gatekeeper = { + getDIDs: jest.fn(async () => dids), + exportBatch: jest.fn(async (batchDids?: string[]) => { + return (batchDids ?? []).map((_, index) => + makeEvent(makeOperation(index % 2 === 0 ? 'a' : 'b', '2026-02-10T10:00:00.000Z')) + ); + }), + }; + + await bootstrapSyncStoreIfEmpty(store, gatekeeper); + + expect(gatekeeper.getDIDs).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(2); + expect(gatekeeper.exportBatch).toHaveBeenNthCalledWith(1, dids.slice(0, 500)); + expect(gatekeeper.exportBatch).toHaveBeenNthCalledWith(2, dids.slice(500, 501)); + }); }); From 2a09903c45e9e57afa4ec01954cbdaf742bfdd08 Mon Sep 17 00:00:00 2001 From: Bushstar Date: Wed, 25 Feb 2026 10:46:56 +0000 Subject: [PATCH 31/31] Use gatekeeper type from package --- .../mediators/hyperswarm/src/bootstrap.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/services/mediators/hyperswarm/src/bootstrap.ts b/services/mediators/hyperswarm/src/bootstrap.ts index 823f95c82..8a610205b 100644 --- a/services/mediators/hyperswarm/src/bootstrap.ts +++ b/services/mediators/hyperswarm/src/bootstrap.ts @@ -1,14 +1,11 @@ -import type { GatekeeperEvent } from '@mdip/gatekeeper/types'; +import type { GatekeeperEvent, GatekeeperInterface, MdipDocument } from '@mdip/gatekeeper/types'; import type { OperationSyncStore } from './db/types.js'; import { mapAcceptedOperationsToSyncRecords } from './sync-persistence.js'; const DEFAULT_DRIFT_THRESHOLD_PCT = 0.01; // 1% const BOOTSTRAP_DID_BATCH_SIZE = 500; -export interface BootstrapGatekeeper { - getDIDs(): Promise; - exportBatch(dids?: string[]): Promise; -} +export type BootstrapGatekeeper = Pick; export interface BootstrapResult { skipped: boolean; @@ -51,6 +48,19 @@ function toOperations(events: GatekeeperEvent[]): NonNullable => !!operation); } +function normalizeDids(input: string[] | MdipDocument[]): string[] { + const dids = input + .map(item => { + if (typeof item === 'string') { + return item; + } + return item?.didDocument?.id; + }) + .filter((did): did is string => typeof did === 'string' && did !== ''); + + return Array.from(new Set(dids)); +} + async function exportBatchForDids( gatekeeper: BootstrapGatekeeper, dids: string[], @@ -139,7 +149,7 @@ export async function bootstrapSyncStoreIfEmpty( const startedAt = Date.now(); const countBefore = await syncStore.count(); - const dids = await gatekeeper.getDIDs(); + const dids = normalizeDids(await gatekeeper.getDIDs()); let exported = 0; let mapped = 0;