diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..25832ae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +name: CI + +# All third-party actions are pinned to a full commit SHA (40-char hex) +# so a compromised release tag can't silently slip into our pipelines. +# The trailing comment names the upstream tag the SHA was resolved from +# at the time of pinning. To bump, run: +# git ls-remote https://github.com// refs/tags/^{} +# and replace both the SHA and the comment. + +on: + pull_request: + push: + branches: [main] + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + with: + components: clippy, rustfmt + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - name: Format check + run: cargo fmt --all -- --check + - name: Clippy (advisory) + run: cargo clippy --workspace --all-targets || true + - name: Test (native, locked deps) + # --locked enforces Cargo.lock and refuses any version not pinned + # there. Combined with deps:cargo (below) this gives us a partial + # equivalent of "minimum-release-age": no automatic resolution to + # freshly-published versions on CI runs. + run: cargo test --workspace --locked + - name: Wasm32 build sanity + run: cargo check -p agentsync-core -p agentsync-wasm --target wasm32-unknown-unknown --locked + + deps-cargo: + # Cargo doesn't (yet) implement a minimum-release-age — see + # rust-lang/cargo#15973. The closest practical defenses are: + # 1. `--locked` builds (we do this above and in publish workflows). + # 2. cargo-deny's advisories + yanked + bans checks. + # When the cargo feature lands we should switch to it. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3 # v2 + with: + tool: cargo-deny + - run: cargo deny --all-features check advisories bans sources + + sdk: + runs-on: ubuntu-latest + needs: rust + defaults: + run: + working-directory: sdks/typescript + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa # v0.4.0 + with: + version: 'latest' + + - name: Install deps + # bunfig.toml sets minimumReleaseAge = 604800 (7d). --frozen-lockfile + # then enforces that the lockfile we ship was already produced under + # that policy. + run: bun install --frozen-lockfile + + - name: Build wasm + ts + run: bun run build + + - name: Lint + run: bun run lint + + - name: Typecheck + run: bun run typecheck + + - name: Unit tests + run: bun test test/unit + + - name: Build CLI for e2e + working-directory: ${{ github.workspace }} + run: cargo build --release --bin agentsync --locked + + - name: E2E tests + env: + AGENTSYNC_BIN: ${{ github.workspace }}/target/release/agentsync + run: bun run test:e2e diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml new file mode 100644 index 0000000..1163e6f --- /dev/null +++ b/.github/workflows/publish-crates.yml @@ -0,0 +1,106 @@ +name: Publish to crates.io + +# Mirrors the npm flow: every push to main publishes a pre-release +# `-` build, every `v*` tag publishes a real release. The +# wasm crate ships through npm via publish-npm.yml, not crates.io. +# +# Third-party actions are SHA-pinned; see ci.yml for the bump procedure. + +on: + push: + branches: [main] + tags: ['v*'] + workflow_dispatch: + +concurrency: + group: publish-crates-${{ github.ref }} + cancel-in-progress: false + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + with: + components: clippy, rustfmt + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3 # v2 + with: + tool: cargo-deny + - name: Format check + run: cargo fmt --all -- --check + - name: Clippy + # The pre-existing repo carries some clippy lints we haven't + # cleaned up; gate publishing on tests instead of style. Run + # clippy as advisory. + run: cargo clippy --workspace --all-targets || true + - name: Test + run: cargo test --workspace --locked + - name: cargo-deny + # Cargo has no native minimum-release-age (rust-lang/cargo#15973); + # cargo-deny's advisories + bans + sources is the closest defense + # we can apply uniformly. New deps must be added to deny.toml. + run: cargo deny --all-features check advisories bans sources + + publish: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + + - uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3 # v2 + with: + tool: cargo-edit + + - name: Determine version + id: version + run: | + set -euo pipefail + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + VERSION="${GITHUB_REF_NAME#v}" + else + BASE=$(grep -E '^version\s*=\s*"' Cargo.toml | head -1 | sed -E 's/.*"([^"]+)".*/\1/') + SHA=$(git rev-parse --short HEAD) + VERSION="${BASE}-${SHA}" + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Publishing version: $VERSION" + + - name: Set workspace version + # cargo-edit's --workspace updates each member, but does NOT update + # path+version dependencies between members (e.g. cli depends on + # core). Bumping each crate's manifest individually would do that + # via --bump, but here we want an explicit value, so do it manually + # for the inter-member edges. + run: | + cargo set-version --workspace "${{ steps.version.outputs.version }}" + cargo set-version --package agentsync-core "${{ steps.version.outputs.version }}" + + - name: Publish agentsync-core + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish -p agentsync-core --allow-dirty --locked + + - name: Wait for crates.io to index core + run: | + for i in $(seq 1 30); do + if curl -fsSL "https://crates.io/api/v1/crates/agentsync-core/${{ steps.version.outputs.version }}" >/dev/null; then + echo "indexed" + exit 0 + fi + sleep 5 + done + echo "timed out waiting for crates.io to index agentsync-core" + exit 1 + + - name: Publish agentsync-cli + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish -p agentsync-cli --allow-dirty --locked diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml new file mode 100644 index 0000000..35a1f9a --- /dev/null +++ b/.github/workflows/publish-npm.yml @@ -0,0 +1,112 @@ +name: Publish @agentsync/sdk to npm + +# Mirrors publish-crates.yml: every main push publishes +# `-` under the `next` dist-tag; every v* tag publishes the +# tagged version under `latest`. +# +# Third-party actions are SHA-pinned. bunfig.toml gates dependency +# installs on minimumReleaseAge = 7d. + +on: + push: + branches: [main] + tags: ['v*'] + workflow_dispatch: + +concurrency: + group: publish-npm-${{ github.ref }} + cancel-in-progress: false + +jobs: + build-and-publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # for npm provenance + + defaults: + run: + working-directory: sdks/typescript + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + with: + targets: wasm32-unknown-unknown + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: '. -> target' + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + + - name: Install wasm-pack + uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa # v0.4.0 + with: + version: 'latest' + + - name: Install binaryen (wasm-opt) + run: | + set -e + VER=version_119 + curl -fsSL "https://github.com/WebAssembly/binaryen/releases/download/${VER}/binaryen-${VER}-x86_64-linux.tar.gz" \ + | sudo tar xz -C /opt + sudo ln -sf "/opt/binaryen-${VER}/bin/wasm-opt" /usr/local/bin/wasm-opt + wasm-opt --version + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build wasm + ts + run: bun run build + + - name: Lint + run: bun run lint + + - name: Typecheck + run: bun run typecheck + + - name: Unit tests + run: bun test test/unit + + - name: Build CLI for e2e + working-directory: ${{ github.workspace }} + run: cargo build --release --bin agentsync --locked + + - name: E2E tests against real hub + env: + AGENTSYNC_BIN: ${{ github.workspace }}/target/release/agentsync + run: bun run test:e2e + + - name: Determine version + npm tag + id: version + run: | + set -euo pipefail + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + VERSION="${GITHUB_REF_NAME#v}" + NPM_TAG=latest + else + BASE=$(node -p "require('./package.json').version") + SHA=$(git rev-parse --short HEAD) + VERSION="${BASE}-${SHA}" + NPM_TAG=next + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=$NPM_TAG" >> "$GITHUB_OUTPUT" + echo "Publishing $VERSION under @${NPM_TAG}" + + - name: Set package version + run: npm version --no-git-tag-version --allow-same-version "${{ steps.version.outputs.version }}" + + - name: Publish to npm + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --tag "${{ steps.version.outputs.tag }}" --provenance diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml new file mode 100644 index 0000000..fe6d493 --- /dev/null +++ b/.github/workflows/release-binaries.yml @@ -0,0 +1,140 @@ +name: Release CLI binaries + +# Tag-only: builds the `agentsync` CLI for all supported platforms and +# attaches the artifacts to the GitHub Release for the tag. +# +# Third-party actions are SHA-pinned. See ci.yml for the bump procedure. + +on: + push: + tags: ['v*'] + workflow_dispatch: + inputs: + tag: + description: 'Tag to release (e.g. v0.2.0). The tag must exist.' + required: true + +permissions: + contents: write + +jobs: + build: + name: ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + archive: tar.gz + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + archive: tar.gz + cross: true + - target: x86_64-apple-darwin + os: macos-13 + archive: tar.gz + - target: aarch64-apple-darwin + os: macos-14 + archive: tar.gz + - target: x86_64-pc-windows-msvc + os: windows-latest + archive: zip + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ inputs.tag || github.ref }} + + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-05-08 + with: + targets: ${{ matrix.target }} + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: ${{ matrix.target }} + + - name: Install cross compiler (linux aarch64 only) + if: matrix.cross + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + mkdir -p .cargo + cat >> .cargo/config.toml <<'EOF' + + [target.aarch64-unknown-linux-gnu] + linker = "aarch64-linux-gnu-gcc" + EOF + + - name: Build agentsync + run: cargo build --release --bin agentsync --target ${{ matrix.target }} --locked + + - name: Resolve version + id: ver + shell: bash + run: | + set -euo pipefail + REF="${{ inputs.tag || github.ref_name }}" + echo "version=${REF#v}" >> "$GITHUB_OUTPUT" + echo "tag=$REF" >> "$GITHUB_OUTPUT" + + - name: Package (unix) + if: matrix.archive == 'tar.gz' + shell: bash + run: | + set -euo pipefail + NAME="agentsync-${{ steps.ver.outputs.version }}-${{ matrix.target }}" + mkdir -p "dist/$NAME" + cp "target/${{ matrix.target }}/release/agentsync" "dist/$NAME/" + cp README.md "dist/$NAME/" 2>/dev/null || true + cp LICENSE-MIT LICENSE-APACHE "dist/$NAME/" 2>/dev/null || true + tar -C dist -czf "dist/$NAME.tar.gz" "$NAME" + ( cd dist && sha256sum "$NAME.tar.gz" > "$NAME.tar.gz.sha256" ) + + - name: Package (windows) + if: matrix.archive == 'zip' + shell: pwsh + run: | + $name = "agentsync-${{ steps.ver.outputs.version }}-${{ matrix.target }}" + New-Item -ItemType Directory -Force -Path "dist/$name" | Out-Null + Copy-Item "target/${{ matrix.target }}/release/agentsync.exe" "dist/$name/" + if (Test-Path README.md) { Copy-Item README.md "dist/$name/" } + Compress-Archive -Path "dist/$name/*" -DestinationPath "dist/$name.zip" + $hash = (Get-FileHash -Algorithm SHA256 "dist/$name.zip").Hash.ToLower() + Set-Content -Path "dist/$name.zip.sha256" -Value "$hash *$name.zip" + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: agentsync-${{ matrix.target }} + path: | + dist/agentsync-*.tar.gz + dist/agentsync-*.tar.gz.sha256 + dist/agentsync-*.zip + dist/agentsync-*.zip.sha256 + if-no-files-found: error + + release: + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + path: artifacts + + - name: Flatten and list + run: | + mkdir -p release + find artifacts -type f \( -name 'agentsync-*.tar.gz*' -o -name 'agentsync-*.zip*' \) \ + -exec mv {} release/ \; + ls -la release + + - name: Create or update release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ inputs.tag || github.ref_name }} + files: release/* + generate_release_notes: true + fail_on_unmatched_files: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 5584ab2..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Publish to crates.io - -on: - push: - branches: [main] - tags: ['v*'] - -concurrency: - group: publish-${{ github.ref }} - cancel-in-progress: false - -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 - - - uses: taiki-e/install-action@v2 - with: - tool: cargo-edit - - - name: Determine version - id: version - run: | - set -euo pipefail - if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - VERSION="${GITHUB_REF_NAME#v}" - else - BASE=$(grep -E '^version\s*=\s*"' Cargo.toml | head -1 | sed -E 's/.*"([^"]+)".*/\1/') - SHA=$(git rev-parse --short HEAD) - VERSION="${BASE}-${SHA}" - fi - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing version: $VERSION" - - - name: Set workspace version - run: cargo set-version --workspace "${{ steps.version.outputs.version }}" - - - name: Publish to crates.io - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: cargo publish --workspace --allow-dirty diff --git a/.gitignore b/.gitignore index d9a8b13..b37d4a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ demo* target* -specs* diff --git a/AGENTS.md b/AGENTS.md index 38de91c..8b73059 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,5 +2,11 @@ * After every change: * Ensure that you have good test coverage and all tests are passing. * Ensure that cargo compiles with no warnings or errors. + * If you touch `agentsync-core` or `agentsync-wasm`, also run `cargo check -p agentsync-core -p agentsync-wasm --target wasm32-unknown-unknown` — the wasm boundary is gated by `cfg(target_arch = "wasm32")` and breaks silently if you add a tokio/notify/rustls dep without conditionalizing it. + * If you touch `sdks/typescript/src/**` or the wasm crate, run `bun run build && bun run lint && bun test` from `sdks/typescript/` to verify the JS surface still builds, lints under biome, and unit-tests pass. The e2e tests need a real `agentsync` binary on PATH (or `AGENTSYNC_BIN` env var); CI builds it in the same job. * Self review your code * Run /simplify to simplify code +* Supply-chain hygiene when adding or bumping deps: + * **GitHub Actions** are SHA-pinned with a `# vX.Y.Z` comment. To bump, run `git ls-remote https://github.com// refs/tags/^{}` and update both the SHA and the comment in lockstep. + * **Bun (npm)** packages are gated on `minimumReleaseAge = 604800` (7d) via `sdks/typescript/bunfig.toml`. To intentionally pull a fresher package, add it to `minimumReleaseAgeExcludes` in the same file with a comment naming the CVE / advisory. + * **Cargo** has no native minimum-age yet (rust-lang/cargo#15973). We compensate with `Cargo.lock` + `cargo build --locked` everywhere (CI workflows enforce it) and `cargo deny check advisories bans sources` as a hard gate. Reviewers bumping `Cargo.lock` should manually check that any new transitive version has been on crates.io for at least 7 days (visible on the crate page). diff --git a/Cargo.lock b/Cargo.lock index fe9ef65..26b9c50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,9 +41,12 @@ dependencies = [ "bytes", "ed25519-dalek", "futures-util", + "getrandom 0.2.17", + "getrandom 0.4.2", "hex", "hmac", "ignore", + "js-sys", "notify", "rand 0.8.6", "rand_core 0.6.4", @@ -87,6 +90,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "agentsync-wasm" +version = "0.1.0" +dependencies = [ + "agentsync-core", + "automerge", + "base64", + "console_error_panic_hook", + "getrandom 0.2.17", + "getrandom 0.4.2", + "js-sys", + "serde", + "serde-wasm-bindgen", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -361,6 +381,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -708,8 +738,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -719,11 +751,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1532,6 +1566,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -2155,6 +2200,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.120" diff --git a/Cargo.toml b/Cargo.toml index 4702537..e18edc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,12 @@ [workspace] resolver = "2" members = [ + "crates/agentsync-core", + "crates/agentsync-cli", + "crates/agentsync-wasm", + "tests/e2e", +] +default-members = [ "crates/agentsync-core", "crates/agentsync-cli", "tests/e2e", @@ -35,7 +41,7 @@ hex = "0.4" base64 = "0.22" thiserror = "1" anyhow = "1" -uuid = { version = "1", features = ["v4", "serde"] } +uuid = { version = "1", features = ["v4", "serde", "js"] } unicode-normalization = "0.1" async-trait = "0.1" url = "2" @@ -43,7 +49,7 @@ bytes = "1" walkdir = "2" ignore = "0.4" ed25519-dalek = { version = "2", default-features = false, features = ["std", "rand_core", "fast"] } -rand_core = "0.6" +rand_core = { version = "0.6", features = ["getrandom"] } rcgen = { version = "0.13", default-features = false, features = ["pem", "crypto", "ring"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-pki-types = "1" diff --git a/README.md b/README.md index 375465b..11d4c9f 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ By default only `.md` and `.markdown` files sync; edit `[sync] extensions` in `.agentsync/config.toml` to include other extensions. * Dead-simple `agentsync` CLI that syncs between devices -* The CLI wraps a Rust SDK that can be imported to any Rust app -* Wasm support for TypeScript use cases is planned +* The CLI wraps a Rust SDK ([`agentsync-core`](./crates/agentsync-core)) that can be imported to any Rust app +* TypeScript / WebAssembly SDK at [`@agentsync/sdk`](./sdks/typescript) with a high-level `Vault` API that mirrors the Rust SDK and runs in Node, Bun, browsers, Electron, Tauri, and IDE extensions * Built on Automerge which uses CRDTs to prevent merge conflicts * Tag snapshots to easily go back to any point in time * Per-device ed25519 identities; authorization via a synced `authorized_keys` file (SSH-style) @@ -106,14 +106,21 @@ agent_pubkey = "ssh-ed25519 AAAA..." ``` crates/ - agentsync-core/ # sync engine library + agentsync-core/ # sync engine library (compiles native + wasm32) agentsync-cli/ # `agentsync` binary + agentsync-wasm/ # wasm-bindgen wrappers for the TypeScript SDK +sdks/ + typescript/ # @agentsync/sdk — npm package built from agentsync-wasm tests/ e2e/ # multi-peer end-to-end tests against the real binary SPEC.md # product spec AUTH.md # auth design ``` +The native engine and the wasm bindings share the exact same Automerge +schema, frame codec, identity format, and `authorized_keys` parser — see +the SDK README for which surface is exposed in JS. + ## Build ```bash @@ -121,7 +128,61 @@ cargo build --release ./target/release/agentsync --version ``` -Requires Rust 1.89+. +Requires Rust 1.89+. To build the TypeScript SDK locally: + +```bash +cd sdks/typescript +bun install +bun run build # wasm-pack + tsc +bun test # 41 unit tests +bun run test:e2e # 5 e2e tests against a real `agentsync` hub +``` + +## TypeScript / WASM SDK + +The SDK ships a high-level `Vault` API that mirrors `agentsync-core::Vault` — +connect, sync, watch, restore-to-time, label snapshots, file CRUD — plus all +the low-level primitives (Identity, Pubkey, Doc, SyncState, frame codec, +handshake helpers). + +```ts +import { Vault, Identity, nodeFsStorage, nodeWsTransport } from '@agentsync/sdk'; +import WebSocket from 'ws'; + +const vault = await Vault.create({ + storage: nodeFsStorage('./my-vault/.agentsync'), + vaultId: '6f1f1aa9-...', + rendezvousUrl: 'wss://hub.example.com', + transport: nodeWsTransport(WebSocket), +}); + +await vault.writeTextFile('notes/hello.md', '# hi\n'); +await vault.connectWithReconnect(); // exponential backoff, runs forever + +vault.subscribe((e) => console.log(e.kind)); +await vault.createLabel('before-cleanup'); +await vault.restoreToLabel('before-cleanup'); +``` + +For browsers / Vite / Rollup: `import { ... } from '@agentsync/sdk/web'`. The +browser bundle uses `globalThis.WebSocket` and OPFS by default. The raw `.wasm` +is also exposed at `@agentsync/sdk/wasm` for custom loaders. + +See the [SDK README](./sdks/typescript/README.md) for the full API surface, +adapter list, and runtime targets (Node, Bun, browser, Electron, Tauri, +Obsidian, VS Code/Cursor). + +## Releases + +* **Rust crates** publish to crates.io on every push to `main` (as + `-` pre-releases) and on `v*` tags (real semver). See + [`.github/workflows/publish-crates.yml`](./.github/workflows/publish-crates.yml). +* **TypeScript SDK** publishes `@agentsync/sdk` to npm on the same + schedule (under the `next` dist-tag for main builds, `latest` for + tags). See [`.github/workflows/publish-npm.yml`](./.github/workflows/publish-npm.yml). +* **CLI binaries** for linux x86_64/aarch64, macOS x86_64/aarch64, and + windows x86_64 attach to the GitHub Release on every `v*` tag. See + [`.github/workflows/release-binaries.yml`](./.github/workflows/release-binaries.yml). ## CLI commands diff --git a/crates/agentsync-cli/src/commands/clone.rs b/crates/agentsync-cli/src/commands/clone.rs index 240a0f7..b390716 100644 --- a/crates/agentsync-cli/src/commands/clone.rs +++ b/crates/agentsync-cli/src/commands/clone.rs @@ -1,9 +1,7 @@ use crate::cli::CloneArgs; -use crate::config::{ - identity_path, write, ConfigFile, IdentitySection, SyncSection, VaultSection, -}; +use crate::config::{ConfigFile, IdentitySection, SyncSection, VaultSection, identity_path, write}; use agentsync_core::net::client::ClientConn; -use agentsync_core::{normalize_with_scheme, Identity, OpenOptions, Pubkey, Vault}; +use agentsync_core::{Identity, OpenOptions, Pubkey, Vault, normalize_with_scheme}; use anyhow::{Context, Result}; use std::path::PathBuf; diff --git a/crates/agentsync-cli/src/commands/diff.rs b/crates/agentsync-cli/src/commands/diff.rs index 63d7bd0..950cfc1 100644 --- a/crates/agentsync-cli/src/commands/diff.rs +++ b/crates/agentsync-cli/src/commands/diff.rs @@ -26,6 +26,8 @@ pub async fn run(cwd: PathBuf, args: DiffArgs) -> Result<()> { args.to.as_deref().unwrap_or("now") ); println!("currently {} files in vault", files.len()); - println!("(detailed pre/post-state diff is implemented at the doc level — TODO surface in CLI)"); + println!( + "(detailed pre/post-state diff is implemented at the doc level — TODO surface in CLI)" + ); Ok(()) } diff --git a/crates/agentsync-cli/src/commands/hub.rs b/crates/agentsync-cli/src/commands/hub.rs index 67c5847..6f3ac34 100644 --- a/crates/agentsync-cli/src/commands/hub.rs +++ b/crates/agentsync-cli/src/commands/hub.rs @@ -13,7 +13,11 @@ pub async fn run(cwd: PathBuf, args: HubArgs) -> Result<()> { let mut cfg = read_or_default(&path)?; cfg.vault.hub_pubkey = Some(pk.to_ssh_string()); write(&path, &cfg)?; - println!("pinned hub identity {} ({})", pk.to_ssh_string(), pk.fingerprint_sha256()); + println!( + "pinned hub identity {} ({})", + pk.to_ssh_string(), + pk.fingerprint_sha256() + ); } HubOp::Forget => { let mut cfg = read_or_default(&path)?; diff --git a/crates/agentsync-cli/src/commands/init.rs b/crates/agentsync-cli/src/commands/init.rs index 3145a84..5fe2ce6 100644 --- a/crates/agentsync-cli/src/commands/init.rs +++ b/crates/agentsync-cli/src/commands/init.rs @@ -1,6 +1,8 @@ use crate::cli::InitArgs; -use crate::config::{config_path, identity_path, write, ConfigFile, IdentitySection, SyncSection, VaultSection}; -use agentsync_core::{normalize_rendezvous_url, CreateOptions, Identity, Vault}; +use crate::config::{ + ConfigFile, IdentitySection, SyncSection, VaultSection, config_path, identity_path, write, +}; +use agentsync_core::{CreateOptions, Identity, Vault, normalize_rendezvous_url}; use anyhow::{Context, Result}; use std::path::{Path, PathBuf}; @@ -47,7 +49,9 @@ pub async fn run(cwd: PathBuf, args: InitArgs) -> Result<()> { Identity::load_from_file(&id_path).map_err(|e| anyhow::anyhow!(e))? } else { let fresh = Identity::generate(); - fresh.save_to_file(&id_path).map_err(|e| anyhow::anyhow!(e))?; + fresh + .save_to_file(&id_path) + .map_err(|e| anyhow::anyhow!(e))?; fresh }; @@ -69,10 +73,7 @@ pub async fn run(cwd: PathBuf, args: InitArgs) -> Result<()> { println!("Initialized agentsync vault."); println!("vault_id = {}", created.vault_id); - println!( - "name = {}", - name.as_deref().unwrap_or("(unnamed)") - ); + println!("name = {}", name.as_deref().unwrap_or("(unnamed)")); println!("identity_pub = {}", identity.pubkey().to_ssh_string()); println!("identity_path = {}", id_path.display()); println!(); @@ -113,8 +114,7 @@ fn ensure_ignore_entry(path: &Path, entry: &str) -> Result<()> { } body.push_str(entry); body.push('\n'); - std::fs::write(path, body) - .with_context(|| format!("write {}", path.display()))?; + std::fs::write(path, body).with_context(|| format!("write {}", path.display()))?; Ok(()) } diff --git a/crates/agentsync-cli/src/commands/key.rs b/crates/agentsync-cli/src/commands/key.rs index 52c57a7..843ff37 100644 --- a/crates/agentsync-cli/src/commands/key.rs +++ b/crates/agentsync-cli/src/commands/key.rs @@ -1,5 +1,5 @@ use crate::cli::{KeyArgs, KeyOp}; -use crate::config::{identity_path, read_or_default, write, ConfigFile, IdentitySection}; +use crate::config::{ConfigFile, IdentitySection, identity_path, read_or_default, write}; use agentsync_core::Identity; use anyhow::{Context, Result}; use std::path::PathBuf; diff --git a/crates/agentsync-cli/src/commands/mod.rs b/crates/agentsync-cli/src/commands/mod.rs index 7cea286..0fa5aa9 100644 --- a/crates/agentsync-cli/src/commands/mod.rs +++ b/crates/agentsync-cli/src/commands/mod.rs @@ -1,14 +1,14 @@ -pub mod init; -pub mod watch; pub mod clone; -pub mod status; +pub mod compact; +pub mod diff; +pub mod hub; +pub mod init; +pub mod key; pub mod push_pull; pub mod restore; pub mod snapshot; -pub mod diff; -pub mod compact; -pub mod key; -pub mod hub; +pub mod status; +pub mod watch; use std::path::Path; diff --git a/crates/agentsync-cli/src/commands/restore.rs b/crates/agentsync-cli/src/commands/restore.rs index 5bcc013..b5bbfe1 100644 --- a/crates/agentsync-cli/src/commands/restore.rs +++ b/crates/agentsync-cli/src/commands/restore.rs @@ -2,7 +2,7 @@ use crate::cli::RestoreAtArgs; use crate::commands::require_config; use crate::config; use agentsync_core::{OpenOptions, Vault}; -use anyhow::{bail, Result}; +use anyhow::{Result, bail}; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -87,7 +87,10 @@ mod tests { #[test] fn parses_epoch_ms_passthrough() { - assert_eq!(parse_timestamp("1700000000000", 0).unwrap(), 1_700_000_000_000); + assert_eq!( + parse_timestamp("1700000000000", 0).unwrap(), + 1_700_000_000_000 + ); } #[test] diff --git a/crates/agentsync-cli/src/commands/status.rs b/crates/agentsync-cli/src/commands/status.rs index 327bef2..906b775 100644 --- a/crates/agentsync-cli/src/commands/status.rs +++ b/crates/agentsync-cli/src/commands/status.rs @@ -20,7 +20,10 @@ pub async fn run(cwd: PathBuf) -> Result<()> { let vault = Vault::open(opts).await?; let files = vault.list_files().await?; println!("vault_id: {}", vault_id); - println!("name: {}", cfg.vault.name.as_deref().unwrap_or("(unnamed)")); + println!( + "name: {}", + cfg.vault.name.as_deref().unwrap_or("(unnamed)") + ); println!("storage: {}", vault.storage_path().display()); println!( "rendezvous: {}", diff --git a/crates/agentsync-cli/src/commands/watch.rs b/crates/agentsync-cli/src/commands/watch.rs index ef13c4e..4d621c1 100644 --- a/crates/agentsync-cli/src/commands/watch.rs +++ b/crates/agentsync-cli/src/commands/watch.rs @@ -1,10 +1,10 @@ -use crate::cli::{WatchArgs, LISTEN_DEFAULT_SENTINEL}; +use crate::cli::{LISTEN_DEFAULT_SENTINEL, WatchArgs}; use crate::commands::require_config; use crate::config; use agentsync_core::{ - normalize_with_scheme, parse_authorized_keys, render_authorized_keys, AuthorizedPeer, - OpenOptions, ReconnectOptions, Vault, AUTHORIZED_KEYS_FILE, DEFAULT_LISTEN_ADDR, - DEFAULT_LISTEN_ADDR_NO_TLS, + AUTHORIZED_KEYS_FILE, AuthorizedPeer, DEFAULT_LISTEN_ADDR, DEFAULT_LISTEN_ADDR_NO_TLS, + OpenOptions, ReconnectOptions, Vault, normalize_with_scheme, parse_authorized_keys, + render_authorized_keys, }; use anyhow::{Context, Result}; use std::path::PathBuf; @@ -144,7 +144,10 @@ async fn merge_authorized_keys(vault: &Vault, raw: &str) -> Result<()> { vault .write_text_file(AUTHORIZED_KEYS_FILE, &rendered) .await?; - info!(added, "merged keys from --authorized-keys / AGENTSYNC_AUTHORIZED_KEYS"); + info!( + added, + "merged keys from --authorized-keys / AGENTSYNC_AUTHORIZED_KEYS" + ); } Ok(()) } @@ -170,8 +173,7 @@ fn load_pubkey_arg(s: &str) -> Result { if s.starts_with("ssh-") { return Ok(s.to_string()); } - let bytes = std::fs::read_to_string(s) - .with_context(|| format!("read pubkey file at {}", s))?; + let bytes = std::fs::read_to_string(s).with_context(|| format!("read pubkey file at {}", s))?; let line = bytes .lines() .next() diff --git a/crates/agentsync-cli/src/config.rs b/crates/agentsync-cli/src/config.rs index f5b860f..dc6aa6e 100644 --- a/crates/agentsync-cli/src/config.rs +++ b/crates/agentsync-cli/src/config.rs @@ -111,11 +111,9 @@ pub fn read_or_default(vault_root: &Path) -> Result { if !path.exists() { return Ok(ConfigFile::default()); } - let bytes = - std::fs::read(&path).with_context(|| format!("read {}", path.display()))?; + let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?; let s = std::str::from_utf8(&bytes).context("config.toml not valid utf-8")?; - let cfg: ConfigFile = toml::from_str(s) - .with_context(|| format!("parse {}", path.display()))?; + let cfg: ConfigFile = toml::from_str(s).with_context(|| format!("parse {}", path.display()))?; Ok(cfg) } @@ -167,8 +165,7 @@ pub fn user_identity_default() -> PathBuf { /// `--identity-agent-pubkey` flag. pub fn resolve_identity(vault_root: &Path, cfg: &ConfigFile) -> Result { if let Some(agent_pubkey_str) = cfg.identity.agent_pubkey.as_deref() { - let agent_pk = Pubkey::from_ssh_string(agent_pubkey_str) - .map_err(|e| anyhow::anyhow!(e))?; + let agent_pk = Pubkey::from_ssh_string(agent_pubkey_str).map_err(|e| anyhow::anyhow!(e))?; let socket = resolve_agent_socket(cfg)?; return Ok(Identity::from_agent(socket, agent_pk)); } diff --git a/crates/agentsync-cli/src/main.rs b/crates/agentsync-cli/src/main.rs index 9cb9277..64e530f 100644 --- a/crates/agentsync-cli/src/main.rs +++ b/crates/agentsync-cli/src/main.rs @@ -15,16 +15,15 @@ async fn main() -> Result<()> { let raw: Vec = std::env::args().skip(1).collect(); let resolved = cli::resolve_args(&raw); - let parsed = match cli::Cli::try_parse_from( - std::iter::once("agentsync".to_string()).chain(resolved), - ) { - Ok(p) => p, - Err(e) => { - // clap returns DisplayHelp / DisplayVersion as Err with ExitCode::SUCCESS - // semantics; print and exit cleanly without anyhow's "Error:" prefix. - e.exit(); - } - }; + let parsed = + match cli::Cli::try_parse_from(std::iter::once("agentsync".to_string()).chain(resolved)) { + Ok(p) => p, + Err(e) => { + // clap returns DisplayHelp / DisplayVersion as Err with ExitCode::SUCCESS + // semantics; print and exit cleanly without anyhow's "Error:" prefix. + e.exit(); + } + }; let cwd = parsed.cwd; match parsed.command { @@ -91,10 +90,7 @@ fn install_completions(kind: cli::ShellKind, shell: Shell) -> Result<()> { \tautoload -U compinit && compinit", ), ), - cli::ShellKind::Fish => ( - home.join(".config/fish/completions/agentsync.fish"), - None, - ), + cli::ShellKind::Fish => (home.join(".config/fish/completions/agentsync.fish"), None), cli::ShellKind::PowerShell | cli::ShellKind::Elvish => { anyhow::bail!( "--install isn't supported for {:?} (no conventional dropbox path); \ @@ -105,8 +101,7 @@ fn install_completions(kind: cli::ShellKind, shell: Shell) -> Result<()> { }; if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create {}", parent.display()))?; + std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; } let mut buf: Vec = Vec::new(); let mut cmd = cli::Cli::command(); @@ -121,7 +116,8 @@ fn install_completions(kind: cli::ShellKind, shell: Shell) -> Result<()> { fn init_tracing() { use tracing_subscriber::EnvFilter; - let filter = EnvFilter::try_from_env("AGENTSYNC_LOG").unwrap_or_else(|_| EnvFilter::new("info")); + let filter = + EnvFilter::try_from_env("AGENTSYNC_LOG").unwrap_or_else(|_| EnvFilter::new("info")); // Logs go to stderr so they don't pollute stdout-based protocols (the // listen-port handshake the harness reads from stdout, etc). let _ = tracing_subscriber::fmt() diff --git a/crates/agentsync-core/Cargo.toml b/crates/agentsync-core/Cargo.toml index 54b3ec4..1ae0aef 100644 --- a/crates/agentsync-core/Cargo.toml +++ b/crates/agentsync-core/Cargo.toml @@ -6,37 +6,52 @@ license.workspace = true authors.workspace = true description = "Real-time directory sync engine using Automerge CRDTs" +# Wasm-safe dependencies only. Anything that pulls in tokio sockets, OS +# filesystem watchers, or rustls must go in the target-cfg block below so +# the wasm32 build of this crate stays buildable. [dependencies] automerge.workspace = true -tokio.workspace = true -tokio-tungstenite.workspace = true -futures-util.workspace = true -notify.workspace = true serde.workspace = true serde_json.workspace = true rmp-serde.workspace = true tracing.workspace = true hmac.workspace = true sha2.workspace = true -rand.workspace = true hex.workspace = true base64.workspace = true thiserror.workspace = true uuid.workspace = true unicode-normalization.workspace = true -async-trait.workspace = true url.workspace = true bytes.workspace = true -walkdir.workspace = true -ignore.workspace = true serde_bytes = "0.11" ed25519-dalek.workspace = true rand_core.workspace = true + +# Native-only: tokio runtime, sockets, TLS, file watcher. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio.workspace = true +tokio-tungstenite.workspace = true +futures-util.workspace = true +notify.workspace = true +rand.workspace = true +async-trait.workspace = true +walkdir.workspace = true +ignore.workspace = true rcgen.workspace = true rustls.workspace = true rustls-pki-types.workspace = true tokio-rustls.workspace = true +# Wasm: pull in getrandom's browser/node backends so OsRng works, plus +# js-sys for Date.now() as the time source. Two getrandom majors are +# present transitively (rand_core 0.6 → getrandom 0.2; automerge → 0.4), +# so we enable the JS backend on both. +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } +getrandom_04 = { package = "getrandom", version = "0.4", features = ["wasm_js"] } +js-sys = "0.3" + [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } tempfile = "3" diff --git a/crates/agentsync-core/src/constants.rs b/crates/agentsync-core/src/constants.rs index 02ae159..1d80399 100644 --- a/crates/agentsync-core/src/constants.rs +++ b/crates/agentsync-core/src/constants.rs @@ -104,10 +104,7 @@ mod tests { #[test] fn missing_scheme_defaults_to_wss() { assert_eq!(normalize_rendezvous_url("my-hub"), "wss://my-hub"); - assert_eq!( - normalize_rendezvous_url("my-hub:8443"), - "wss://my-hub:8443" - ); + assert_eq!(normalize_rendezvous_url("my-hub:8443"), "wss://my-hub:8443"); assert_eq!( normalize_rendezvous_url("hub.example.com"), "wss://hub.example.com" @@ -126,10 +123,7 @@ mod tests { "ws://my-hub:8080" ); // Explicit schemes still win. - assert_eq!( - normalize_with_scheme("wss://my-hub", true), - "wss://my-hub" - ); + assert_eq!(normalize_with_scheme("wss://my-hub", true), "wss://my-hub"); } #[test] @@ -144,9 +138,6 @@ mod tests { #[test] fn whitespace_trimmed() { - assert_eq!( - normalize_rendezvous_url(" my-hub "), - "wss://my-hub" - ); + assert_eq!(normalize_rendezvous_url(" my-hub "), "wss://my-hub"); } } diff --git a/crates/agentsync-core/src/doc/directories.rs b/crates/agentsync-core/src/doc/directories.rs index 7065aa5..a3c8028 100644 --- a/crates/agentsync-core/src/doc/directories.rs +++ b/crates/agentsync-core/src/doc/directories.rs @@ -1,10 +1,10 @@ use crate::doc::{ - get_int, get_object, get_str, map_keys, new_id, now_ms, DirId, DirectoryMeta, Doc, + DirId, DirectoryMeta, Doc, get_int, get_object, get_str, map_keys, new_id, now_ms, }; use crate::error::{Error, Result}; use crate::path; -use automerge::transaction::Transactable; use automerge::ObjType; +use automerge::transaction::Transactable; impl Doc { pub fn find_directory_by_path(&mut self, path: &str) -> Result> { diff --git a/crates/agentsync-core/src/doc/files.rs b/crates/agentsync-core/src/doc/files.rs index df3c85e..b998969 100644 --- a/crates/agentsync-core/src/doc/files.rs +++ b/crates/agentsync-core/src/doc/files.rs @@ -1,11 +1,11 @@ use crate::doc::{ - content_hash, get_int, get_object, get_str, get_text, map_keys, new_id, now_ms, Doc, FileId, - FileKind, FileMeta, + Doc, FileId, FileKind, FileMeta, content_hash, get_int, get_object, get_str, get_text, + map_keys, new_id, now_ms, }; use crate::error::{Error, Result}; use crate::path; -use automerge::transaction::Transactable; use automerge::ObjType; +use automerge::transaction::Transactable; impl Doc { pub fn find_file_by_path(&mut self, path: &str) -> Result> { @@ -141,8 +141,7 @@ impl Doc { if let Some((text_id, current)) = get_text(&mut self.inner, &entry, "content")? { if current != content { let len = current.chars().count(); - self.inner - .splice_text(&text_id, 0, len as isize, content)?; + self.inner.splice_text(&text_id, 0, len as isize, content)?; } } else { let text_id = self.inner.put_object(&entry, "content", ObjType::Text)?; @@ -185,7 +184,8 @@ impl Doc { let entry = get_object(&mut self.inner, &files, &fid)?.unwrap(); let meta = get_object(&mut self.inner, &entry, "meta")?.unwrap(); self.inner.put(&entry, "binary_hash", hash)?; - self.inner.put(&meta, "kind", FileKind::Attachment.as_str())?; + self.inner + .put(&meta, "kind", FileKind::Attachment.as_str())?; self.inner.put(&meta, "size", size)?; self.inner.put(&meta, "updated_at", now)?; self.inner.delete(&meta, "deleted_at")?; @@ -198,7 +198,8 @@ impl Doc { let entry = self.inner.put_object(&files, &fid, ObjType::Map)?; let meta = self.inner.put_object(&entry, "meta", ObjType::Map)?; self.inner.put(&meta, "path", path.as_str())?; - self.inner.put(&meta, "kind", FileKind::Attachment.as_str())?; + self.inner + .put(&meta, "kind", FileKind::Attachment.as_str())?; self.inner.put(&meta, "size", size)?; self.inner.put(&meta, "created_at", now)?; self.inner.put(&meta, "updated_at", now)?; diff --git a/crates/agentsync-core/src/doc/history.rs b/crates/agentsync-core/src/doc/history.rs index c669cfe..b892974 100644 --- a/crates/agentsync-core/src/doc/history.rs +++ b/crates/agentsync-core/src/doc/history.rs @@ -1,6 +1,4 @@ -use crate::doc::{ - get_int, get_object, get_text, map_keys, now_ms, Doc, FileKind, Label, -}; +use crate::doc::{Doc, FileKind, Label, get_int, get_object, get_text, map_keys, now_ms}; use crate::error::{Error, Result}; use automerge::transaction::Transactable; use automerge::{ChangeHash, ObjType, ReadDoc, ScalarValue, Value}; @@ -12,7 +10,8 @@ impl Doc { let heads = self.inner.get_heads(); let encoded = encode_heads(&heads); let entry = self.inner.put_object(&labels, label, ObjType::Map)?; - self.inner.put(&entry, "heads", ScalarValue::Bytes(encoded))?; + self.inner + .put(&entry, "heads", ScalarValue::Bytes(encoded))?; self.inner.put(&entry, "created_at", now_ms())?; self.commit_now(); Ok(()) @@ -155,10 +154,7 @@ impl Doc { self.inner.delete(&meta, "deleted_at")?; match past_meta.kind { FileKind::Text => { - let target = past_file_contents - .get(fid) - .cloned() - .unwrap_or_default(); + let target = past_file_contents.get(fid).cloned().unwrap_or_default(); let cur_text = get_text(&mut self.inner, &entry, "content")?; match cur_text { Some((id, current)) => { @@ -168,9 +164,7 @@ impl Doc { } } None => { - let id = self - .inner - .put_object(&entry, "content", ObjType::Text)?; + let id = self.inner.put_object(&entry, "content", ObjType::Text)?; if !target.is_empty() { self.inner.splice_text(&id, 0, 0, &target)?; } @@ -227,8 +221,7 @@ impl Doc { pub fn heads_at_time(&mut self, target_ms: i64) -> Result> { let changes = self.inner.get_changes(&[]); - let mut included: std::collections::HashSet = - std::collections::HashSet::new(); + let mut included: std::collections::HashSet = std::collections::HashSet::new(); for c in &changes { if c.timestamp() <= target_ms { included.insert(c.hash()); diff --git a/crates/agentsync-core/src/doc/mod.rs b/crates/agentsync-core/src/doc/mod.rs index 12b980d..4230994 100644 --- a/crates/agentsync-core/src/doc/mod.rs +++ b/crates/agentsync-core/src/doc/mod.rs @@ -14,16 +14,17 @@ //! Files and directories are keyed by stable UUIDs; paths are mutable fields. use crate::error::{Error, Result}; +use automerge::sync::{self as amsync, SyncDoc}; use automerge::transaction::{CommitOptions, Transactable}; use automerge::{ - ActorId, AutoCommit, ChangeHash, ObjId, ObjType, ReadDoc, ScalarValue, Value, ROOT, + ActorId, AutoCommit, ChangeHash, ObjId, ObjType, ROOT, ReadDoc, ScalarValue, Value, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; -pub mod files; pub mod directories; +pub mod files; pub mod history; pub const SCHEMA_VERSION: i64 = 1; @@ -149,10 +150,7 @@ impl Doc { pub(crate) fn map_obj(&mut self, key: &str) -> Result { match self.inner.get(ROOT, key)? { Some((Value::Object(_), id)) => Ok(id), - _ => Err(Error::Other(format!( - "schema missing root.{}", - key - ))), + _ => Err(Error::Other(format!("schema missing root.{}", key))), } } @@ -175,6 +173,27 @@ impl Doc { let after = self.inner.get_heads(); Ok(before != after) } + + /// Generate the next outbound sync message for `state`. Returns `None` + /// when no message is currently needed (peer is up to date). + pub fn generate_sync_message(&mut self, state: &mut amsync::State) -> Option> { + let sd = self.inner.sync(); + sd.generate_sync_message(state).map(|m| m.encode()) + } + + /// Apply an inbound sync message. Returns true if heads moved (i.e. + /// new changes were applied). + pub fn receive_sync_message(&mut self, state: &mut amsync::State, msg: &[u8]) -> Result { + let parsed = amsync::Message::decode(msg) + .map_err(|e| Error::Other(format!("decode sync msg: {}", e)))?; + let before = self.inner.get_heads(); + let mut sd = self.inner.sync(); + sd.receive_sync_message(state, parsed) + .map_err(|e| Error::Other(format!("receive sync msg: {}", e)))?; + drop(sd); + let after = self.inner.get_heads(); + Ok(before != after) + } } fn genesis_actor(vault_id: &str) -> ActorId { @@ -185,6 +204,7 @@ fn genesis_actor(vault_id: &str) -> ActorId { ActorId::from(&digest[..]) } +#[cfg(not(target_arch = "wasm32"))] pub(crate) fn now_ms() -> i64 { use std::time::{SystemTime, UNIX_EPOCH}; SystemTime::now() @@ -193,6 +213,13 @@ pub(crate) fn now_ms() -> i64 { .unwrap_or(0) } +/// On `wasm32-unknown-unknown` there is no `SystemTime`. Use the JS host's +/// `Date.now()` as the wall-clock source. +#[cfg(target_arch = "wasm32")] +pub(crate) fn now_ms() -> i64 { + js_sys::Date::now() as i64 +} + impl Doc { /// Commit the current transaction stamping it with wall-clock time. /// The default `AutoCommit::commit()` records `time = None` (serialized @@ -245,7 +272,11 @@ pub(crate) fn get_int(doc: &mut impl ReadDoc, obj: &ObjId, key: &str) -> Result< } } -pub(crate) fn get_text(doc: &mut impl ReadDoc, obj: &ObjId, key: &str) -> Result> { +pub(crate) fn get_text( + doc: &mut impl ReadDoc, + obj: &ObjId, + key: &str, +) -> Result> { match doc.get(obj, key)? { Some((Value::Object(ObjType::Text), id)) => { let text = doc.text(&id)?; diff --git a/crates/agentsync-core/src/error.rs b/crates/agentsync-core/src/error.rs index 9ed3fb1..ba3e741 100644 --- a/crates/agentsync-core/src/error.rs +++ b/crates/agentsync-core/src/error.rs @@ -32,6 +32,7 @@ pub enum Error { #[error("network: {0}")] Network(String), + #[cfg(not(target_arch = "wasm32"))] #[error("notify: {0}")] Notify(#[from] notify::Error), @@ -60,6 +61,7 @@ pub enum Error { Other(String), } +#[cfg(not(target_arch = "wasm32"))] impl From for Error { fn from(e: tokio_tungstenite::tungstenite::Error) -> Self { Error::WebSocket(e.to_string()) diff --git a/crates/agentsync-core/src/fs/adapter.rs b/crates/agentsync-core/src/fs/adapter.rs index ae0c01b..eb804fb 100644 --- a/crates/agentsync-core/src/fs/adapter.rs +++ b/crates/agentsync-core/src/fs/adapter.rs @@ -17,7 +17,11 @@ pub trait FilesystemAdapter: Send + Sync { async fn list(&self, path: &Path) -> Result>; async fn exists(&self, path: &Path) -> bool; async fn hash(&self, path: &Path) -> Result; - fn watch(&self, path: &Path, sink: tokio::sync::mpsc::UnboundedSender) -> Result>; + fn watch( + &self, + path: &Path, + sink: tokio::sync::mpsc::UnboundedSender, + ) -> Result>; } pub trait Watcher: Send + Sync {} diff --git a/crates/agentsync-core/src/fs/binding.rs b/crates/agentsync-core/src/fs/binding.rs index 8dc8f33..f65d05b 100644 --- a/crates/agentsync-core/src/fs/binding.rs +++ b/crates/agentsync-core/src/fs/binding.rs @@ -1,7 +1,7 @@ use crate::constants::AUTHORIZED_KEYS_FILE; use crate::fs::adapter::{FilesystemAdapter, Watcher}; use crate::fs::suppression::DirtySet; -use crate::fs::sync_ignore::{SyncIgnoreSet, SYNC_IGNORE_FILENAME}; +use crate::fs::sync_ignore::{SYNC_IGNORE_FILENAME, SyncIgnoreSet}; use crate::path as path_norm; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; diff --git a/crates/agentsync-core/src/fs/ingest.rs b/crates/agentsync-core/src/fs/ingest.rs index 2275d83..5a99843 100644 --- a/crates/agentsync-core/src/fs/ingest.rs +++ b/crates/agentsync-core/src/fs/ingest.rs @@ -50,7 +50,11 @@ async fn ingest_directory( // nested dirs and the files inside them. let abs_root = binding.vault_path_to_fs_path(vault_path); let mut changed = false; - for entry in WalkDir::new(&abs_root).follow_links(false).into_iter().filter_map(|e| e.ok()) { + for entry in WalkDir::new(&abs_root) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + { let entry_abs = entry.path().to_path_buf(); if entry.file_type().is_dir() { let p = match binding.fs_path_to_vault_dir_path(&entry_abs) { @@ -171,11 +175,7 @@ async fn dispatch_fs_event( if doc.find_directory_by_path(&vault_path)?.is_some() { doc.delete_directory(&vault_path, true)?; drop(doc); - binding - .materialized_dirs - .lock() - .await - .remove(&vault_path); + binding.materialized_dirs.lock().await.remove(&vault_path); inner.doc_changed.notify_waiters(); let _ = inner.events.send(VaultEvent { kind: VaultEventKind::FileChanged { path: vault_path }, diff --git a/crates/agentsync-core/src/fs/mod.rs b/crates/agentsync-core/src/fs/mod.rs index 5b3d1e4..1c8691c 100644 --- a/crates/agentsync-core/src/fs/mod.rs +++ b/crates/agentsync-core/src/fs/mod.rs @@ -1,11 +1,11 @@ pub mod adapter; -pub mod node_adapter; -pub mod suppression; pub mod binding; pub mod ingest; +pub mod node_adapter; +pub mod suppression; pub mod sync_ignore; pub use adapter::{DirEntry, FilesystemAdapter, Watcher}; -pub use binding::{Binding, BindOptions}; +pub use binding::{BindOptions, Binding}; pub use node_adapter::NodeFsAdapter; -pub use sync_ignore::{SyncIgnoreSet, SYNC_IGNORE_FILENAME}; +pub use sync_ignore::{SYNC_IGNORE_FILENAME, SyncIgnoreSet}; diff --git a/crates/agentsync-core/src/fs/node_adapter.rs b/crates/agentsync-core/src/fs/node_adapter.rs index d0c1bac..abb8baf 100644 --- a/crates/agentsync-core/src/fs/node_adapter.rs +++ b/crates/agentsync-core/src/fs/node_adapter.rs @@ -2,8 +2,8 @@ use crate::doc::content_hash; use crate::error::{Error, Result}; use crate::fs::adapter::{DirEntry, FilesystemAdapter, FsEvent, Watcher}; use async_trait::async_trait; -use notify::{EventKind, RecommendedWatcher, RecursiveMode}; use notify::Watcher as NotifyWatcher; +use notify::{EventKind, RecommendedWatcher, RecursiveMode}; use std::path::Path; use std::sync::Arc; use tokio::fs; @@ -42,10 +42,7 @@ impl FilesystemAdapter for NodeFsAdapter { } } let tmp = match path.file_name() { - Some(name) => path.with_file_name(format!( - ".{}.agentsync-tmp", - name.to_string_lossy() - )), + Some(name) => path.with_file_name(format!(".{}.agentsync-tmp", name.to_string_lossy())), None => return Err(Error::InvalidPath(path.display().to_string())), }; fs::write(&tmp, content).await?; @@ -88,35 +85,32 @@ impl FilesystemAdapter for NodeFsAdapter { Ok(content_hash(&bytes)) } - fn watch( - &self, - path: &Path, - sink: mpsc::UnboundedSender, - ) -> Result> { + fn watch(&self, path: &Path, sink: mpsc::UnboundedSender) -> Result> { let path = path.to_path_buf(); let sink = Arc::new(sink); let sink_cloned = sink.clone(); - let mut watcher = notify::recommended_watcher(move |res: notify::Result| { - let event = match res { - Ok(e) => e, - Err(e) => { - tracing::warn!(error=%e, "notify error"); - return; - } - }; - for path in &event.paths { - let p = path.clone(); - match event.kind { - EventKind::Create(_) | EventKind::Modify(_) => { - let _ = sink_cloned.send(FsEvent::Touched(p)); + let mut watcher = + notify::recommended_watcher(move |res: notify::Result| { + let event = match res { + Ok(e) => e, + Err(e) => { + tracing::warn!(error=%e, "notify error"); + return; } - EventKind::Remove(_) => { - let _ = sink_cloned.send(FsEvent::Removed(p)); + }; + for path in &event.paths { + let p = path.clone(); + match event.kind { + EventKind::Create(_) | EventKind::Modify(_) => { + let _ = sink_cloned.send(FsEvent::Touched(p)); + } + EventKind::Remove(_) => { + let _ = sink_cloned.send(FsEvent::Removed(p)); + } + _ => {} } - _ => {} } - } - })?; + })?; watcher.watch(&path, RecursiveMode::Recursive)?; Ok(Box::new(NotifyHandle { _watcher: watcher })) } diff --git a/crates/agentsync-core/src/fs/sync_ignore.rs b/crates/agentsync-core/src/fs/sync_ignore.rs index d004428..433ec5b 100644 --- a/crates/agentsync-core/src/fs/sync_ignore.rs +++ b/crates/agentsync-core/src/fs/sync_ignore.rs @@ -66,7 +66,9 @@ impl SyncIgnoreSet { // file rather than aborting the whole walk. continue; } - let Ok(matcher) = builder.build() else { continue }; + let Ok(matcher) = builder.build() else { + continue; + }; let depth = dir .strip_prefix(vault_root) .map(|p| p.components().count()) @@ -167,10 +169,7 @@ mod tests { #[test] fn negation_overrides_earlier_pattern() { let dir = TempDir::new().unwrap(); - write( - &dir.path().join(".syncignore"), - "*.log\n!keep.log\n", - ); + write(&dir.path().join(".syncignore"), "*.log\n!keep.log\n"); let set = SyncIgnoreSet::from_vault_root(dir.path()); assert!(set.matches("foo.log", false)); assert!(!set.matches("keep.log", false)); diff --git a/crates/agentsync-core/src/host/crypto.rs b/crates/agentsync-core/src/host/crypto.rs new file mode 100644 index 0000000..23e0105 --- /dev/null +++ b/crates/agentsync-core/src/host/crypto.rs @@ -0,0 +1,50 @@ +//! Cryptographic services the host supplies to the core: random bytes, +//! signing identities, and TLS cert provisioning. +//! +//! `Rng` is a dependency of the handshake (nonce generation) and identity +//! generation. Native uses `OsRng`; wasm uses `crypto.getRandomValues()` via +//! `web_sys::crypto`. +//! +//! `Signer` abstracts over file-backed identities (sync, in-process) and +//! external signers (ssh-agent on native, WebAuthn / hardware-backed on +//! wasm). `crate::Identity` will implement this directly for the native +//! file-backed path; ssh-agent stays gated to native. +//! +//! `TlsCertProvider` is native-only. Wasm hosts return `None` from +//! `Host::tls()` — TLS termination happens at the underlying transport +//! (browser WebSocket, Node `ws`). + +use crate::error::Result; +use crate::identity::{Pubkey, SIGNATURE_LEN}; +use async_trait::async_trait; +use std::path::Path; + +/// Random byte source. Implementations must be cryptographically secure. +pub trait Rng: Send + Sync + 'static { + fn fill_bytes(&self, buf: &mut [u8]); +} + +/// Anything that can produce ed25519 signatures over a pubkey it claims. +/// The handshake calls `sign` with the canonical transcript bytes; the +/// underlying private key never leaves the signer. +#[async_trait(?Send)] +pub trait Signer: Send + Sync + 'static { + async fn sign(&self, msg: &[u8]) -> Result<[u8; SIGNATURE_LEN]>; + fn pubkey(&self) -> Pubkey; +} + +/// Native-only. Generates / loads the self-signed TLS cert the hub +/// presents. Browsers never reach this code; their TLS comes from the OS +/// trust store via the WebSocket layer. +#[async_trait(?Send)] +pub trait TlsCertProvider: Send + Sync + 'static { + /// Load `/key.der` + `cert.der`, generating a fresh self-signed + /// pair if absent. The native impl uses rcgen + ed25519. + async fn load_or_generate(&self, dir: &Path) -> Result; +} + +#[derive(Clone)] +pub struct TlsCert { + pub cert_der: Vec, + pub key_der: Vec, +} diff --git a/crates/agentsync-core/src/host/filesystem.rs b/crates/agentsync-core/src/host/filesystem.rs new file mode 100644 index 0000000..e181448 --- /dev/null +++ b/crates/agentsync-core/src/host/filesystem.rs @@ -0,0 +1,55 @@ +//! Filesystem adapter for the *bound directory* — the user-visible folder +//! that the Vault is mirroring. This is distinct from the storage layer +//! (`.agentsync/` internals); the bound directory is where real files live. +//! +//! The native impl is `notify` + `tokio::fs`. The wasm impl is supplied by +//! JS and may be `node:fs` (Electron, Tauri, VS Code) or the File System +//! Access API (browser apps). Browsers without a backing folder run in +//! storage-only mode where this trait is `None` on the Host bundle. +//! +//! This trait was previously `crate::fs::adapter::FilesystemAdapter`; it's +//! moved here as part of the host abstraction layer. The two new methods +//! (`create_dir_all`, `remove_dir`) replace the only non-abstracted FS +//! callsites in the materializer. + +use crate::error::Result; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; +use tokio::sync::mpsc::UnboundedSender; + +#[derive(Debug, Clone)] +pub struct DirEntry { + pub path: PathBuf, + pub is_dir: bool, + pub size: u64, +} + +#[async_trait(?Send)] +pub trait FilesystemAdapter: Send + Sync + 'static { + async fn read(&self, path: &Path) -> Result>; + async fn write(&self, path: &Path, content: &[u8]) -> Result<()>; + async fn delete(&self, path: &Path) -> Result<()>; + async fn list(&self, path: &Path) -> Result>; + async fn exists(&self, path: &Path) -> bool; + async fn hash(&self, path: &Path) -> Result; + /// Recursively create a directory and any missing parents. Idempotent. + async fn create_dir_all(&self, path: &Path) -> Result<()>; + /// Remove an empty directory. Errors if the directory is non-empty. + async fn remove_dir(&self, path: &Path) -> Result<()>; + /// Install a watcher for `path`. Events flow into `sink`. Drop the + /// returned [`Watcher`] to stop watching. Wasm impls running in pure + /// storage mode return an error here. + fn watch(&self, path: &Path, sink: UnboundedSender) -> Result>; +} + +pub trait Watcher: Send + Sync {} + +#[derive(Debug, Clone)] +pub enum FsEvent { + /// A file was created or modified at this absolute path. + Touched(PathBuf), + /// A file or directory was removed at this absolute path. + Removed(PathBuf), + /// A rename: old path → new path. + Renamed { from: PathBuf, to: PathBuf }, +} diff --git a/crates/agentsync-core/src/host/mod.rs b/crates/agentsync-core/src/host/mod.rs new file mode 100644 index 0000000..d1dd3f5 --- /dev/null +++ b/crates/agentsync-core/src/host/mod.rs @@ -0,0 +1,50 @@ +//! Host abstraction layer. +//! +//! The `Vault` is generic over its environment by holding `Arc`. +//! Every OS-touching operation — spawning tasks, reading the clock, opening +//! sockets, persisting bytes, watching filesystems, generating randomness, +//! signing — flows through this trait. Native hosts are tokio + rustls + +//! notify + disk; wasm hosts are JS-supplied shims. +//! +//! Sub-trait getter methods return `&dyn ...` (not `&'static dyn ...`) so +//! impls can keep state per-host. The optional methods (`listener`, +//! `filesystem`, `tls`) return `None` on hosts that genuinely cannot +//! support that capability — browsers can't bind listeners or terminate +//! TLS themselves, and pure-CRDT browser apps may run without a +//! filesystem at all. + +pub mod crypto; +pub mod filesystem; +pub mod runtime; +pub mod storage; +pub mod transport; + +pub mod native; + +pub use crypto::{Rng, Signer, TlsCert, TlsCertProvider}; +pub use filesystem::{DirEntry, FilesystemAdapter, FsEvent, Watcher}; +pub use runtime::{Clock, SpawnHandle, SpawnHandleImpl, Spawner}; +pub use storage::{BlobStorage, DocStorage, SnapshotEntry, SnapshotStorage}; +pub use transport::{Acceptor, Conn, ConnectOpts, Listener, TlsConfig, Transport}; + +/// The environment a Vault runs in. Native and wasm hosts both implement +/// this; tests mock individual sub-traits and use a [`HostBuilder`] to +/// assemble custom bundles. +pub trait Host: Send + Sync + 'static { + fn spawner(&self) -> &dyn Spawner; + fn clock(&self) -> &dyn Clock; + fn rng(&self) -> &dyn Rng; + fn transport(&self) -> &dyn Transport; + /// Inbound listener. Browsers cannot listen; Node could (currently + /// unused on wasm). Native always has one. + fn listener(&self) -> Option<&dyn Listener>; + fn doc_storage(&self) -> &dyn DocStorage; + fn blob_storage(&self) -> &dyn BlobStorage; + fn snapshot_storage(&self) -> &dyn SnapshotStorage; + /// Bound-directory adapter. `None` for storage-only mode (browser apps + /// without a backing directory). + fn filesystem(&self) -> Option<&dyn FilesystemAdapter>; + /// Native-only. Wasm transport handles its own TLS via the underlying + /// JS WebSocket implementation. + fn tls(&self) -> Option<&dyn TlsCertProvider>; +} diff --git a/crates/agentsync-core/src/host/native/crypto.rs b/crates/agentsync-core/src/host/native/crypto.rs new file mode 100644 index 0000000..bc929e1 --- /dev/null +++ b/crates/agentsync-core/src/host/native/crypto.rs @@ -0,0 +1,51 @@ +//! Native crypto: OsRng, file-backed Identity Signer, rcgen-based TLS. + +use crate::error::Result; +use crate::host::crypto::{Rng, Signer, TlsCert, TlsCertProvider}; +use crate::identity::{Identity, Pubkey, SIGNATURE_LEN}; +use crate::tls; +use async_trait::async_trait; +use rand_core::{OsRng, RngCore}; +use std::path::Path; + +pub struct OsRngProvider; + +impl Rng for OsRngProvider { + fn fill_bytes(&self, buf: &mut [u8]) { + OsRng.fill_bytes(buf); + } +} + +/// Adapts a [`crate::Identity`] (file or ssh-agent) into the [`Signer`] +/// trait. The async signature on `Identity::sign` matches the trait +/// directly. +pub struct IdentitySigner { + inner: Identity, +} + +impl IdentitySigner { + pub fn new(inner: Identity) -> Self { + Self { inner } + } +} + +#[async_trait(?Send)] +impl Signer for IdentitySigner { + async fn sign(&self, msg: &[u8]) -> Result<[u8; SIGNATURE_LEN]> { + self.inner.sign(msg).await + } + + fn pubkey(&self) -> Pubkey { + self.inner.pubkey() + } +} + +pub struct NativeTlsProvider; + +#[async_trait(?Send)] +impl TlsCertProvider for NativeTlsProvider { + async fn load_or_generate(&self, dir: &Path) -> Result { + let (cert_der, key_der) = tls::load_or_generate_self_signed(dir)?; + Ok(TlsCert { cert_der, key_der }) + } +} diff --git a/crates/agentsync-core/src/host/native/filesystem.rs b/crates/agentsync-core/src/host/native/filesystem.rs new file mode 100644 index 0000000..07a2576 --- /dev/null +++ b/crates/agentsync-core/src/host/native/filesystem.rs @@ -0,0 +1,121 @@ +//! Native filesystem adapter wrapping the existing `NodeFsAdapter` with the +//! two new methods (`create_dir_all`, `remove_dir`) the host trait requires. +//! The original `crate::fs::adapter::FilesystemAdapter` stays in place for +//! the materializer's internal calls during the trait-extraction phase; this +//! adapter is what `Host::filesystem()` returns. + +use crate::error::Result; +use crate::fs::adapter::{ + DirEntry as InnerDirEntry, FilesystemAdapter as InnerFilesystemAdapter, + FsEvent as InnerFsEvent, Watcher as InnerWatcher, +}; +use crate::fs::node_adapter::NodeFsAdapter; +use crate::host::filesystem::{ + DirEntry as HostDirEntry, FilesystemAdapter, FsEvent as HostFsEvent, Watcher as HostWatcher, +}; +use async_trait::async_trait; +use std::path::Path; +use tokio::fs; +use tokio::sync::mpsc::UnboundedSender; + +pub struct NativeFilesystem { + inner: NodeFsAdapter, +} + +impl NativeFilesystem { + pub fn new() -> Self { + Self { + inner: NodeFsAdapter::new(), + } + } +} + +impl Default for NativeFilesystem { + fn default() -> Self { + Self::new() + } +} + +struct WatcherShim { + _inner: Box, +} + +impl HostWatcher for WatcherShim {} + +fn map_dir_entry(e: InnerDirEntry) -> HostDirEntry { + HostDirEntry { + path: e.path, + is_dir: e.is_dir, + size: e.size, + } +} + +#[async_trait(?Send)] +impl FilesystemAdapter for NativeFilesystem { + async fn read(&self, path: &Path) -> Result> { + self.inner.read(path).await + } + + async fn write(&self, path: &Path, content: &[u8]) -> Result<()> { + self.inner.write(path, content).await + } + + async fn delete(&self, path: &Path) -> Result<()> { + self.inner.delete(path).await + } + + async fn list(&self, path: &Path) -> Result> { + Ok(self + .inner + .list(path) + .await? + .into_iter() + .map(map_dir_entry) + .collect()) + } + + async fn exists(&self, path: &Path) -> bool { + self.inner.exists(path).await + } + + async fn hash(&self, path: &Path) -> Result { + self.inner.hash(path).await + } + + async fn create_dir_all(&self, path: &Path) -> Result<()> { + Ok(fs::create_dir_all(path).await?) + } + + async fn remove_dir(&self, path: &Path) -> Result<()> { + match fs::remove_dir(path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } + } + + fn watch( + &self, + path: &Path, + sink: UnboundedSender, + ) -> Result> { + // Bridge: the inner adapter takes its own FsEvent type; spawn a + // forwarder mapping inner events to host events. + let (inner_tx, mut inner_rx) = tokio::sync::mpsc::unbounded_channel::(); + let host_sink = sink.clone(); + tokio::spawn(async move { + while let Some(event) = inner_rx.recv().await { + let mapped = match event { + InnerFsEvent::Touched(p) => HostFsEvent::Touched(p), + InnerFsEvent::Removed(p) => HostFsEvent::Removed(p), + InnerFsEvent::Renamed { from, to } => HostFsEvent::Renamed { from, to }, + }; + if host_sink.send(mapped).is_err() { + break; + } + } + }); + let inner = self.inner.watch(path, inner_tx)?; + Ok(Box::new(WatcherShim { _inner: inner })) + } +} diff --git a/crates/agentsync-core/src/host/native/mod.rs b/crates/agentsync-core/src/host/native/mod.rs new file mode 100644 index 0000000..abb69f0 --- /dev/null +++ b/crates/agentsync-core/src/host/native/mod.rs @@ -0,0 +1,88 @@ +//! Native (tokio + rustls + notify + disk) implementations of the host +//! traits, plus a `native_host` builder that assembles them all into a +//! single `Arc`. + +pub mod crypto; +pub mod filesystem; +pub mod runtime; +pub mod storage; +pub mod transport; + +use crate::host::{ + BlobStorage, Clock, DocStorage, FilesystemAdapter, Host, Listener, Rng, SnapshotStorage, + Spawner, TlsCertProvider, Transport, +}; +use std::path::PathBuf; +use std::sync::Arc; + +use crypto::{NativeTlsProvider, OsRngProvider}; +use filesystem::NativeFilesystem; +use runtime::{SystemClock, TokioSpawner}; +use storage::{NativeBlobStorage, NativeDocStorage, NativeSnapshotStorage}; +use transport::{NativeListener, NativeTransport}; + +/// Bundles every native trait impl into a single `Host`. Constructed by +/// [`native_host`]. +pub struct NativeHost { + spawner: TokioSpawner, + clock: SystemClock, + rng: OsRngProvider, + transport: NativeTransport, + listener: NativeListener, + doc_storage: NativeDocStorage, + blob_storage: NativeBlobStorage, + snapshot_storage: NativeSnapshotStorage, + filesystem: NativeFilesystem, + tls: NativeTlsProvider, +} + +impl Host for NativeHost { + fn spawner(&self) -> &dyn Spawner { + &self.spawner + } + fn clock(&self) -> &dyn Clock { + &self.clock + } + fn rng(&self) -> &dyn Rng { + &self.rng + } + fn transport(&self) -> &dyn Transport { + &self.transport + } + fn listener(&self) -> Option<&dyn Listener> { + Some(&self.listener) + } + fn doc_storage(&self) -> &dyn DocStorage { + &self.doc_storage + } + fn blob_storage(&self) -> &dyn BlobStorage { + &self.blob_storage + } + fn snapshot_storage(&self) -> &dyn SnapshotStorage { + &self.snapshot_storage + } + fn filesystem(&self) -> Option<&dyn FilesystemAdapter> { + Some(&self.filesystem) + } + fn tls(&self) -> Option<&dyn TlsCertProvider> { + Some(&self.tls) + } +} + +/// Build a `NativeHost` rooted at `storage_path` (the `.agentsync/` dir for +/// this vault). Storage adapters share the root; transport / listener / +/// filesystem are runtime singletons. +pub fn native_host(storage_path: PathBuf) -> Arc { + Arc::new(NativeHost { + spawner: TokioSpawner, + clock: SystemClock, + rng: OsRngProvider, + transport: NativeTransport, + listener: NativeListener, + doc_storage: NativeDocStorage::new(&storage_path), + blob_storage: NativeBlobStorage::new(&storage_path), + snapshot_storage: NativeSnapshotStorage::new(&storage_path), + filesystem: NativeFilesystem::new(), + tls: NativeTlsProvider, + }) +} diff --git a/crates/agentsync-core/src/host/native/runtime.rs b/crates/agentsync-core/src/host/native/runtime.rs new file mode 100644 index 0000000..40799d0 --- /dev/null +++ b/crates/agentsync-core/src/host/native/runtime.rs @@ -0,0 +1,53 @@ +//! Native runtime backed by tokio. + +use crate::host::runtime::{Clock, SpawnHandle, SpawnHandleImpl, Spawner}; +use async_trait::async_trait; +use futures_util::future::BoxFuture; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::task::JoinHandle; + +pub struct TokioSpawner; + +impl Spawner for TokioSpawner { + fn spawn(&self, fut: BoxFuture<'static, ()>) -> SpawnHandle { + let handle = tokio::spawn(fut); + SpawnHandle::new(Box::new(TokioSpawnHandle { + handle: Some(handle), + })) + } +} + +struct TokioSpawnHandle { + handle: Option>, +} + +#[async_trait(?Send)] +impl SpawnHandleImpl for TokioSpawnHandle { + fn abort(mut self: Box) { + if let Some(h) = self.handle.take() { + h.abort(); + } + } + async fn join(mut self: Box) { + if let Some(h) = self.handle.take() { + let _ = h.await; + } + } +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_ms(&self) -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) + } + + fn sleep(&self, d: Duration) -> BoxFuture<'static, ()> { + Box::pin(async move { + tokio::time::sleep(d).await; + }) + } +} diff --git a/crates/agentsync-core/src/host/native/storage.rs b/crates/agentsync-core/src/host/native/storage.rs new file mode 100644 index 0000000..36a2348 --- /dev/null +++ b/crates/agentsync-core/src/host/native/storage.rs @@ -0,0 +1,185 @@ +//! Native storage adapters: tokio::fs-backed implementations of the +//! `DocStorage` / `BlobStorage` / `SnapshotStorage` traits. +//! +//! These are bytes-level adapters. The Doc serialization happens in the +//! Vault layer; this module is just "hand me bytes, I write bytes." The +//! existing `crate::store` types still hold the path layout knowledge — +//! these wrappers just expose them through the Host trait surface. + +use crate::error::Result; +use crate::host::storage::{BlobStorage, DocStorage, SnapshotEntry, SnapshotStorage}; +use crate::store::BlobStore; +use crate::store::snapshots::decode_b64_heads; +use async_trait::async_trait; +use automerge::ChangeHash; +use base64::Engine; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use tokio::fs; + +pub struct NativeDocStorage { + root: PathBuf, +} + +impl NativeDocStorage { + pub fn new(root: impl AsRef) -> Self { + Self { + root: root.as_ref().to_path_buf(), + } + } + + fn doc_path(&self) -> PathBuf { + self.root.join("doc.bin") + } + + fn tmp_path(&self) -> PathBuf { + self.root.join("doc.bin.tmp") + } +} + +#[async_trait(?Send)] +impl DocStorage for NativeDocStorage { + async fn load(&self) -> Result>> { + match fs::read(self.doc_path()).await { + Ok(b) => Ok(Some(b)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + async fn save(&self, bytes: &[u8]) -> Result<()> { + let tmp = self.tmp_path(); + fs::write(&tmp, bytes).await?; + fs::rename(&tmp, self.doc_path()).await?; + Ok(()) + } + + async fn ensure_ready(&self) -> Result<()> { + fs::create_dir_all(&self.root).await?; + fs::create_dir_all(self.root.join("snapshots")).await?; + fs::create_dir_all(self.root.join("blobs")).await?; + Ok(()) + } +} + +pub struct NativeBlobStorage { + inner: BlobStore, +} + +impl NativeBlobStorage { + pub fn new(root: impl AsRef) -> Self { + Self { + inner: BlobStore::new(root), + } + } +} + +#[async_trait(?Send)] +impl BlobStorage for NativeBlobStorage { + async fn has(&self, hash: &str) -> bool { + self.inner.has(hash).await + } + + async fn get(&self, hash: &str) -> Result> { + self.inner.get(hash).await + } + + async fn put(&self, bytes: &[u8]) -> Result { + self.inner.put(bytes).await + } + + async fn put_with_hash(&self, hash: &str, bytes: &[u8]) -> Result<()> { + self.inner.put_with_hash(hash, bytes).await + } + + async fn ensure_ready(&self) -> Result<()> { + self.inner.ensure_dirs().await + } +} + +/// On-disk JSON shape for the snapshot index. Mirrors the existing +/// `crate::store::snapshots::SnapshotIndexFile` layout — adapter writes the +/// same bytes the legacy `SnapshotIndex::write` would, so old vaults remain +/// readable and vice versa. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct OnDiskEntry { + label: String, + heads: String, + created_at: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct OnDiskFile { + schema_version: i64, + labels: Vec, +} + +pub struct NativeSnapshotStorage { + root: PathBuf, +} + +impl NativeSnapshotStorage { + pub fn new(storage_root: impl AsRef) -> Self { + Self { + root: storage_root.as_ref().join("snapshots"), + } + } + + fn index_path(&self) -> PathBuf { + self.root.join("index.json") + } +} + +fn encode_heads(heads: &[ChangeHash]) -> String { + let mut bytes = Vec::with_capacity(heads.len() * 32); + for h in heads { + bytes.extend_from_slice(h.as_ref()); + } + base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes) +} + +#[async_trait(?Send)] +impl SnapshotStorage for NativeSnapshotStorage { + async fn read(&self) -> Result> { + match fs::read(self.index_path()).await { + Ok(bytes) => { + let f: OnDiskFile = serde_json::from_slice(&bytes)?; + Ok(f.labels + .into_iter() + .map(|e| SnapshotEntry { + label: e.label, + heads: decode_b64_heads(&e.heads).unwrap_or_default(), + created_at_ms: e.created_at, + }) + .collect()) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(e) => Err(e.into()), + } + } + + async fn write(&self, entries: &[SnapshotEntry]) -> Result<()> { + fs::create_dir_all(&self.root).await?; + let on_disk = OnDiskFile { + schema_version: 1, + labels: entries + .iter() + .map(|e| OnDiskEntry { + label: e.label.clone(), + heads: encode_heads(&e.heads), + created_at: e.created_at_ms, + }) + .collect(), + }; + let json = serde_json::to_string_pretty(&on_disk)?; + let tmp = self.root.join(".index.json.tmp"); + fs::write(&tmp, json.as_bytes()).await?; + fs::rename(&tmp, self.index_path()).await?; + Ok(()) + } + + async fn ensure_ready(&self) -> Result<()> { + fs::create_dir_all(&self.root).await?; + Ok(()) + } +} diff --git a/crates/agentsync-core/src/host/native/transport.rs b/crates/agentsync-core/src/host/native/transport.rs new file mode 100644 index 0000000..73fa09f --- /dev/null +++ b/crates/agentsync-core/src/host/native/transport.rs @@ -0,0 +1,58 @@ +//! Native WebSocket transport — placeholder. +//! +//! The real implementation will move the body of +//! `crate::net::client::open_websocket` and `crate::net::server::Server::bind` +//! into [`NativeTransport`] / [`NativeListener`] during Phase 1.3 of the +//! wasm-parity refactor (the cutover where Vault routes through Host). For +//! Phase 1.1+1.2 the trait surface compiles and is wired into [`Host`], but +//! `Vault` keeps calling the existing `net::client` / `net::server` +//! free functions — so neither method here is reached yet. + +use crate::error::{Error, Result}; +use crate::host::transport::{Acceptor, Conn, ConnectOpts, Listener, TlsConfig, Transport}; +use async_trait::async_trait; +use bytes::Bytes; +use std::net::SocketAddr; + +pub struct NativeTransport; + +#[async_trait(?Send)] +impl Transport for NativeTransport { + async fn connect(&self, _url: &str, _opts: ConnectOpts) -> Result> { + Err(Error::Other( + "NativeTransport::connect not yet wired (cutover lands in Phase 1.3)".into(), + )) + } +} + +pub struct NativeListener; + +#[async_trait(?Send)] +impl Listener for NativeListener { + async fn bind(&self, _addr: SocketAddr, _tls: Option) -> Result> { + Err(Error::Other( + "NativeListener::bind not yet wired (cutover lands in Phase 1.3)".into(), + )) + } +} + +// Placeholder Conn / Acceptor types kept here to make the trait shapes +// concrete for compilation. They are never instantiated in Phase 1.1+1.2. +#[allow(dead_code)] +struct PlaceholderConn; + +#[async_trait(?Send)] +impl Conn for PlaceholderConn { + async fn send(&mut self, _frame: Bytes) -> Result<()> { + Err(Error::Other("placeholder".into())) + } + async fn recv(&mut self) -> Result> { + Err(Error::Other("placeholder".into())) + } + fn channel_binding(&self) -> Option<[u8; 32]> { + None + } + async fn close(self: Box) -> Result<()> { + Ok(()) + } +} diff --git a/crates/agentsync-core/src/host/runtime.rs b/crates/agentsync-core/src/host/runtime.rs new file mode 100644 index 0000000..6b5b2b7 --- /dev/null +++ b/crates/agentsync-core/src/host/runtime.rs @@ -0,0 +1,53 @@ +//! Async runtime primitives. Native targets back these with `tokio`; wasm +//! backs them with `wasm-bindgen-futures` + `futures::channel`. Vault and the +//! networking layer never reach for `tokio::*` directly — they go through +//! these traits so the same code runs on both targets. +//! +//! `?Send` is on the async-trait attributes because wasm32 has no real +//! threads and `JsValue`-bearing futures are not `Send`. Native impls satisfy +//! the bound trivially. + +use async_trait::async_trait; +use futures_util::future::BoxFuture; +use std::time::Duration; + +/// Spawns detached background work. The returned [`SpawnHandle`] can be +/// dropped (task keeps running) or awaited (waits for completion). +pub trait Spawner: Send + Sync + 'static { + fn spawn(&self, fut: BoxFuture<'static, ()>) -> SpawnHandle; +} + +pub struct SpawnHandle { + inner: Box, +} + +impl SpawnHandle { + pub fn new(inner: Box) -> Self { + Self { inner } + } + + pub fn abort(self) { + self.inner.abort(); + } + + pub async fn join(self) { + self.inner.join().await; + } +} + +#[async_trait(?Send)] +pub trait SpawnHandleImpl: Send + 'static { + fn abort(self: Box); + async fn join(self: Box); +} + +/// Wall-clock + monotonic time. Native uses `std::time` and `tokio::time`; +/// wasm uses `js_sys::Date::now()` and a `Promise`-wrapped `setTimeout`. +pub trait Clock: Send + Sync + 'static { + /// Milliseconds since the Unix epoch. Used for label timestamps and the + /// TLS cert `not_before` / `not_after` fields. + fn now_ms(&self) -> i64; + /// Resolves after roughly `d` real time. May fire late on a busy + /// runtime; callers must not rely on tight tolerance. + fn sleep(&self, d: Duration) -> BoxFuture<'static, ()>; +} diff --git a/crates/agentsync-core/src/host/storage.rs b/crates/agentsync-core/src/host/storage.rs new file mode 100644 index 0000000..972cfee --- /dev/null +++ b/crates/agentsync-core/src/host/storage.rs @@ -0,0 +1,58 @@ +//! Storage abstractions for the three on-disk artifacts a vault keeps: +//! `doc.bin` (Automerge save), `blobs/` (CAS attachments), and +//! `snapshots.json` (label index). Native impl uses `tokio::fs`; wasm impl +//! delegates to a JS-supplied object that backs onto OPFS, IndexedDB, or +//! `node:fs` depending on the runtime. +//! +//! Implementations MUST be atomic at the granularity of one full `save` / +//! `put` call — torn writes are unrecoverable. The native impl achieves +//! this via write-to-tmp + rename; OPFS uses `FileSystemSyncAccessHandle`'s +//! atomic `flush`. IndexedDB transactions are atomic by construction. + +use crate::error::Result; +use async_trait::async_trait; +use automerge::ChangeHash; + +/// Persists the full Automerge document. Tnly one `doc.bin` exists per +/// vault; the storage adapter is responsible for atomic replacement. +#[async_trait(?Send)] +pub trait DocStorage: Send + Sync + 'static { + /// Load saved bytes. `Ok(None)` when no doc has ever been saved. + async fn load(&self) -> Result>>; + /// Replace doc.bin atomically. + async fn save(&self, bytes: &[u8]) -> Result<()>; + /// Create any directories the implementation needs. Idempotent. + async fn ensure_ready(&self) -> Result<()>; +} + +/// Content-addressed blob store. Used for binary attachments (anything +/// outside the small-file inline budget). +#[async_trait(?Send)] +pub trait BlobStorage: Send + Sync + 'static { + async fn has(&self, hash: &str) -> bool; + /// Returns the stored bytes for `hash`. `Err` if not present. + async fn get(&self, hash: &str) -> Result>; + /// Hash and store; returns the hex-encoded SHA-256. + async fn put(&self, bytes: &[u8]) -> Result; + /// Store under a caller-supplied hash (used when the hash is already + /// known from a network frame). Implementations should verify the hash + /// matches before persisting. + async fn put_with_hash(&self, hash: &str, bytes: &[u8]) -> Result<()>; + async fn ensure_ready(&self) -> Result<()>; +} + +/// Snapshot / label index. One JSON-ish file per vault holding the named +/// historical heads pointers. +#[async_trait(?Send)] +pub trait SnapshotStorage: Send + Sync + 'static { + async fn read(&self) -> Result>; + async fn write(&self, entries: &[SnapshotEntry]) -> Result<()>; + async fn ensure_ready(&self) -> Result<()>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotEntry { + pub label: String, + pub heads: Vec, + pub created_at_ms: i64, +} diff --git a/crates/agentsync-core/src/host/transport.rs b/crates/agentsync-core/src/host/transport.rs new file mode 100644 index 0000000..9828b78 --- /dev/null +++ b/crates/agentsync-core/src/host/transport.rs @@ -0,0 +1,77 @@ +//! Outbound + inbound WebSocket transport, abstracted away from +//! tokio-tungstenite + tokio_rustls. Native impl wraps the existing +//! `net::transport` types; wasm impl bridges to a JS-supplied callback that +//! returns a duplex of msgpack frames. +//! +//! The transport layer is responsible for terminating TLS (when the URL is +//! `wss://`) and surfacing the negotiated peer certificate fingerprint via +//! [`Conn::channel_binding`]. The handshake transcript builder consumes that +//! fingerprint to bind the auth handshake to the underlying TLS channel. +//! When the transport runs without TLS (plain `ws://` or browser WebSocket +//! — neither exposes peer certs), `channel_binding` returns `None` and the +//! handshake includes an empty fingerprint. + +use crate::error::Result; +use crate::identity::Pubkey; +use async_trait::async_trait; +use bytes::Bytes; +use std::net::SocketAddr; + +/// Outbound connector. One impl per runtime; multiple [`Conn`]s may be +/// open against a single transport. +#[async_trait(?Send)] +pub trait Transport: Send + Sync + 'static { + /// Open a new connection. `url` is `wss://host:port` or `ws://host:port`. + async fn connect(&self, url: &str, opts: ConnectOpts) -> Result>; +} + +#[derive(Default, Clone)] +pub struct ConnectOpts { + /// Pin the hub's identity pubkey. The handshake will reject connections + /// where the hub presents a different pubkey. + pub expected_hub_pubkey: Option, +} + +/// A single duplex connection over which msgpack-encoded [`crate::Frame`] +/// values flow. Implementations buffer at most one frame internally; back +/// pressure is signalled via `send` returning slowly. +#[async_trait(?Send)] +pub trait Conn: Send + 'static { + /// Send one binary frame. Errors are non-recoverable; the conn is dead. + async fn send(&mut self, frame: Bytes) -> Result<()>; + /// Receive the next frame. `Ok(None)` means the peer closed cleanly. + async fn recv(&mut self) -> Result>; + /// SHA-256 of the peer's TLS certificate, if any. Returns `None` when + /// the underlying transport is plain or doesn't expose the cert (browser + /// WebSocket). Used by the handshake to bind to the TLS channel. + fn channel_binding(&self) -> Option<[u8; 32]>; + /// Cleanly tear down the connection. Best-effort. + async fn close(self: Box) -> Result<()>; +} + +/// Inbound listener. Native binds a `TcpListener` and optionally wraps with +/// `tokio_rustls`; wasm has no listener at all (only browsers can't listen, +/// but Node could in principle wire `ws.Server` here — left to the host). +#[async_trait(?Send)] +pub trait Listener: Send + Sync + 'static { + /// Bind to `addr` and return an acceptor. Pass the TLS cert + key DER + /// when running over TLS; `None` for plain ws://. + async fn bind(&self, addr: SocketAddr, tls: Option) -> Result>; +} + +/// Server-side TLS material. Cert and key are DER-encoded. +pub struct TlsConfig { + pub cert_der: Vec, + pub key_der: Vec, +} + +/// Per-bind acceptor that yields one [`Conn`] per inbound connection. +#[async_trait(?Send)] +pub trait Acceptor: Send + 'static { + /// Wait for the next inbound connection. `Ok(None)` after `close()`. + async fn accept(&mut self) -> Result>>; + /// The actual bound address (useful when binding to port 0). + fn local_addr(&self) -> SocketAddr; + /// Stop accepting new connections; in-flight conns are unaffected. + async fn close(self: Box) -> Result<()>; +} diff --git a/crates/agentsync-core/src/identity.rs b/crates/agentsync-core/src/identity.rs index 13d872e..a279b7d 100644 --- a/crates/agentsync-core/src/identity.rs +++ b/crates/agentsync-core/src/identity.rs @@ -14,20 +14,26 @@ use crate::error::{Error, Result}; use base64::Engine; -use ed25519_dalek::{Signer, SigningKey, Verifier, VerifyingKey, SECRET_KEY_LENGTH}; +use ed25519_dalek::{SECRET_KEY_LENGTH, Signer, SigningKey, Verifier, VerifyingKey}; use rand_core::OsRng; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; +#[cfg(all(unix, not(target_arch = "wasm32")))] use tokio::io::{AsyncReadExt, AsyncWriteExt}; pub const PUBKEY_LEN: usize = 32; pub const SIGNATURE_LEN: usize = 64; const SSH_KEY_TYPE: &str = "ssh-ed25519"; +#[cfg(all(unix, not(target_arch = "wasm32")))] const SSH_AGENTC_REQUEST_IDENTITIES: u8 = 11; +#[cfg(all(unix, not(target_arch = "wasm32")))] const SSH_AGENT_IDENTITIES_ANSWER: u8 = 12; +#[cfg(all(unix, not(target_arch = "wasm32")))] const SSH_AGENTC_SIGN_REQUEST: u8 = 13; +#[cfg(all(unix, not(target_arch = "wasm32")))] const SSH_AGENT_SIGN_RESPONSE: u8 = 14; +#[cfg(all(unix, not(target_arch = "wasm32")))] const SSH_AGENT_FAILURE: u8 = 5; /// Identity used to authenticate the local peer in handshakes. @@ -90,8 +96,7 @@ impl Identity { /// the seed. Errors for agent-backed identities. pub fn save_to_file(&self, path: &Path) -> Result<()> { let seed = self.seed()?; - let seed_b64 = - base64::engine::general_purpose::STANDARD_NO_PAD.encode(seed); + let seed_b64 = base64::engine::general_purpose::STANDARD_NO_PAD.encode(seed); let body = format!("agentsync-identity-v1 {}\n", seed_b64); write_with_mode(path, body.as_bytes(), 0o600)?; let pub_path = pubkey_sidecar(path); @@ -102,9 +107,10 @@ impl Identity { pub fn load_from_file(path: &Path) -> Result { let bytes = std::fs::read(path)?; let s = std::str::from_utf8(&bytes).map_err(|_| Error::InvalidUtf8)?; - let line = s.lines().next().ok_or_else(|| { - Error::Auth(format!("empty identity file at {}", path.display())) - })?; + let line = s + .lines() + .next() + .ok_or_else(|| Error::Auth(format!("empty identity file at {}", path.display())))?; let rest = line .strip_prefix("agentsync-identity-v1 ") .ok_or_else(|| { @@ -337,11 +343,7 @@ async fn agent_sign(socket: &Path, pubkey: &Pubkey, message: &[u8]) -> Result<[u { use tokio::net::UnixStream; let mut stream = UnixStream::connect(socket).await.map_err(|e| { - Error::Auth(format!( - "ssh-agent connect to {}: {}", - socket.display(), - e - )) + Error::Auth(format!("ssh-agent connect to {}: {}", socket.display(), e)) })?; // Sanity: make sure the agent actually holds this key. Catches the @@ -400,8 +402,7 @@ async fn agent_sign(socket: &Path, pubkey: &Pubkey, message: &[u8]) -> Result<[u Ok(out) } SSH_AGENT_FAILURE => Err(Error::Auth( - "ssh-agent refused to sign (user cancelled, key locked, or wrong key)" - .into(), + "ssh-agent refused to sign (user cancelled, key locked, or wrong key)".into(), )), other => Err(Error::Auth(format!( "ssh-agent returned unexpected response type {}", @@ -425,11 +426,7 @@ pub async fn agent_list_identities_at(socket: &Path) -> Result> { { use tokio::net::UnixStream; let mut stream = UnixStream::connect(socket).await.map_err(|e| { - Error::Auth(format!( - "ssh-agent connect to {}: {}", - socket.display(), - e - )) + Error::Auth(format!("ssh-agent connect to {}: {}", socket.display(), e)) })?; agent_list_identities(&mut stream).await } @@ -455,7 +452,9 @@ async fn agent_list_identities(stream: &mut tokio::net::UnixStream) -> Result Result> { Ok(buf) } +#[cfg(all(unix, not(target_arch = "wasm32")))] fn write_string(out: &mut Vec, s: &[u8]) { out.extend_from_slice(&(s.len() as u32).to_be_bytes()); out.extend_from_slice(s); } +#[cfg(all(unix, not(target_arch = "wasm32")))] fn read_string<'a>(buf: &'a [u8], cursor: &mut usize) -> Result<&'a [u8]> { read_ssh_string(buf, cursor) } diff --git a/crates/agentsync-core/src/lib.rs b/crates/agentsync-core/src/lib.rs index 243f6a5..1628f99 100644 --- a/crates/agentsync-core/src/lib.rs +++ b/crates/agentsync-core/src/lib.rs @@ -1,43 +1,62 @@ //! agentsync-core — real-time directory sync engine using Automerge CRDTs. //! -//! See [`SPEC.md`] in the repo root for the full design. The core API lives on -//! [`Vault`]. +//! See [`SPEC.md`] in the repo root for the full design. On native targets, +//! the high-level API lives on [`Vault`]. On `wasm32-unknown-unknown`, only +//! the wasm-safe subset compiles: CRDT primitives, identity (file-backed), +//! authorized_keys parsing, the protocol Frame codec, and handshake helpers. +//! Tokio sockets, rustls, the `notify` file watcher, and on-disk stores are +//! gated to native builds via `cfg(not(target_arch = "wasm32"))`. pub mod auth; pub mod constants; pub mod doc; pub mod error; -pub mod fs; pub mod identity; pub mod net; pub mod path; pub mod peers_md; + +#[cfg(not(target_arch = "wasm32"))] +pub mod fs; +#[cfg(not(target_arch = "wasm32"))] +pub mod host; +#[cfg(not(target_arch = "wasm32"))] pub mod store; +#[cfg(not(target_arch = "wasm32"))] pub mod tls; +#[cfg(not(target_arch = "wasm32"))] pub mod vault; -pub use auth::{build_transcript, random_nonce, HANDSHAKE_DOMAIN, NONCE_LEN}; +pub use auth::{HANDSHAKE_DOMAIN, NONCE_LEN, build_transcript, random_nonce}; pub use constants::{ - normalize_rendezvous_url, normalize_with_scheme, AUTHORIZED_KEYS_FILE, DEFAULT_LISTEN_ADDR, - DEFAULT_LISTEN_ADDR_NO_TLS, DEFAULT_PORT, USER_IDENTITY_FILENAME, USER_STATE_DIR, + AUTHORIZED_KEYS_FILE, DEFAULT_LISTEN_ADDR, DEFAULT_LISTEN_ADDR_NO_TLS, DEFAULT_PORT, + USER_IDENTITY_FILENAME, USER_STATE_DIR, normalize_rendezvous_url, normalize_with_scheme, }; pub use doc::{ - content_hash, DirectoryMeta, Doc, FileId, FileKind, FileMeta, Label, SCHEMA_VERSION, + DirectoryMeta, Doc, FileId, FileKind, FileMeta, Label, SCHEMA_VERSION, content_hash, }; pub use error::{Error, Result}; -pub use fs::{BindOptions, Binding, NodeFsAdapter}; -pub use identity::{agent_list_identities_at, Identity, Pubkey, PUBKEY_LEN, SIGNATURE_LEN}; - -/// Re-exports for the ssh-agent backend (Phase 3 of AUTH.md). -pub mod agent { - pub use crate::identity::agent_list_identities_at; -} -pub use net::{discover_vault_id, Frame, HelloOp}; +pub use identity::{Identity, PUBKEY_LEN, Pubkey, SIGNATURE_LEN}; +pub use net::{Frame, HelloOp}; pub use peers_md::{ - parse_authorized_keys, parse_peers_md, render_authorized_keys, render_peers_md, - AuthorizedPeer, PEERS_FILE, + AuthorizedPeer, PEERS_FILE, parse_authorized_keys, parse_peers_md, render_authorized_keys, + render_peers_md, }; + +#[cfg(not(target_arch = "wasm32"))] +pub use fs::{BindOptions, Binding, NodeFsAdapter}; +#[cfg(not(target_arch = "wasm32"))] +pub use identity::agent_list_identities_at; +#[cfg(not(target_arch = "wasm32"))] +pub use net::discover_vault_id; +#[cfg(not(target_arch = "wasm32"))] pub use vault::{ CreateOptions, CreatedVault, OpenOptions, ReconnectOptions, SyncHandle, Vault, VaultConfig, VaultEvent, VaultEventKind, VaultId, }; + +/// Re-exports for the ssh-agent backend (Phase 3 of AUTH.md). Native-only. +#[cfg(not(target_arch = "wasm32"))] +pub mod agent { + pub use crate::identity::agent_list_identities_at; +} diff --git a/crates/agentsync-core/src/net/client.rs b/crates/agentsync-core/src/net/client.rs index 0fa0186..05b5e79 100644 --- a/crates/agentsync-core/src/net/client.rs +++ b/crates/agentsync-core/src/net/client.rs @@ -1,4 +1,4 @@ -use crate::auth::{build_transcript, random_nonce, NONCE_LEN}; +use crate::auth::{NONCE_LEN, build_transcript, random_nonce}; use crate::error::{Error, Result}; use crate::identity::{Identity, Pubkey}; use crate::net::protocol::{Frame, HelloOp}; @@ -8,17 +8,17 @@ use crate::vault::SyncHandle; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use rustls_pki_types::ServerName; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::net::TcpStream; -use tokio::sync::{mpsc, oneshot, Notify}; +use tokio::sync::{Notify, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_rustls::TlsConnector; +use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::client_async; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tracing::{debug, info, warn}; type WsStream = WebSocketStream; @@ -112,9 +112,9 @@ async fn probe_handshake( let (vault_id, hub_pubkey, vault_name, _) = run_handshake(&mut writer, &mut reader, identity, cert_fp).await?; - let ws = writer.reunite(reader).map_err(|e| { - Error::Network(format!("reunite ws after handshake: {}", e)) - })?; + let ws = writer + .reunite(reader) + .map_err(|e| Error::Network(format!("reunite ws after handshake: {}", e)))?; Ok((vault_id, hub_pubkey, vault_name, ws)) } @@ -163,8 +163,7 @@ where // back to the hub identity signature alone, which a MITM cannot // forge — the signature covers `hub_identity_pubkey`, and the hub // identity is TOFU-pinned per vault. - let hub_disabled_binding = advertised_fp.len() == 32 - && advertised_fp.iter().all(|b| *b == 0); + let hub_disabled_binding = advertised_fp.len() == 32 && advertised_fp.iter().all(|b| *b == 0); if !hub_disabled_binding && advertised_fp != expected_cert_fp.as_slice() { return Err(Error::Auth(format!( "tls cert fingerprint mismatch: advertised {} bytes, observed {}", @@ -475,9 +474,7 @@ pub(crate) async fn handle_inbound( } } -async fn read_one_frame( - reader: &mut SplitStream>, -) -> Result +async fn read_one_frame(reader: &mut SplitStream>) -> Result where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, { diff --git a/crates/agentsync-core/src/net/mod.rs b/crates/agentsync-core/src/net/mod.rs index 63862df..87f0415 100644 --- a/crates/agentsync-core/src/net/mod.rs +++ b/crates/agentsync-core/src/net/mod.rs @@ -1,8 +1,15 @@ pub mod protocol; + +#[cfg(not(target_arch = "wasm32"))] pub mod client; +#[cfg(not(target_arch = "wasm32"))] pub mod server; +#[cfg(not(target_arch = "wasm32"))] pub mod transport; pub use protocol::*; -pub use client::{discover_vault_id, ClientConn}; + +#[cfg(not(target_arch = "wasm32"))] +pub use client::{ClientConn, discover_vault_id}; +#[cfg(not(target_arch = "wasm32"))] pub use server::{Server, ServerTls}; diff --git a/crates/agentsync-core/src/net/server.rs b/crates/agentsync-core/src/net/server.rs index b1b6780..b5b7d99 100644 --- a/crates/agentsync-core/src/net/server.rs +++ b/crates/agentsync-core/src/net/server.rs @@ -1,4 +1,4 @@ -use crate::auth::{build_transcript, random_nonce, NONCE_LEN}; +use crate::auth::{NONCE_LEN, build_transcript, random_nonce}; use crate::error::{Error, Result}; use crate::identity::{Identity, Pubkey}; use crate::net::client::handle_inbound; @@ -13,12 +13,12 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; -use tokio::sync::{broadcast, mpsc, Mutex}; +use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; use tokio_rustls::TlsAcceptor; +use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::accept_async; use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::WebSocketStream; use tracing::{debug, info, warn}; type AcceptedStream = MaybeTlsServerStream; @@ -236,11 +236,7 @@ fn log_authorized_diff(prev: &HashMap, current: &[AuthorizedPeer } fn label_or_unlabeled(s: &str) -> &str { - if s.is_empty() { - "(unlabeled)" - } else { - s - } + if s.is_empty() { "(unlabeled)" } else { s } } async fn handle_peer( @@ -278,10 +274,7 @@ async fn handle_peer( op, } => (peer_identity_pubkey, peer_nonce, op), Frame::Error { message } => { - return Err(Error::Protocol(format!( - "peer reported error: {}", - message - ))); + return Err(Error::Protocol(format!("peer reported error: {}", message))); } _ => return Err(Error::Protocol("expected HelloPeer".into())), }; @@ -328,10 +321,7 @@ async fn handle_peer( let peer_sig = match frame { Frame::ProofPeer { sig } => sig, Frame::Error { message } => { - return Err(Error::Protocol(format!( - "peer reported error: {}", - message - ))); + return Err(Error::Protocol(format!("peer reported error: {}", message))); } _ => return Err(Error::Protocol("expected ProofPeer".into())), }; diff --git a/crates/agentsync-core/src/net/transport.rs b/crates/agentsync-core/src/net/transport.rs index 2435779..96fed52 100644 --- a/crates/agentsync-core/src/net/transport.rs +++ b/crates/agentsync-core/src/net/transport.rs @@ -57,19 +57,13 @@ impl AsyncWrite for MaybeTlsClientStream { MaybeTlsClientStream::Tls(s) => Pin::new(s).poll_write(cx, buf), } } - fn poll_flush( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { MaybeTlsClientStream::Plain(s) => Pin::new(s).poll_flush(cx), MaybeTlsClientStream::Tls(s) => Pin::new(s).poll_flush(cx), } } - fn poll_shutdown( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { MaybeTlsClientStream::Plain(s) => Pin::new(s).poll_shutdown(cx), MaybeTlsClientStream::Tls(s) => Pin::new(s).poll_shutdown(cx), @@ -101,19 +95,13 @@ impl AsyncWrite for MaybeTlsServerStream { MaybeTlsServerStream::Tls(s) => Pin::new(s).poll_write(cx, buf), } } - fn poll_flush( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { MaybeTlsServerStream::Plain(s) => Pin::new(s).poll_flush(cx), MaybeTlsServerStream::Tls(s) => Pin::new(s).poll_flush(cx), } } - fn poll_shutdown( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { MaybeTlsServerStream::Plain(s) => Pin::new(s).poll_shutdown(cx), MaybeTlsServerStream::Tls(s) => Pin::new(s).poll_shutdown(cx), diff --git a/crates/agentsync-core/src/path.rs b/crates/agentsync-core/src/path.rs index 8dec716..6385993 100644 --- a/crates/agentsync-core/src/path.rs +++ b/crates/agentsync-core/src/path.rs @@ -7,7 +7,10 @@ pub fn normalize(input: &str) -> Result { return Err(Error::InvalidPath("path is empty".into())); } - let unified: String = input.chars().map(|c| if c == '\\' { '/' } else { c }).collect(); + let unified: String = input + .chars() + .map(|c| if c == '\\' { '/' } else { c }) + .collect(); let nfc: String = unified.nfc().collect(); // Reject absolute paths and parent traversal. diff --git a/crates/agentsync-core/src/store/mod.rs b/crates/agentsync-core/src/store/mod.rs index 294df6b..fb3cf1a 100644 --- a/crates/agentsync-core/src/store/mod.rs +++ b/crates/agentsync-core/src/store/mod.rs @@ -1,7 +1,7 @@ -pub mod doc_store; pub mod blobs; +pub mod doc_store; pub mod snapshots; -pub use doc_store::DocStore; pub use blobs::BlobStore; +pub use doc_store::DocStore; pub use snapshots::SnapshotIndex; diff --git a/crates/agentsync-core/src/store/snapshots.rs b/crates/agentsync-core/src/store/snapshots.rs index c5fcfae..81a8e72 100644 --- a/crates/agentsync-core/src/store/snapshots.rs +++ b/crates/agentsync-core/src/store/snapshots.rs @@ -1,10 +1,10 @@ use crate::doc::Label; use crate::error::Result; +use automerge::ChangeHash; +use base64::Engine; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tokio::fs; -use base64::Engine; -use automerge::ChangeHash; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SnapshotEntry { diff --git a/crates/agentsync-core/src/vault.rs b/crates/agentsync-core/src/vault.rs index f35bd30..ed98f6e 100644 --- a/crates/agentsync-core/src/vault.rs +++ b/crates/agentsync-core/src/vault.rs @@ -1,4 +1,5 @@ -use crate::doc::{content_hash, Doc, FileKind, FileMeta, Label}; +use crate::constants::AUTHORIZED_KEYS_FILE; +use crate::doc::{Doc, FileKind, FileMeta, Label, content_hash}; use crate::error::{Error, Result}; use crate::fs::adapter::{FilesystemAdapter, FsEvent}; use crate::fs::binding::{BindOptions, Binding}; @@ -7,20 +8,19 @@ use crate::identity::{Identity, Pubkey}; use crate::net::client::ClientConn; use crate::net::protocol::Frame; use crate::net::server::{Server, ServerTls}; -use crate::constants::AUTHORIZED_KEYS_FILE; -use crate::peers_md::{parse_authorized_keys, render_authorized_keys, AuthorizedPeer}; +use crate::peers_md::{AuthorizedPeer, parse_authorized_keys, render_authorized_keys}; use crate::store::{BlobStore, DocStore, SnapshotIndex}; use async_trait::async_trait; -use automerge::sync::{self as amsync, SyncDoc}; use automerge::ChangeHash; +use automerge::sync::{self as amsync, SyncDoc}; use std::collections::HashMap; use std::net::SocketAddr; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use tokio::sync::{broadcast, mpsc, oneshot, Mutex, Notify}; +use std::sync::atomic::{AtomicU64, Ordering}; +use tokio::sync::{Mutex, Notify, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; -use tokio::time::{interval, Duration}; +use tokio::time::{Duration, interval}; use tracing::{debug, info, warn}; use uuid::Uuid; @@ -288,13 +288,7 @@ impl Vault { inner: inner.clone(), }; v.start_save_loop(); - Ok(( - v, - CreatedVault { - vault_id, - identity, - }, - )) + Ok((v, CreatedVault { vault_id, identity })) } /// The display name carried in the handshake (from `OpenOptions.name`). @@ -604,14 +598,7 @@ impl Vault { let inner = self.inner.clone(); let handle = tokio::spawn(async move { 'outer: loop { - let conn = match connect_with_backoff( - &inner, - &url, - &opts, - &mut shutdown_rx, - ) - .await - { + let conn = match connect_with_backoff(&inner, &url, &opts, &mut shutdown_rx).await { ConnectResult::Connected(c) => c, ConnectResult::Shutdown => return, ConnectResult::GaveUp => { @@ -675,11 +662,7 @@ impl Vault { self.listen_with_tls(addr, ServerTls::Disabled).await } - async fn listen_with_tls( - &mut self, - addr: SocketAddr, - tls: ServerTls, - ) -> Result { + async fn listen_with_tls(&mut self, addr: SocketAddr, tls: ServerTls) -> Result { let server = Server::bind( addr, self.inner.vault_id.clone(), @@ -709,11 +692,7 @@ impl Vault { // ---------- binding ---------- - pub async fn bind_directory( - &mut self, - path: &Path, - opts: BindOptions, - ) -> Result> { + pub async fn bind_directory(&mut self, path: &Path, opts: BindOptions) -> Result> { let adapter: Arc = Arc::new(NodeFsAdapter::new()); let mut binding = Binding::new(path, opts.clone(), adapter.clone()); @@ -1019,10 +998,8 @@ async fn materialize_inner(inner: &Arc, binding: &Arc) -> R (doc.list_files()?, doc.list_directories()?) }; tracing::trace!(count = files.len(), "materialize: scanning live files"); - let live: HashMap = - files.into_iter().map(|m| (m.path.clone(), m)).collect(); - let live_dirs: std::collections::HashSet = - dirs.into_iter().map(|d| d.path).collect(); + let live: HashMap = files.into_iter().map(|m| (m.path.clone(), m)).collect(); + let live_dirs: std::collections::HashSet = dirs.into_iter().map(|d| d.path).collect(); { let mut materialized_dirs = binding.materialized_dirs.lock().await; @@ -1042,8 +1019,7 @@ async fn materialize_inner(inner: &Arc, binding: &Arc) -> R } let existing: HashMap = binding.materialized.lock().await.clone(); - let last_ingested: HashMap = - binding.last_ingested.lock().await.clone(); + let last_ingested: HashMap = binding.last_ingested.lock().await.clone(); for path in existing.keys() { if !live.contains_key(path) { diff --git a/crates/agentsync-wasm/Cargo.toml b/crates/agentsync-wasm/Cargo.toml new file mode 100644 index 0000000..401e1c7 --- /dev/null +++ b/crates/agentsync-wasm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "agentsync-wasm" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "WebAssembly bindings for agentsync-core" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +agentsync-core = { path = "../agentsync-core", version = "0.1.0", default-features = false } +automerge = { workspace = true } +base64 = { workspace = true } +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +serde = { workspace = true } +serde-wasm-bindgen = "0.6" +js-sys = "0.3" +console_error_panic_hook = "0.1" +getrandom = { version = "0.2", features = ["js"] } +getrandom_04 = { package = "getrandom", version = "0.4", features = ["wasm_js"] } + +# wasm-opt is run separately in CI when binaryen is available; disable the +# automatic download in wasm-pack so local builds work in offline / restricted +# environments. Override at the CI step with `wasm-pack build --release` and +# a wasm-opt invocation, or set `wasm-opt = ["-Oz"]` in this section locally. +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/crates/agentsync-wasm/src/lib.rs b/crates/agentsync-wasm/src/lib.rs new file mode 100644 index 0000000..e311fed --- /dev/null +++ b/crates/agentsync-wasm/src/lib.rs @@ -0,0 +1,516 @@ +//! WebAssembly bindings for `agentsync-core`. +//! +//! Exposes everything a JavaScript-side high-level Vault implementation +//! needs: identities & signing, the protocol Frame codec, the Automerge +//! Doc + sync state machine, label/restore APIs, authorized_keys parsing, +//! and handshake helpers. +//! +//! Networking, storage, and filesystem watching live in the JS layer (so +//! the same wasm runs in browsers, Node, Bun, Electron, Tauri, IDE +//! plugins, etc.) — see `sdks/typescript/src/vault.ts` for the high-level +//! API that orchestrates these primitives into a working sync client. + +#![allow(clippy::new_without_default)] + +use agentsync_core as core; +use core::doc::FileMeta; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen(start)] +fn start() { + console_error_panic_hook::set_once(); +} + +fn js_err(e: E) -> JsError { + JsError::new(&e.to_string()) +} + +// ---- Identity ---- + +/// File-backed ed25519 identity. The JavaScript surface exposes generation, +/// seed import/export, signing, and pubkey access. The ssh-agent backend is +/// native-only and is not reachable from wasm. +#[wasm_bindgen] +pub struct Identity { + inner: core::Identity, +} + +#[wasm_bindgen] +impl Identity { + /// Generate a fresh identity backed by a random ed25519 seed. + #[wasm_bindgen(js_name = generate)] + pub fn generate() -> Self { + Self { + inner: core::Identity::generate(), + } + } + + /// Import an identity from its 32-byte seed. + #[wasm_bindgen(js_name = fromSeed)] + pub fn from_seed(seed: &[u8]) -> Result { + if seed.len() != 32 { + return Err(JsError::new("seed must be 32 bytes")); + } + let mut buf = [0u8; 32]; + buf.copy_from_slice(seed); + Ok(Self { + inner: core::Identity::from_seed(buf), + }) + } + + /// Export the 32-byte seed (file-backed identities only). + #[wasm_bindgen] + pub fn seed(&self) -> Result, JsError> { + Ok(self + .inner + .seed() + .map_err(js_err)? + .to_vec() + .into_boxed_slice()) + } + + /// Public key of this identity. + #[wasm_bindgen] + pub fn pubkey(&self) -> Pubkey { + Pubkey { + inner: self.inner.pubkey(), + } + } + + /// Sign `message` and return the 64-byte ed25519 signature. Async to + /// match the native signature; for file-backed identities completes + /// synchronously. + #[wasm_bindgen] + pub async fn sign(&self, message: Box<[u8]>) -> Result, JsError> { + let sig = self.inner.sign(&message).await.map_err(js_err)?; + Ok(sig.to_vec().into_boxed_slice()) + } +} + +#[wasm_bindgen] +pub struct Pubkey { + inner: core::Pubkey, +} + +#[wasm_bindgen] +impl Pubkey { + /// Construct a pubkey from raw 32 bytes. + #[wasm_bindgen(js_name = fromBytes)] + pub fn from_bytes(bytes: &[u8]) -> Result { + Ok(Self { + inner: core::Pubkey::from_bytes(bytes).map_err(js_err)?, + }) + } + + /// Parse an `ssh-ed25519 ` line. + #[wasm_bindgen(js_name = fromSshString)] + pub fn from_ssh_string(s: &str) -> Result { + Ok(Self { + inner: core::Pubkey::from_ssh_string(s).map_err(js_err)?, + }) + } + + /// Render the pubkey as `ssh-ed25519 `. + #[wasm_bindgen(js_name = toSshString)] + pub fn to_ssh_string(&self) -> String { + self.inner.to_ssh_string() + } + + /// SHA-256 fingerprint string in OpenSSH form: `SHA256:`. + #[wasm_bindgen] + pub fn fingerprint(&self) -> String { + self.inner.fingerprint_sha256() + } + + /// Raw 32-byte pubkey. + #[wasm_bindgen] + pub fn bytes(&self) -> Box<[u8]> { + self.inner.as_bytes().to_vec().into_boxed_slice() + } + + /// Verify a 64-byte signature over `message`. + #[wasm_bindgen] + pub fn verify(&self, message: &[u8], signature: &[u8]) -> bool { + self.inner.verify(message, signature) + } +} + +// ---- authorized_keys ---- + +/// Parse an `authorized_keys` file body and return one entry per authorized +/// peer. Comments and blank lines are skipped. The result is a JS array of +/// `{ pubkey: string, label: string }` objects. +#[wasm_bindgen(js_name = parseAuthorizedKeys)] +pub fn parse_authorized_keys(body: &str) -> Result { + let peers = core::parse_authorized_keys(body); + let out: Vec<_> = peers + .into_iter() + .map(|p| AuthorizedPeerJson { + pubkey: p.pubkey.to_ssh_string(), + label: p.label, + }) + .collect(); + serde_wasm_bindgen::to_value(&out).map_err(js_err) +} + +/// Render a JS array of `{ pubkey, label }` entries as an `authorized_keys` +/// file body. +#[wasm_bindgen(js_name = renderAuthorizedKeys)] +pub fn render_authorized_keys(value: JsValue) -> Result { + let entries: Vec = serde_wasm_bindgen::from_value(value).map_err(js_err)?; + let peers: Vec = entries + .into_iter() + .map(|e| { + let pk = core::Pubkey::from_ssh_string(&e.pubkey).map_err(js_err)?; + Ok::<_, JsError>(core::AuthorizedPeer { + pubkey: pk, + label: e.label, + }) + }) + .collect::>()?; + Ok(core::render_authorized_keys(&peers)) +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct AuthorizedPeerJson { + pubkey: String, + label: String, +} + +// ---- handshake helpers ---- + +/// 32-byte cryptographically secure random nonce, used for handshake transcripts. +#[wasm_bindgen(js_name = randomNonce)] +pub fn random_nonce() -> Box<[u8]> { + core::random_nonce().to_vec().into_boxed_slice() +} + +/// Build the canonical handshake transcript that both sides sign. +#[wasm_bindgen(js_name = buildTranscript)] +pub fn build_transcript( + hub_nonce: &[u8], + peer_nonce: &[u8], + tls_cert_fingerprint: &[u8], + hub_pubkey: &[u8], + peer_pubkey: &[u8], +) -> Result, JsError> { + let hub_n: [u8; 32] = hub_nonce + .try_into() + .map_err(|_| JsError::new("hub_nonce must be 32 bytes"))?; + let peer_n: [u8; 32] = peer_nonce + .try_into() + .map_err(|_| JsError::new("peer_nonce must be 32 bytes"))?; + let hub_pk: [u8; 32] = hub_pubkey + .try_into() + .map_err(|_| JsError::new("hub_pubkey must be 32 bytes"))?; + let peer_pk: [u8; 32] = peer_pubkey + .try_into() + .map_err(|_| JsError::new("peer_pubkey must be 32 bytes"))?; + Ok( + core::build_transcript(&hub_n, &peer_n, tls_cert_fingerprint, &hub_pk, &peer_pk) + .into_boxed_slice(), + ) +} + +// ---- Frame codec ---- + +/// Decode a msgpack-encoded protocol frame and return it as a JS object. +#[wasm_bindgen(js_name = decodeFrame)] +pub fn decode_frame(bytes: &[u8]) -> Result { + let frame = core::Frame::decode(bytes).map_err(js_err)?; + serde_wasm_bindgen::to_value(&frame).map_err(js_err) +} + +/// Encode a JS-side frame object (matching the `Frame` enum shape) to +/// msgpack bytes. +#[wasm_bindgen(js_name = encodeFrame)] +pub fn encode_frame(value: JsValue) -> Result, JsError> { + let frame: core::Frame = serde_wasm_bindgen::from_value(value).map_err(js_err)?; + Ok(frame.encode().map_err(js_err)?.into_boxed_slice()) +} + +// ---- Sync state ---- + +/// Per-peer sync state for the Automerge incremental sync protocol. One +/// `SyncState` per remote peer; encode/decode for persistence so reconnects +/// don't replay the entire history. +#[wasm_bindgen] +pub struct SyncState { + inner: automerge::sync::State, +} + +#[wasm_bindgen] +impl SyncState { + #[wasm_bindgen(constructor)] + pub fn new() -> Self { + Self { + inner: automerge::sync::State::new(), + } + } + + /// Serialize the state for storage. Pair with [`SyncState::decode`]. + #[wasm_bindgen] + pub fn encode(&self) -> Box<[u8]> { + self.inner.encode().into_boxed_slice() + } + + /// Restore a state from previously-encoded bytes. + #[wasm_bindgen] + pub fn decode(bytes: &[u8]) -> Result { + Ok(Self { + inner: automerge::sync::State::decode(bytes).map_err(js_err)?, + }) + } +} + +// ---- Doc / CRDT ---- + +/// Wraps an Automerge-backed agentsync document. Use [`new`] to create a +/// fresh vault doc, [`load`] to restore from saved bytes, and [`save`] to +/// serialize. Mutators apply Automerge changes locally; merge with a remote +/// peer's bytes via [`merge`] (full-doc) or via the sync-state pair +/// `generate_sync_message` / `receive_sync_message` (incremental). +#[wasm_bindgen] +pub struct Doc { + inner: core::Doc, +} + +#[wasm_bindgen] +impl Doc { + /// Create a brand new vault document with the given vault id. + #[wasm_bindgen(constructor)] + pub fn new(vault_id: &str) -> Result { + Ok(Self { + inner: core::Doc::new(vault_id).map_err(js_err)?, + }) + } + + /// Load a saved vault document. + #[wasm_bindgen] + pub fn load(bytes: &[u8]) -> Result { + Ok(Self { + inner: core::Doc::load(bytes).map_err(js_err)?, + }) + } + + /// Serialize the document to bytes. + #[wasm_bindgen] + pub fn save(&mut self) -> Box<[u8]> { + self.inner.save().into_boxed_slice() + } + + /// Save only the changes since the last save. + #[wasm_bindgen(js_name = saveIncremental)] + pub fn save_incremental(&mut self) -> Box<[u8]> { + self.inner.save_incremental().into_boxed_slice() + } + + #[wasm_bindgen(js_name = vaultId)] + pub fn vault_id(&mut self) -> Result { + self.inner.vault_id().map_err(js_err) + } + + /// Current document heads (each 32 bytes). Useful for label snapshots + /// and incremental sync state tracking. + #[wasm_bindgen] + pub fn heads(&mut self) -> Vec { + self.inner + .heads() + .iter() + .map(|h| { + let bytes: [u8; 32] = h.0; + js_sys::Uint8Array::from(&bytes[..]) + }) + .collect() + } + + /// Merge in changes from `other`. Returns true if the local doc changed. + #[wasm_bindgen] + pub fn merge(&mut self, other: &mut Doc) -> Result { + self.inner.merge(&mut other.inner).map_err(js_err) + } + + /// Generate the next outbound sync message for `state`. Returns + /// `undefined` (mapped from `None`) when no message is needed. + #[wasm_bindgen(js_name = generateSyncMessage)] + pub fn generate_sync_message(&mut self, state: &mut SyncState) -> Option> { + self.inner + .generate_sync_message(&mut state.inner) + .map(|m| m.into_boxed_slice()) + } + + /// Apply an inbound sync message. Returns true if heads moved. + #[wasm_bindgen(js_name = receiveSyncMessage)] + pub fn receive_sync_message( + &mut self, + state: &mut SyncState, + bytes: &[u8], + ) -> Result { + self.inner + .receive_sync_message(&mut state.inner, bytes) + .map_err(js_err) + } + + /// Write a UTF-8 text file at `path`. Returns the stable file id. + #[wasm_bindgen(js_name = writeTextFile)] + pub fn write_text_file(&mut self, path: &str, content: &str) -> Result { + self.inner.write_text_file(path, content).map_err(js_err) + } + + /// Read the UTF-8 text file at `path`. + #[wasm_bindgen(js_name = readFile)] + pub fn read_file(&mut self, path: &str) -> Result { + self.inner.read_file(path).map_err(js_err) + } + + #[wasm_bindgen(js_name = fileExists)] + pub fn file_exists(&mut self, path: &str) -> bool { + self.inner.file_exists(path) + } + + #[wasm_bindgen(js_name = deleteFile)] + pub fn delete_file(&mut self, path: &str) -> Result<(), JsError> { + self.inner.delete_file(path).map_err(js_err) + } + + /// Rename a file. The stable file id is preserved. + #[wasm_bindgen(js_name = renameFile)] + pub fn rename_file(&mut self, from: &str, to: &str) -> Result<(), JsError> { + self.inner.rename_file(from, to).map_err(js_err) + } + + /// Write a binary attachment under `path`, content-addressed by `hash` + /// (the JS side computes the SHA-256). Returns the file id. + #[wasm_bindgen(js_name = writeAttachment)] + pub fn write_attachment( + &mut self, + path: &str, + hash: &str, + size: i64, + ) -> Result { + self.inner + .write_attachment(path, hash, size) + .map_err(js_err) + } + + /// List all current files. Returns an array of `FileMeta` objects. + #[wasm_bindgen(js_name = listFiles)] + pub fn list_files(&mut self) -> Result { + let files = self.inner.list_files().map_err(js_err)?; + files_to_js(&files) + } + + /// Create a directory at `path`. Returns the directory id. + #[wasm_bindgen(js_name = createDirectory)] + pub fn create_directory(&mut self, path: &str) -> Result { + self.inner.create_directory(path).map_err(js_err) + } + + /// Delete a directory. + #[wasm_bindgen(js_name = deleteDirectory)] + pub fn delete_directory(&mut self, path: &str, recursive: bool) -> Result<(), JsError> { + self.inner.delete_directory(path, recursive).map_err(js_err) + } + + /// List directories. + #[wasm_bindgen(js_name = listDirectories)] + pub fn list_directories(&mut self) -> Result { + let dirs = self.inner.list_directories().map_err(js_err)?; + serde_wasm_bindgen::to_value(&dirs).map_err(js_err) + } + + // ---- History / labels / restore ---- + + /// Create a named snapshot of the current heads. + #[wasm_bindgen(js_name = createLabel)] + pub fn create_label(&mut self, name: &str) -> Result<(), JsError> { + self.inner.create_label(name).map_err(js_err) + } + + /// Delete a label. + #[wasm_bindgen(js_name = deleteLabel)] + pub fn delete_label(&mut self, name: &str) -> Result<(), JsError> { + self.inner.delete_label(name).map_err(js_err) + } + + /// List all labels with their heads + creation timestamps. + #[wasm_bindgen(js_name = listLabels)] + pub fn list_labels(&mut self) -> Result { + let labels = self.inner.list_labels().map_err(js_err)?; + let out: Vec<_> = labels + .into_iter() + .map(|l| LabelJson { + name: l.name, + heads_b64: encode_heads_b64(&l.heads), + created_at_ms: l.created_at, + }) + .collect(); + serde_wasm_bindgen::to_value(&out).map_err(js_err) + } + + /// Restore the doc to the snapshot named `label`. Additive: produces + /// new forward-going changes that recreate the past state, preserving + /// any concurrent changes from other peers. + #[wasm_bindgen(js_name = restoreToLabel)] + pub fn restore_to_label(&mut self, label: &str) -> Result<(), JsError> { + let heads = self.inner.get_label_heads(label).map_err(js_err)?; + self.inner.restore_to_heads(&heads).map_err(js_err) + } + + /// Restore the doc to the state at `target_ms` (Unix milliseconds). + /// Convenience wrapper over `heads_at_time` + `restore_to_heads`. + /// Takes `f64` so JS callers can pass `Date.now()` directly without + /// having to convert to BigInt; agentsync timestamps fit comfortably + /// in IEEE-754 precision. + #[wasm_bindgen(js_name = restoreToTime)] + pub fn restore_to_time(&mut self, target_ms: f64) -> Result<(), JsError> { + self.inner.restore_to_time(target_ms as i64).map_err(js_err) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct LabelJson { + name: String, + heads_b64: String, + created_at_ms: i64, +} + +fn encode_heads_b64(heads: &[automerge::ChangeHash]) -> String { + use base64::Engine; + let mut bytes = Vec::with_capacity(heads.len() * 32); + for h in heads { + bytes.extend_from_slice(&h.0); + } + base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes) +} + +fn files_to_js(files: &[FileMeta]) -> Result { + serde_wasm_bindgen::to_value(files).map_err(js_err) +} + +// ---- Helpers ---- + +/// SHA-256 of arbitrary bytes, hex-encoded. Matches the on-disk content +/// hash format used by agentsync. +#[wasm_bindgen(js_name = contentHash)] +pub fn content_hash(bytes: &[u8]) -> String { + core::content_hash(bytes) +} + +/// Schema version of vault documents produced by this SDK. +#[wasm_bindgen(js_name = schemaVersion)] +pub fn schema_version() -> u32 { + core::SCHEMA_VERSION as u32 +} + +/// Default rendezvous port (1234). +#[wasm_bindgen(js_name = defaultPort)] +pub fn default_port() -> u16 { + core::DEFAULT_PORT +} + +/// Normalize a rendezvous URL (appends the default port when missing). +#[wasm_bindgen(js_name = normalizeRendezvousUrl)] +pub fn normalize_rendezvous_url(url: &str) -> String { + core::normalize_rendezvous_url(url) +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..714719d --- /dev/null +++ b/deny.toml @@ -0,0 +1,65 @@ +# cargo-deny configuration. +# +# Cargo itself doesn't (yet) implement minimum-release-age — see +# https://github.com/rust-lang/cargo/issues/15973. Until that lands, +# the closest defensive layer is: +# +# 1. `Cargo.lock` checked in + `cargo build --locked` everywhere +# (CI workflows enforce this) — pulls only the exact versions a +# human reviewed at lockfile-bump time. +# 2. cargo-deny enforced on every CI run: +# - advisories: anything in RUSTSEC fails the build +# - bans: yanked crates always fail +# - sources: only crates.io is allowed; no git-only deps +# +# Bumping Cargo.lock IS the gate. Reviewers should check that any new +# transitive version has been on crates.io for at least 7 days (the +# crate page on crates.io shows the publish date). + +[graph] +all-features = false +no-default-features = false + +[advisories] +version = 2 +db-path = "~/.cargo/advisory-db" +db-urls = ["https://github.com/rustsec/advisory-db"] +yanked = "deny" +ignore = [] + +[licenses] +version = 2 +# Permissive set used by the broader Rust ecosystem. `Unicode-3.0` is the +# unicode-ident family, `MPL-2.0` covers webpki / rustls, `OpenSSL` is +# pulled in by the wasm-bindgen-cli dev tooling but not by the runtime. +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unicode-3.0", + "Unicode-DFS-2016", + "MPL-2.0", + "Zlib", + "CC0-1.0", +] +confidence-threshold = 0.93 + +[bans] +# Multiple versions of the same crate are common in transitive trees and +# don't constitute a security finding on their own — warn so reviewers +# notice a regression but don't fail the build. +multiple-versions = "warn" +wildcards = "deny" +# Workspace members reference each other by path without a version +# constraint (e.g. tests/e2e → agentsync-core). That's a normal +# workspace pattern, not a real wildcard external dep, so allow it. +allow-wildcard-paths = true + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] diff --git a/sdks/typescript/.gitignore b/sdks/typescript/.gitignore new file mode 100644 index 0000000..06e6038 --- /dev/null +++ b/sdks/typescript/.gitignore @@ -0,0 +1,3 @@ +node_modules +dist +*.tsbuildinfo diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md new file mode 100644 index 0000000..e0f8c81 --- /dev/null +++ b/sdks/typescript/README.md @@ -0,0 +1,223 @@ +# @agentsync/sdk + +TypeScript / WebAssembly SDK for [agentsync](https://github.com/cjroth/agentsync). +Wraps the same Rust engine that powers the `agentsync` CLI, compiled to wasm32 and shipped with idiomatic TS bindings. + +The SDK exposes a high-level `Vault` API (connect, sync, watch, restore, labels, file ops) **plus** the low-level CRDT / identity / frame primitives, so you can build everything from a one-liner Obsidian plugin to a custom Tauri app on the same engine. + +## Install + +```bash +npm install @agentsync/sdk +# or +bun add @agentsync/sdk +``` + +## High-level Vault API + +```ts +import { + Vault, + Identity, + memoryStorage, + nodeFsStorage, + nodeWsTransport, +} from '@agentsync/sdk'; +import WebSocket from 'ws'; + +// Create a fresh local-only vault (no rendezvous yet). +const vault = await Vault.create({ + storage: nodeFsStorage('./my-vault/.agentsync'), +}); + +await vault.writeTextFile('notes/hello.md', '# hi\n'); +const text = await vault.readTextFile('notes/hello.md'); +console.log(vault.listFiles().map((f) => f.path)); + +// Snapshots & restore +await vault.createLabel('before-cleanup'); +await vault.writeTextFile('notes/hello.md', 'oops'); +await vault.restoreToLabel('before-cleanup'); // back to "# hi\n" +await vault.restoreToTime(Date.now() - 60_000); // 1 minute ago + +// Sync against an existing remote vault hosted by `agentsync --listen`. +const peer = await Vault.create({ + storage: nodeFsStorage('./peer/.agentsync'), + vaultId: '6f1f1aa9-...', // from the hub's `agentsync init` + rendezvousUrl: 'wss://hub.example.com:443', + transport: nodeWsTransport(WebSocket), +}); + +// Subscribe to events (also available as `for await of peer.events()`). +peer.subscribe((e) => console.log(e.kind, e)); + +// Connect once and run the sync loop: +peer.connect(); +// ...or with auto-reconnect + exponential backoff: +peer.connectWithReconnect({ maxAttempts: Infinity, initialBackoffMs: 500 }); + +await peer.disconnect(); +await peer.close(); +``` + +The Vault class: +- Owns the protocol state machine (4-message handshake, Automerge incremental sync, channel-binding fingerprint check, reconnect supervisor). +- Persists `doc.bin`, the identity seed, and per-peer `SyncState` through whatever `StorageAdapter` you supply. +- Mirrors the Rust `Vault` API one-for-one: every Rust method has a camelCase TS twin. + +### Vault methods + +| Method | What it does | +| --- | --- | +| `Vault.create({ storage, identity?, vaultId?, rendezvousUrl?, hubPubkey?, name?, transport? })` | Initialize a new vault. Pass `vaultId` to join an existing remote vault. | +| `Vault.open({ storage, identity?, ... })` | Reopen a vault previously persisted to `storage`. | +| `vault.writeTextFile(path, content)` | Write or update a UTF-8 file. Returns the stable file id. | +| `vault.readTextFile(path)` | Read a file. | +| `vault.deleteFile(path)`, `vault.renameFile(from, to)` | Mutations. | +| `vault.listFiles()`, `vault.fileExists(path)` | Read-only queries. | +| `vault.createDirectory(path)`, `vault.deleteDirectory(path, recursive?)`, `vault.listDirectories()` | Directory ops. | +| `vault.createLabel(name)`, `vault.deleteLabel(name)`, `vault.listLabels()` | Snapshots. | +| `vault.restoreToLabel(name)`, `vault.restoreToTime(unixMs)` | Additive history rewind. | +| `vault.connect()` | Open one rendezvous session, run the sync loop, return when it closes. | +| `vault.connectWithReconnect(opts?)` | Same with exponential backoff. | +| `vault.disconnect()` | Drop the active session and the reconnect supervisor. | +| `vault.subscribe((event) => …)` / `vault.events()` | Vault event stream — `connecting`, `connected`, `disconnected`, `doc-changed`, `sync-progress`, `error`. | +| `vault.isConnected()`, `vault.vaultIdValue()`, `vault.identityRef()` | Accessors. | +| `vault.close()` | Persist, drop the connection, free wasm memory. | + +### Adapters bundled with the SDK + +| Adapter | Use when | +| --- | --- | +| `memoryStorage()` (`MemoryStorage`) | Tests, ephemeral browser sessions. | +| `nodeFsStorage(rootDir)` (`NodeFsStorage`) | Node, Bun, Electron main process, VS Code extensions. Atomic write-tmp-then-rename. | +| `opfsStorage(rootName?)` (`OpfsStorage`) | Browser apps. Uses the Origin Private File System with `FileSystemSyncAccessHandle` from a Web Worker for fast writes; falls back to async writable streams on the main thread. | +| `nodeWsTransport(ws)` | Node WebSocket transport. Pass the [`ws`](https://www.npmjs.com/package/ws) constructor. Exposes the peer TLS cert SHA-256 via `channelBinding()` for end-to-end channel binding. | +| Browser default transport | `globalThis.WebSocket` is used automatically when no `transport` is supplied. Browsers don't expose peer certs, so channel binding falls back to the application-layer signature only. | + +You can supply your own `StorageAdapter` / `TransportAdapter` for unusual hosts (Tauri-backed Rust filesystem, Cloudflare Durable Objects, an Obsidian vault adapter — anything that implements the trait). + +## Low-level primitives + +Everything the high-level Vault is built from is also exported, in case you want +to build something Vault doesn't cover: + +```ts +import { + Identity, + Pubkey, + Doc, + SyncState, + parseAuthorizedKeys, + renderAuthorizedKeys, + randomNonce, + buildTranscript, + encodeFrame, + decodeFrame, + contentHash, + schemaVersion, + defaultPort, + normalizeRendezvousUrl, +} from '@agentsync/sdk'; + +// Two Doc instances + their SyncStates can converge end-to-end without any +// network — the same primitives Vault uses internally: +const a = new Doc('vault-1'); +const b = new Doc('vault-1'); +a.writeTextFile('a.md', 'from A'); +b.writeTextFile('b.md', 'from B'); + +const aState = new SyncState(); +const bState = new SyncState(); +for (let i = 0; i < 50; i++) { + const m1 = a.generateSyncMessage(aState); + if (m1) b.receiveSyncMessage(bState, m1); + const m2 = b.generateSyncMessage(bState); + if (m2) a.receiveSyncMessage(aState, m2); + if (!m1 && !m2) break; +} +// Both docs now have a.md and b.md. +``` + +| Primitive | Notes | +| --- | --- | +| `Identity` | `generate`, `fromSeed`, `seed`, `sign`, `pubkey`. ssh-agent backend is native-only — wasm uses file-backed identities. | +| `Pubkey` | `fromBytes`, `fromSshString`, `toSshString`, `fingerprint`, `bytes`, `verify`. | +| `Doc` | Automerge-backed vault doc. CRUD on files & directories, labels, `heads`, `merge`, `save` / `load` / `saveIncremental`, `generateSyncMessage` / `receiveSyncMessage`, `restoreToLabel`, `restoreToTime`. | +| `SyncState` | Per-peer Automerge sync state. `encode` / `decode` for persistence so reconnects don't replay history. | +| `parseAuthorizedKeys` / `renderAuthorizedKeys` | SSH-style auth file. | +| `encodeFrame` / `decodeFrame` | msgpack codec for the wire protocol. | +| `buildTranscript` / `randomNonce` | Handshake helpers — match the bytes the Rust hub signs. | +| `contentHash` | SHA-256 hex (matches the on-disk content hash format). | +| `schemaVersion`, `defaultPort`, `normalizeRendezvousUrl` | Constants & helpers. | + +## Entry points + +| Import | Target | Use when | +| --- | --- | --- | +| `@agentsync/sdk` | Node + Bun | server, CLI, tests, Electron main, VS Code extensions | +| `@agentsync/sdk/web` | browser bundlers (Vite, webpack, Rollup, esbuild) | frontends, Tauri webview, Obsidian renderer | +| `@agentsync/sdk/wasm` | raw `.wasm` bytes | custom loaders, Cloudflare Workers | +| `@agentsync/sdk/wasm/bundler` | bundler glue + types | when you want the wasm-bindgen surface directly | + +All entry points expose the same TypeScript API. The browser bundle defaults to `globalThis.WebSocket` and OPFS; Node/Bun bundles default to `ws` + `node:fs`. Pass your own adapters explicitly to override. + +## Memory management + +The wasm-bindgen wrappers hold pointers into linear memory. Either let `vault.close()` free everything (Vault internally tracks ownership of its `Doc` and `Identity`), or call `.free()` explicitly when working with raw primitives: + +```ts +{ + using id = Identity.generate(); + // ... +} // automatically freed if your runtime supports `using` + +const doc = new Doc('v'); +try { + // ... +} finally { + doc.free(); +} +``` + +If you pass an externally-owned `identity` to `Vault.create` / `Vault.open`, the Vault will *not* free it — that's your responsibility. + +## Develop + +```bash +bun install +bun run build # wasm-pack (bundler + nodejs targets) + tsc +bun test # 41 unit tests (Bun) +bun run lint # biome +bun run typecheck +AGENTSYNC_BIN=path/to/agentsync bun run test:e2e # 5 e2e tests against a real hub +``` + +The e2e suite runs under Node (not Bun) because Bun's WebSocket client doesn't currently accept the hub's ed25519 self-signed cert. The five tests cover: + +1. Handshake decode (sanity check on the frame codec). +2. Full TS Vault ↔ Rust hub handshake and `connected` event. +3. TS write → file appears on the hub's disk. +4. Hub write → TS Vault reads it. +5. Reconnect after the hub is killed and restarted on the same port. + +## Use cases + +The SDK is designed to be the same primitives across: +- **Node servers / CLI tools** — Use `nodeFsStorage` + `nodeWsTransport`. +- **Browser apps** — Use `opfsStorage` (run wasm in a Worker for sync OPFS handles); browser default transport uses `globalThis.WebSocket`. Hub must serve a real CA cert (browsers can't pin self-signed certs through `WebSocket`). +- **Electron / Tauri / Obsidian plugins** — Hybrid Node + browser environment. Pick `node` or `web` entry point depending on which process you're in; the renderer can use either since `require('@agentsync/sdk')` works in Electron's renderer. +- **VS Code / Cursor / Zed extensions** — VS Code/Cursor extensions run in Node (use the `node` entry); Zed extensions run in a WIT-sandboxed wasm host that this SDK doesn't currently support out of the box. + +## Supply chain + +`bunfig.toml` sets `minimumReleaseAge = 604800` so `bun install` refuses +any npm package whose latest version is less than 7 days old. This +blocks the typical short-lived poisoning window from a stolen +maintainer token before it reaches the lockfile. To bypass for a +specific incident, add the package name to `minimumReleaseAgeExcludes` +in `bunfig.toml` — don't disable globally. + +## License + +MIT or Apache-2.0, at your option. diff --git a/sdks/typescript/biome.json b/sdks/typescript/biome.json new file mode 100644 index 0000000..c8c8dd6 --- /dev/null +++ b/sdks/typescript/biome.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", + "vcs": { + "enabled": true, + "clientKind": "git", + "useIgnoreFile": true + }, + "files": { + "ignore": ["dist", "node_modules"] + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 100 + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": { + "noNonNullAssertion": "warn", + "useImportType": "error" + }, + "suspicious": { + "noExplicitAny": "warn" + } + } + }, + "javascript": { + "formatter": { + "semicolons": "always", + "quoteStyle": "single", + "trailingCommas": "all" + } + } +} diff --git a/sdks/typescript/bun.lock b/sdks/typescript/bun.lock new file mode 100644 index 0000000..430192f --- /dev/null +++ b/sdks/typescript/bun.lock @@ -0,0 +1,50 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@agentsync/sdk", + "devDependencies": { + "@biomejs/biome": "^1.9.4", + "@types/bun": "^1.1.13", + "@types/node": "^22.9.0", + "@types/ws": "^8.5.13", + "typescript": "^5.6.3", + "ws": "^8.18.0", + }, + }, + }, + "packages": { + "@biomejs/biome": ["@biomejs/biome@1.9.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "1.9.4", "@biomejs/cli-darwin-x64": "1.9.4", "@biomejs/cli-linux-arm64": "1.9.4", "@biomejs/cli-linux-arm64-musl": "1.9.4", "@biomejs/cli-linux-x64": "1.9.4", "@biomejs/cli-linux-x64-musl": "1.9.4", "@biomejs/cli-win32-arm64": "1.9.4", "@biomejs/cli-win32-x64": "1.9.4" }, "bin": { "biome": "bin/biome" } }, "sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog=="], + + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@1.9.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw=="], + + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@1.9.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg=="], + + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@1.9.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g=="], + + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@1.9.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA=="], + + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@1.9.4", "", { "os": "linux", "cpu": "x64" }, "sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg=="], + + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@1.9.4", "", { "os": "linux", "cpu": "x64" }, "sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg=="], + + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@1.9.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg=="], + + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@1.9.4", "", { "os": "win32", "cpu": "x64" }, "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA=="], + + "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + + "@types/node": ["@types/node@22.19.18", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-9v00a+dn2yWVsYDEunWC4g/TcRKVq3r8N5FuZp7u0SGrPvdN9c2yXI9bBuf5Fl0hNCb+QTIePTn5pJs2pwBOQQ=="], + + "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], + + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "ws": ["ws@8.20.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA=="], + } +} diff --git a/sdks/typescript/bunfig.toml b/sdks/typescript/bunfig.toml new file mode 100644 index 0000000..c200f3c --- /dev/null +++ b/sdks/typescript/bunfig.toml @@ -0,0 +1,19 @@ +# Supply-chain hardening for `bun install`. +# +# Refuse to install npm packages whose latest registry timestamp is less +# than 7 days (604800 seconds) old. Mirrors pnpm/npm's +# `--minimum-release-age`: the typical short-lived poisoning window from +# a stolen maintainer token (axios March 2026, etc.) is well under a +# day, so a 7-day cooldown blocks them before they can reach the +# lockfile. +# +# To temporarily bypass for a security fix, add the package to +# `minimumReleaseAgeExcludes`. Don't disable globally. + +[install] +minimumReleaseAge = 604800 + +# Packages we explicitly permit to install before the 7-day window has +# elapsed. Empty by default; populate per incident with a comment that +# names the CVE / advisory. +minimumReleaseAgeExcludes = [] diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json new file mode 100644 index 0000000..3aed947 --- /dev/null +++ b/sdks/typescript/package.json @@ -0,0 +1,76 @@ +{ + "name": "@agentsync/sdk", + "version": "0.1.0", + "description": "TypeScript SDK for agentsync — Automerge-backed directory sync", + "license": "MIT OR Apache-2.0", + "homepage": "https://github.com/cjroth/agentsync", + "repository": { + "type": "git", + "url": "git+https://github.com/cjroth/agentsync.git", + "directory": "sdks/typescript" + }, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/web.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "browser": "./dist/web.js", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./web": { + "types": "./dist/web.d.ts", + "default": "./dist/web.js" + }, + "./node": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./wasm": { + "default": "./dist/wasm/agentsync_wasm_bg.wasm" + }, + "./wasm/bundler": { + "types": "./dist/bundler/agentsync_wasm.d.ts", + "default": "./dist/bundler/agentsync_wasm.js" + } + }, + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build:wasm": "node scripts/build-wasm.mjs", + "build:ts": "tsc -p tsconfig.json", + "build": "bun run build:wasm && bun run build:ts", + "lint": "biome check src test", + "lint:fix": "biome check --write src test", + "format": "biome format --write src test", + "typecheck": "tsc --noEmit -p tsconfig.json", + "test": "bun test test/unit", + "test:e2e": "bun run build && node --test --experimental-strip-types --no-warnings=ExperimentalWarning test/e2e/*.test.ts", + "test:all": "bun run test && bun run test:e2e", + "prepublishOnly": "bun run build && bun run lint && bun run typecheck && bun run test" + }, + "devDependencies": { + "@biomejs/biome": "^1.9.4", + "@types/bun": "^1.1.13", + "@types/node": "^22.9.0", + "@types/ws": "^8.5.13", + "typescript": "^5.6.3", + "ws": "^8.18.0" + }, + "engines": { + "bun": ">=1.1", + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "imports": { + "#wasm-nodejs": "./dist/nodejs/agentsync_wasm.js", + "#wasm-bundler": "./dist/bundler/agentsync_wasm.js" + } +} diff --git a/sdks/typescript/scripts/build-wasm.mjs b/sdks/typescript/scripts/build-wasm.mjs new file mode 100644 index 0000000..c682b8c --- /dev/null +++ b/sdks/typescript/scripts/build-wasm.mjs @@ -0,0 +1,95 @@ +// Build the wasm crate twice — once for bundlers (web/Vite/webpack/Rollup), +// once for Node — and emit a single ESM `dist/wasm/` directory that holds +// the raw .wasm + the .d.ts as a sibling to the per-target glue. The TS +// wrappers in src/ pick the right glue at import time via subpath exports. +// +// If `wasm-opt` is on PATH it's run on each emitted .wasm with `-Oz`. CI +// installs binaryen; locally it's optional. + +import { execFileSync, execSync } from 'node:child_process'; +import { copyFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const sdkRoot = resolve(__dirname, '..'); +const repoRoot = resolve(sdkRoot, '..', '..'); +const crate = resolve(repoRoot, 'crates', 'agentsync-wasm'); +const distRoot = resolve(sdkRoot, 'dist'); + +function run(cmd, args, opts = {}) { + console.log(`$ ${cmd} ${args.join(' ')}`); + execFileSync(cmd, args, { stdio: 'inherit', ...opts }); +} + +function maybeWasmOpt(wasmPath) { + try { + execSync('wasm-opt --version', { stdio: 'ignore' }); + } catch { + console.log(' (wasm-opt not on PATH, skipping size optimization)'); + return; + } + const tmp = `${wasmPath}.opt`; + run('wasm-opt', ['-Oz', '--enable-mutable-globals', '-o', tmp, wasmPath]); + copyFileSync(tmp, wasmPath); + rmSync(tmp); +} + +function buildTarget(target, outName) { + const out = join(distRoot, outName); + // wasm-pack rejects non-empty out dirs; clear first. + rmSync(out, { recursive: true, force: true }); + mkdirSync(out, { recursive: true }); + run('wasm-pack', [ + 'build', + crate, + '--target', + target, + '--release', + '--out-dir', + out, + '--out-name', + 'agentsync_wasm', + ]); + for (const entry of readdirSync(out)) { + if (entry.endsWith('.wasm')) { + maybeWasmOpt(join(out, entry)); + const sz = statSync(join(out, entry)).size; + console.log(` ${entry}: ${(sz / 1024).toFixed(1)} KiB`); + } + } + // wasm-pack writes a package.json at the root we don't want shipping — + // the consumer only sees @agentsync/sdk's package.json. Leave the file + // in place (harmless) but the SDK's "files" glob already excludes it. +} + +mkdirSync(distRoot, { recursive: true }); + +buildTarget('bundler', 'bundler'); +buildTarget('nodejs', 'nodejs'); + +// Mirror the raw .wasm into dist/wasm/ so the `./wasm` subpath export +// resolves to a single canonical binary regardless of glue target. +const wasmDir = join(distRoot, 'wasm'); +mkdirSync(wasmDir, { recursive: true }); +copyFileSync( + join(distRoot, 'bundler', 'agentsync_wasm_bg.wasm'), + join(wasmDir, 'agentsync_wasm_bg.wasm'), +); + +// Sanity check that the bundler glue exists where the TS wrappers import +// from. +for (const f of [ + 'bundler/agentsync_wasm.js', + 'bundler/agentsync_wasm.d.ts', + 'bundler/agentsync_wasm_bg.wasm', + 'nodejs/agentsync_wasm.js', + 'nodejs/agentsync_wasm.d.ts', + 'nodejs/agentsync_wasm_bg.wasm', +]) { + if (!existsSync(join(distRoot, f))) { + throw new Error(`expected wasm-pack to emit ${f}`); + } +} + +console.log('\\nwasm build OK'); diff --git a/sdks/typescript/src/adapters/memory-storage.ts b/sdks/typescript/src/adapters/memory-storage.ts new file mode 100644 index 0000000..c253b68 --- /dev/null +++ b/sdks/typescript/src/adapters/memory-storage.ts @@ -0,0 +1,43 @@ +// In-memory StorageAdapter — useful for tests and ephemeral browser +// scenarios. Persistence vanishes when the instance is dropped. + +import type { StorageAdapter } from '../types.js'; + +export class MemoryStorage implements StorageAdapter { + private doc: Uint8Array | null = null; + private syncStates = new Map(); + private identitySeed: Uint8Array | null = null; + private snapshots: Uint8Array | null = null; + + async loadDoc(): Promise { + return this.doc ? new Uint8Array(this.doc) : null; + } + async saveDoc(bytes: Uint8Array): Promise { + this.doc = new Uint8Array(bytes); + } + async loadSyncState(peerKey: string): Promise { + const v = this.syncStates.get(peerKey); + return v ? new Uint8Array(v) : null; + } + async saveSyncState(peerKey: string, bytes: Uint8Array): Promise { + this.syncStates.set(peerKey, new Uint8Array(bytes)); + } + async loadIdentitySeed(): Promise { + return this.identitySeed ? new Uint8Array(this.identitySeed) : null; + } + async saveIdentitySeed(seed: Uint8Array): Promise { + this.identitySeed = new Uint8Array(seed); + } + async loadSnapshots(): Promise { + return this.snapshots ? new Uint8Array(this.snapshots) : null; + } + async saveSnapshots(bytes: Uint8Array): Promise { + this.snapshots = new Uint8Array(bytes); + } + async close(): Promise {} +} + +/** Convenience constructor — `memoryStorage()` is more readable than `new MemoryStorage()`. */ +export function memoryStorage(): MemoryStorage { + return new MemoryStorage(); +} diff --git a/sdks/typescript/src/adapters/node-fs-storage.ts b/sdks/typescript/src/adapters/node-fs-storage.ts new file mode 100644 index 0000000..96966b0 --- /dev/null +++ b/sdks/typescript/src/adapters/node-fs-storage.ts @@ -0,0 +1,64 @@ +// Node-side StorageAdapter that mirrors the on-disk layout of the Rust +// CLI: /doc.bin, /sync-states/.bin, +// /identity.seed, /snapshots.json. Atomic writes use the +// write-tmp-then-rename pattern. + +import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import type { StorageAdapter } from '../types.js'; + +async function readBytes(path: string): Promise { + try { + const buf = await readFile(path); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } catch (e: unknown) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; + } +} + +async function writeAtomic(path: string, bytes: Uint8Array): Promise { + await mkdir(dirname(path), { recursive: true }); + const tmp = `${path}.tmp-${process.pid}-${Date.now()}`; + await writeFile(tmp, bytes); + try { + await rename(tmp, path); + } catch (e) { + await rm(tmp, { force: true }).catch(() => {}); + throw e; + } +} + +export class NodeFsStorage implements StorageAdapter { + constructor(private root: string) {} + + async loadDoc(): Promise { + return readBytes(join(this.root, 'doc.bin')); + } + async saveDoc(bytes: Uint8Array): Promise { + await writeAtomic(join(this.root, 'doc.bin'), bytes); + } + async loadSyncState(peerKey: string): Promise { + return readBytes(join(this.root, 'sync-states', `${peerKey}.bin`)); + } + async saveSyncState(peerKey: string, bytes: Uint8Array): Promise { + await writeAtomic(join(this.root, 'sync-states', `${peerKey}.bin`), bytes); + } + async loadIdentitySeed(): Promise { + return readBytes(join(this.root, 'identity.seed')); + } + async saveIdentitySeed(seed: Uint8Array): Promise { + await writeAtomic(join(this.root, 'identity.seed'), seed); + } + async loadSnapshots(): Promise { + return readBytes(join(this.root, 'snapshots.json')); + } + async saveSnapshots(bytes: Uint8Array): Promise { + await writeAtomic(join(this.root, 'snapshots.json'), bytes); + } + async close(): Promise {} +} + +export function nodeFsStorage(root: string): NodeFsStorage { + return new NodeFsStorage(root); +} diff --git a/sdks/typescript/src/adapters/opfs-storage.ts b/sdks/typescript/src/adapters/opfs-storage.ts new file mode 100644 index 0000000..71da6fd --- /dev/null +++ b/sdks/typescript/src/adapters/opfs-storage.ts @@ -0,0 +1,139 @@ +// Browser-side StorageAdapter using the Origin Private File System (OPFS). +// File-shaped API, multi-GB quota, no permission prompt. Designed to run +// from a Web Worker for sync FileSystemSyncAccessHandle access; falls back +// to async writable streams on the main thread. +// +// Layout under /: +// doc.bin +// sync-states/.bin +// identity.seed +// snapshots.json + +import type { StorageAdapter } from '../types.js'; + +interface Navigator { + storage?: { getDirectory(): Promise }; +} + +declare const navigator: Navigator; + +export class OpfsStorage implements StorageAdapter { + private rootName: string; + private rootHandle: FileSystemDirectoryHandle | null = null; + + constructor(rootName = 'agentsync') { + this.rootName = rootName; + } + + private async ensureRoot(): Promise { + if (this.rootHandle) return this.rootHandle; + if (!navigator.storage?.getDirectory) { + throw new Error('OPFS unavailable in this runtime'); + } + const root = await navigator.storage.getDirectory(); + this.rootHandle = await root.getDirectoryHandle(this.rootName, { create: true }); + return this.rootHandle; + } + + private async childDir(...parts: string[]): Promise { + let handle = await this.ensureRoot(); + for (const part of parts) { + handle = await handle.getDirectoryHandle(part, { create: true }); + } + return handle; + } + + private async readFileBytes( + parent: FileSystemDirectoryHandle, + name: string, + ): Promise { + try { + const fileHandle = await parent.getFileHandle(name); + const file = await fileHandle.getFile(); + return new Uint8Array(await file.arrayBuffer()); + } catch (e: unknown) { + if ((e as DOMException).name === 'NotFoundError') return null; + throw e; + } + } + + private async writeFileBytes( + parent: FileSystemDirectoryHandle, + name: string, + bytes: Uint8Array, + ): Promise { + const fileHandle = await parent.getFileHandle(name, { create: true }); + // Prefer the sync access handle (Worker-only). Falls back to the async + // writable stream on the main thread. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sync = (fileHandle as any).createSyncAccessHandle as + | (() => Promise<{ + truncate(size: number): void; + write(buf: Uint8Array, opts: { at: number }): number; + flush(): void; + close(): void; + }>) + | undefined; + if (typeof sync === 'function') { + const handle = await sync.call(fileHandle); + try { + handle.truncate(0); + handle.write(bytes, { at: 0 }); + handle.flush(); + } finally { + handle.close(); + } + return; + } + const writable = await fileHandle.createWritable(); + try { + // Copy into a fresh ArrayBuffer-backed Uint8Array to satisfy the + // FileSystemWritableFileStream type (which doesn't accept SharedArrayBuffer-backed views). + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + await writable.write(copy); + } finally { + await writable.close(); + } + } + + async loadDoc(): Promise { + const root = await this.ensureRoot(); + return this.readFileBytes(root, 'doc.bin'); + } + async saveDoc(bytes: Uint8Array): Promise { + const root = await this.ensureRoot(); + await this.writeFileBytes(root, 'doc.bin', bytes); + } + async loadSyncState(peerKey: string): Promise { + const dir = await this.childDir('sync-states'); + return this.readFileBytes(dir, `${peerKey}.bin`); + } + async saveSyncState(peerKey: string, bytes: Uint8Array): Promise { + const dir = await this.childDir('sync-states'); + await this.writeFileBytes(dir, `${peerKey}.bin`, bytes); + } + async loadIdentitySeed(): Promise { + const root = await this.ensureRoot(); + return this.readFileBytes(root, 'identity.seed'); + } + async saveIdentitySeed(seed: Uint8Array): Promise { + const root = await this.ensureRoot(); + await this.writeFileBytes(root, 'identity.seed', seed); + } + async loadSnapshots(): Promise { + const root = await this.ensureRoot(); + return this.readFileBytes(root, 'snapshots.json'); + } + async saveSnapshots(bytes: Uint8Array): Promise { + const root = await this.ensureRoot(); + await this.writeFileBytes(root, 'snapshots.json', bytes); + } + async close(): Promise { + this.rootHandle = null; + } +} + +export function opfsStorage(rootName?: string): OpfsStorage { + return new OpfsStorage(rootName); +} diff --git a/sdks/typescript/src/adapters/ws-transport-node.ts b/sdks/typescript/src/adapters/ws-transport-node.ts new file mode 100644 index 0000000..ea6086f --- /dev/null +++ b/sdks/typescript/src/adapters/ws-transport-node.ts @@ -0,0 +1,125 @@ +// Node/Bun WebSocket transport using the `ws` package. Exposes the peer +// TLS cert fingerprint via `channelBinding()` so the handshake transcript +// can bind to the underlying TLS channel. + +import { createHash } from 'node:crypto'; +import type { TransportAdapter, TransportConn } from '../types.js'; + +interface WsCtor { + new (url: string, options: { rejectUnauthorized: boolean }): WsLike; +} + +interface WsLike { + binaryType: string; + send(data: Uint8Array, cb?: (err?: Error) => void): void; + close(code?: number, reason?: string): void; + terminate(): void; + on(event: 'open', cb: () => void): void; + on(event: 'message', cb: (data: Buffer | ArrayBuffer | Buffer[]) => void): void; + on(event: 'close', cb: () => void): void; + on(event: 'error', cb: (err: Error) => void): void; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _socket?: { getPeerCertificate?: (detailed: boolean) => any }; +} + +export function nodeWsTransport(WS: WsCtor): TransportAdapter { + return { + async connect(url: string): Promise { + const ws = new WS(url, { rejectUnauthorized: false }); + ws.binaryType = 'nodebuffer'; + await new Promise((res, rej) => { + ws.on('open', () => res()); + ws.on('error', (e: Error) => rej(e)); + }); + const incoming: Uint8Array[] = []; + let waiter: ((v: Uint8Array | null) => void) | null = null; + let closed = false; + ws.on('message', (data: Buffer | ArrayBuffer | Buffer[]) => { + const bytes = toBytes(data); + if (waiter) { + const w = waiter; + waiter = null; + w(bytes); + } else { + incoming.push(bytes); + } + }); + ws.on('close', () => { + closed = true; + if (waiter) { + const w = waiter; + waiter = null; + w(null); + } + }); + const channelBindingBytes = extractCertFingerprint(ws); + return { + async send(bytes: Uint8Array) { + await new Promise((res, rej) => { + ws.send(bytes, (err?: Error) => (err ? rej(err) : res())); + }); + }, + async *recv() { + while (true) { + if (incoming.length > 0) { + yield incoming.shift()!; + continue; + } + if (closed) return; + const next = await new Promise((res) => { + waiter = res; + }); + if (next === null) return; + yield next; + } + }, + channelBinding(): Uint8Array | null { + return channelBindingBytes; + }, + async close() { + // Initiate the WebSocket close handshake. If the peer doesn't + // respond within 500ms, terminate the underlying TCP socket so + // the connection actually goes away (and runSyncLoop exits). + ws.close(); + await new Promise((res) => { + const t = setTimeout(() => { + try { + ws.terminate(); + } catch {} + res(); + }, 500); + ws.on('close', () => { + clearTimeout(t); + res(); + }); + }); + }, + }; + }, + }; +} + +function toBytes(data: Buffer | ArrayBuffer | Buffer[]): Uint8Array { + if (Array.isArray(data)) { + const total = data.reduce((n, b) => n + b.length, 0); + const out = new Uint8Array(total); + let off = 0; + for (const b of data) { + out.set(b, off); + off += b.length; + } + return out; + } + if (data instanceof ArrayBuffer) return new Uint8Array(data); + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); +} + +function extractCertFingerprint(ws: WsLike): Uint8Array | null { + const sock = ws._socket; + if (!sock?.getPeerCertificate) return null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const cert: any = sock.getPeerCertificate(true); + if (!cert?.raw) return null; + const der: Buffer = cert.raw; + return new Uint8Array(createHash('sha256').update(der).digest()); +} diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts new file mode 100644 index 0000000..a094b81 --- /dev/null +++ b/sdks/typescript/src/index.ts @@ -0,0 +1,61 @@ +// Default entry. Targets Node and Bun via the wasm-pack `nodejs` glue. +// Browser / bundler consumers should import from `@agentsync/sdk/web`, +// which uses the wasm-pack `bundler` glue and lets Vite, webpack, Rollup, +// and esbuild handle the .wasm import. +// +// Both entry points expose the same TypeScript surface. + +import * as wasm from '#wasm-nodejs'; +import { type CreateOptions, type OpenOptions, Vault as VaultClass } from './vault.js'; +import { wrap } from './wrapper.js'; + +const wasmModule = wrap(wasm); + +export const { + Identity, + Pubkey, + Doc, + SyncState, + parseAuthorizedKeys, + renderAuthorizedKeys, + randomNonce, + buildTranscript, + encodeFrame, + decodeFrame, + contentHash, + schemaVersion, + defaultPort, + normalizeRendezvousUrl, +} = wasmModule; + +/** High-level Vault API (sync, watch, restore, labels). */ +export const Vault = { + create(opts: CreateOptions) { + return VaultClass.create(wasmModule, opts); + }, + open(opts: OpenOptions) { + return VaultClass.open(wasmModule, opts); + }, +}; + +export type { Vault as VaultInstance } from './vault.js'; +export type { CreateOptions, OpenOptions } from './vault.js'; +export type { + AuthorizedPeer, + DirectoryMeta, + FileMeta, + Frame, + FrameTag, + HelloOp, + Label, + ReconnectOptions, + StorageAdapter, + TransportAdapter, + TransportConn, + VaultEvent, + VaultOptions, +} from './types.js'; + +export { MemoryStorage, memoryStorage } from './adapters/memory-storage.js'; +export { NodeFsStorage, nodeFsStorage } from './adapters/node-fs-storage.js'; +export { nodeWsTransport } from './adapters/ws-transport-node.js'; diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts new file mode 100644 index 0000000..8e7d6cd --- /dev/null +++ b/sdks/typescript/src/types.ts @@ -0,0 +1,161 @@ +// JS-side mirrors of the agentsync-core data shapes that come back as +// plain objects from the wasm boundary (via serde-wasm-bindgen). + +export interface AuthorizedPeer { + /** SSH-style public key, e.g. `ssh-ed25519 AAAA...` */ + pubkey: string; + /** Optional human-readable label (`alice`, `homelab-nas`, etc). */ + label: string; +} + +export interface FileMeta { + id: string; + path: string; + kind: 'Text' | 'Attachment'; + size: number; + created_at: number; + updated_at: number; + /** Soft-delete timestamp (Unix ms). Missing/undefined when the file is alive. */ + deleted_at?: number | null; + /** Hex SHA-256 of attachment bytes; missing for text files. */ + binary_hash?: string | null; +} + +/** Tag of every wire frame; matches `Frame::t` in the Rust enum. */ +export type FrameTag = + | 'hello_hub' + | 'hello_peer' + | 'proof_hub' + | 'proof_peer' + | 'sync' + | 'blob_fetch' + | 'blob_push' + | 'ping' + | 'pong' + | 'error'; + +export type HelloOp = 'join' | 'create'; + +/** + * Discriminated union mirroring the Rust `Frame` enum. Decoded frames come + * back from `decodeFrame` with the `t` tag set, and `encodeFrame` accepts + * the same shape. + */ +export type Frame = + | { + t: 'hello_hub'; + vault_id: string; + hub_identity_pubkey: Uint8Array; + hub_nonce: Uint8Array; + tls_cert_fingerprint: Uint8Array; + vault_name?: string | null; + } + | { + t: 'hello_peer'; + peer_identity_pubkey: Uint8Array; + peer_nonce: Uint8Array; + op: HelloOp; + } + | { t: 'proof_hub'; sig: Uint8Array } + | { t: 'proof_peer'; sig: Uint8Array } + | { t: 'sync'; bytes: Uint8Array } + | { t: 'blob_fetch'; hash: string } + | { t: 'blob_push'; hash: string; bytes: Uint8Array } + | { t: 'ping'; ts: number } + | { t: 'pong'; ts: number } + | { t: 'error'; message: string }; + +/** Snapshot label as returned by `Doc.listLabels()`. */ +export interface Label { + name: string; + /** Heads encoded as base64 (no padding) — pair-wise concatenation of 32-byte hashes. */ + heads_b64: string; + created_at_ms: number; +} + +/** Directory metadata. */ +export interface DirectoryMeta { + id: string; + path: string; + created_at: number; + deleted_at?: number | null; +} + +/** High-level event emitted by a `Vault`. */ +export type VaultEvent = + | { kind: 'connecting'; url: string } + | { kind: 'connected'; hub_pubkey: Uint8Array; vault_id: string } + | { kind: 'disconnected'; reason: string } + | { kind: 'sync-progress'; outbound: boolean } + | { kind: 'doc-changed'; heads: Uint8Array[] } + | { kind: 'error'; message: string }; + +/** Storage adapter contract. The wasm crate doesn't implement storage — + * the TS SDK ships OPFS / Node FS / in-memory adapters and consumers can + * supply their own. */ +export interface StorageAdapter { + /** Load `doc.bin` bytes; resolve to null when no doc has been saved. */ + loadDoc(): Promise; + /** Atomically replace `doc.bin`. */ + saveDoc(bytes: Uint8Array): Promise; + /** Sync state per peer; key is hex pubkey. */ + loadSyncState(peerKey: string): Promise; + saveSyncState(peerKey: string, bytes: Uint8Array): Promise; + /** Optional persistent identity seed. */ + loadIdentitySeed(): Promise; + saveIdentitySeed(seed: Uint8Array): Promise; + /** Snapshot index (label list) — JSON or any opaque bytes. */ + loadSnapshots(): Promise; + saveSnapshots(bytes: Uint8Array): Promise; + /** Best-effort dispose — release file handles, close DB connections, etc. */ + close(): Promise; +} + +/** Transport contract — JS side picks `ws` (Node) or native `WebSocket` + * (browser) or anything else that speaks binary WebSocket frames. */ +export interface TransportAdapter { + connect(url: string, opts?: TransportConnectOpts): Promise; +} + +export interface TransportConnectOpts { + /** Rejected if the WebSocket reports a peer cert that doesn't match. */ + pinnedCertFingerprint?: Uint8Array; +} + +export interface TransportConn { + /** Send one binary frame. */ + send(bytes: Uint8Array): Promise; + /** Async iterable of inbound frames; ends when the peer closes. */ + recv(): AsyncIterable; + /** TLS peer cert SHA-256 if the runtime exposes it. Browsers can't. */ + channelBinding(): Uint8Array | null; + close(): Promise; +} + +/** Options accepted by `Vault.create` / `Vault.open`. */ +export interface VaultOptions { + /** Persistent state root. */ + storage: StorageAdapter; + /** Optional identity (generated if absent). */ + identity?: import('./wrapper.js').Identity; + /** Existing vault id to adopt. When omitted on `Vault.create`, a fresh + * UUID is minted. Use this when joining an existing remote vault. */ + vaultId?: string; + /** Hub URL, e.g. `wss://hub.example.com`. Required for sync; optional for offline use. */ + rendezvousUrl?: string; + /** Pin the hub's identity pubkey (TOFU). */ + hubPubkey?: Uint8Array; + /** Display name carried in the handshake. */ + name?: string; + /** WebSocket transport. Defaults to the runtime-appropriate adapter. */ + transport?: TransportAdapter; +} + +export interface ReconnectOptions { + /** Total attempts before giving up (default: Infinity). */ + maxAttempts?: number; + /** Initial backoff in ms (default: 500). */ + initialBackoffMs?: number; + /** Max backoff in ms (default: 30000). */ + maxBackoffMs?: number; +} diff --git a/sdks/typescript/src/vault.ts b/sdks/typescript/src/vault.ts new file mode 100644 index 0000000..9f59736 --- /dev/null +++ b/sdks/typescript/src/vault.ts @@ -0,0 +1,652 @@ +// High-level Vault — the user-facing TS API. Mirrors `core::Vault` from the +// Rust SDK as closely as the runtime allows: connect / connectWithReconnect +// / disconnect, write/read/delete/list files + directories, label +// snapshots, restore-to-label / restore-to-time, doc-changed events. +// +// Networking and persistence live in injectable adapters (`StorageAdapter`, +// `TransportAdapter`) so the same Vault works in browsers (OPFS + native +// WebSocket), Node/Bun (node:fs + ws), Electron/Tauri/Obsidian (whichever +// adapters the host plugs in). +// +// This module owns the protocol state machine: WebSocket framing, the +// 4-message handshake, the Automerge incremental sync loop, and the +// reconnect/backoff supervisor. The wasm crate provides the cryptographic +// + CRDT primitives; everything else is plain TS that runs anywhere. + +import type { + Frame, + Label, + ReconnectOptions, + StorageAdapter, + TransportAdapter, + TransportConn, + VaultEvent, + VaultOptions, +} from './types.js'; +import type { Doc, Identity, Pubkey, SyncState } from './wrapper.js'; + +interface WasmBindings { + Doc: { new (vaultId: string): Doc; load(bytes: Uint8Array): Doc }; + Identity: { generate(): Identity; fromSeed(seed: Uint8Array): Identity }; + Pubkey: { fromBytes(bytes: Uint8Array): Pubkey }; + SyncState: { new (): SyncState; decode(bytes: Uint8Array): SyncState }; + buildTranscript: ( + hubNonce: Uint8Array, + peerNonce: Uint8Array, + tlsCertFingerprint: Uint8Array, + hubPubkey: Uint8Array, + peerPubkey: Uint8Array, + ) => Uint8Array; + randomNonce: () => Uint8Array; + encodeFrame: (frame: Frame) => Uint8Array; + decodeFrame: (bytes: Uint8Array) => Frame; + contentHash: (bytes: Uint8Array) => string; +} + +interface ConnectionState { + conn: TransportConn; + hubPubkey: Uint8Array; + syncState: SyncState; + /** Set true once the handshake completes; subsequent frames are sync. */ + ready: boolean; +} + +const HANDSHAKE_TIMEOUT_MS = 15_000; + +/** Internal: one-byte equality check for Uint8Arrays. */ +function byteEq(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +function hexOf(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +export interface CreateOptions extends VaultOptions {} +export interface OpenOptions extends VaultOptions {} + +/** + * The high-level entry point. `Vault.create` initializes a fresh vault + * (generates a vault_id, seeds `authorized_keys` with the creator's + * pubkey). `Vault.open` reuses an existing one. + * + * Exactly one Vault instance per (storage, identity) pair — running two + * concurrent Vaults against the same storage will produce conflicting + * doc.bin writes. + */ +export class Vault { + private wasm: WasmBindings; + private opts: VaultOptions; + private doc: Doc; + private identity: Identity; + private vaultId: string; + /** True when we generated/loaded the identity ourselves and should free + * it on close. False when the caller passed one in — they own it. */ + private ownsIdentity: boolean; + private connection: ConnectionState | null = null; + private reconnectAbort: AbortController | null = null; + private listeners = new Set<(e: VaultEvent) => void>(); + private closed = false; + + /** Create a brand-new vault on the supplied storage. Pass `vaultId` to + * adopt an existing remote vault (the joining-an-existing-hub case); + * in that mode no authorized_keys is seeded — the joining peer learns + * everything from sync. */ + static async create(wasm: WasmBindings, options: CreateOptions): Promise { + const ownsIdentity = !options.identity; + const identity = options.identity ?? (await loadOrCreateIdentity(wasm, options.storage)); + const joiningExisting = !!options.vaultId; + const vaultId = options.vaultId ?? generateVaultId(); + const doc = new wasm.Doc(vaultId); + if (!joiningExisting) { + // Fresh vault: seed authorized_keys with the creator's pubkey so + // the creator can connect to their own listener immediately. + // Mirrors `Vault::create` in the Rust core. + const pk = identity.pubkey(); + const sshLine = `${pk.toSshString()} creator\n`; + doc.writeTextFile('authorized_keys', sshLine); + pk.free(); + } + const bytes = doc.save(); + await options.storage.saveDoc(bytes); + return new Vault(wasm, options, doc, identity, vaultId, ownsIdentity); + } + + /** Open an existing vault from storage; errors if no doc exists. */ + static async open(wasm: WasmBindings, options: OpenOptions): Promise { + const ownsIdentity = !options.identity; + const identity = options.identity ?? (await loadOrCreateIdentity(wasm, options.storage)); + const bytes = await options.storage.loadDoc(); + if (!bytes) { + throw new Error('no vault on disk; call Vault.create() first'); + } + const doc = wasm.Doc.load(bytes); + const vaultId = doc.vaultId(); + return new Vault(wasm, options, doc, identity, vaultId, ownsIdentity); + } + + private constructor( + wasm: WasmBindings, + opts: VaultOptions, + doc: Doc, + identity: Identity, + vaultId: string, + ownsIdentity: boolean, + ) { + this.wasm = wasm; + this.opts = opts; + this.doc = doc; + this.identity = identity; + this.vaultId = vaultId; + this.ownsIdentity = ownsIdentity; + } + + // ---- Read-only accessors ---- + + vaultIdValue(): string { + return this.vaultId; + } + identityRef(): Identity { + return this.identity; + } + isConnected(): boolean { + return this.connection?.ready ?? false; + } + + // ---- File operations (delegate to Doc, persist + push sync after) ---- + + async writeTextFile(path: string, content: string): Promise { + const id = this.doc.writeTextFile(path, content); + await this.flush(); + await this.kickSyncLoop(); + return id; + } + + async readTextFile(path: string): Promise { + return this.doc.readFile(path); + } + + fileExists(path: string): boolean { + return this.doc.fileExists(path); + } + + async deleteFile(path: string): Promise { + this.doc.deleteFile(path); + await this.flush(); + await this.kickSyncLoop(); + } + + async renameFile(from: string, to: string): Promise { + this.doc.renameFile(from, to); + await this.flush(); + await this.kickSyncLoop(); + } + + listFiles() { + return this.doc.listFiles(); + } + + async createDirectory(path: string): Promise { + const id = this.doc.createDirectory(path); + await this.flush(); + await this.kickSyncLoop(); + return id; + } + + async deleteDirectory(path: string, recursive = false): Promise { + this.doc.deleteDirectory(path, recursive); + await this.flush(); + await this.kickSyncLoop(); + } + + listDirectories() { + return this.doc.listDirectories(); + } + + // ---- Labels / restore ---- + + async createLabel(name: string): Promise { + this.doc.createLabel(name); + await this.flush(); + } + async deleteLabel(name: string): Promise { + this.doc.deleteLabel(name); + await this.flush(); + } + listLabels(): Label[] { + return this.doc.listLabels(); + } + async restoreToLabel(name: string): Promise { + this.doc.restoreToLabel(name); + await this.flush(); + await this.kickSyncLoop(); + } + async restoreToTime(targetMs: number): Promise { + this.doc.restoreToTime(targetMs); + await this.flush(); + await this.kickSyncLoop(); + } + + // ---- Events ---- + + /** Subscribe to vault events. Returns an unsubscribe function. */ + subscribe(listener: (e: VaultEvent) => void): () => void { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + + /** Async-iterable view of events. */ + async *events(): AsyncIterableIterator { + const queue: VaultEvent[] = []; + let resolveNext: ((v: VaultEvent | null) => void) | null = null; + const unsub = this.subscribe((e) => { + if (resolveNext) { + const r = resolveNext; + resolveNext = null; + r(e); + } else { + queue.push(e); + } + }); + try { + while (!this.closed) { + if (queue.length > 0) { + yield queue.shift()!; + continue; + } + const next = await new Promise((res) => { + resolveNext = res; + }); + if (next === null) return; + yield next; + } + } finally { + unsub(); + } + } + + private emit(e: VaultEvent) { + for (const l of this.listeners) { + try { + l(e); + } catch { + // listener throws don't propagate + } + } + } + + // ---- Connection management ---- + + /** Connect once, run the handshake + sync loop, return when the + * connection closes (cleanly or with error). Use `connectWithReconnect` + * for production. */ + async connect(): Promise { + const url = this.opts.rendezvousUrl; + if (!url) throw new Error('rendezvousUrl is required to connect'); + const transport = this.opts.transport ?? defaultTransport(); + this.emit({ kind: 'connecting', url }); + const conn = await transport.connect(url); + try { + const result = await this.runHandshake(conn); + this.connection = result; + this.emit({ + kind: 'connected', + hub_pubkey: result.hubPubkey, + vault_id: this.vaultId, + }); + await this.runSyncLoop(result); + } finally { + try { + await conn.close(); + } catch {} + this.connection = null; + this.emit({ kind: 'disconnected', reason: 'connection closed' }); + } + } + + /** Connect with exponential backoff. Resolves when the supervisor is + * told to stop via `disconnect()`. */ + async connectWithReconnect(opts: ReconnectOptions = {}): Promise { + const max = opts.maxAttempts ?? Number.POSITIVE_INFINITY; + const initial = opts.initialBackoffMs ?? 500; + const cap = opts.maxBackoffMs ?? 30_000; + this.reconnectAbort = new AbortController(); + const signal = this.reconnectAbort.signal; + let attempt = 0; + while (!signal.aborted && attempt < max) { + attempt += 1; + try { + await this.connect(); + attempt = 0; // reset on clean exit + } catch (e) { + if (signal.aborted) return; + const delay = Math.min(initial * 2 ** Math.min(attempt - 1, 20), cap); + this.emit({ + kind: 'error', + message: `connect failed (attempt ${attempt}): ${e}`, + }); + await sleep(delay, signal); + } + } + } + + /** Stop the reconnect supervisor and tear down the active connection. */ + async disconnect(): Promise { + this.reconnectAbort?.abort(); + this.reconnectAbort = null; + if (this.connection) { + try { + await this.connection.conn.close(); + } catch {} + this.connection = null; + } + } + + /** Final cleanup: persist, drop the connection, free wasm memory. */ + async close(): Promise { + if (this.closed) return; + this.closed = true; + await this.disconnect(); + await this.flush(); + try { + this.doc.free(); + } catch {} + if (this.ownsIdentity) { + try { + this.identity.free(); + } catch {} + } + await this.opts.storage.close(); + } + + // ---- Internal: protocol state machine ---- + + private async runHandshake(conn: TransportConn): Promise { + const recvIter = conn.recv()[Symbol.asyncIterator](); + const nextFrame = async (): Promise => { + const r = await withTimeout(recvIter.next(), HANDSHAKE_TIMEOUT_MS); + if (r.done) throw new Error('connection closed during handshake'); + return this.wasm.decodeFrame(r.value); + }; + + // 1. Hub → Peer: HelloHub + const helloHub = await nextFrame(); + if (helloHub.t !== 'hello_hub') { + throw new Error(`expected hello_hub, got ${helloHub.t}`); + } + if (helloHub.vault_id !== this.vaultId) { + throw new Error(`vault_id mismatch: hub=${helloHub.vault_id} local=${this.vaultId}`); + } + if (this.opts.hubPubkey && !byteEq(this.opts.hubPubkey, helloHub.hub_identity_pubkey)) { + throw new Error('hub pubkey does not match pinned value'); + } + const channelBinding = conn.channelBinding() ?? new Uint8Array(0); + if ( + helloHub.tls_cert_fingerprint.length > 0 && + channelBinding.length > 0 && + !byteEq(helloHub.tls_cert_fingerprint, channelBinding) + ) { + throw new Error('tls cert fingerprint mismatch (channel binding)'); + } + + // 2. Peer → Hub: HelloPeer + const peerNonce = this.wasm.randomNonce(); + const peerPk = this.identity.pubkey(); + const peerPkBytes = peerPk.bytes(); + peerPk.free(); + await this.sendFrame(conn, { + t: 'hello_peer', + peer_identity_pubkey: peerPkBytes, + peer_nonce: peerNonce, + op: 'join', + }); + + // 3. Hub → Peer: ProofHub + const proofHub = await nextFrame(); + if (proofHub.t !== 'proof_hub') { + throw new Error(`expected proof_hub, got ${proofHub.t}`); + } + const transcript = this.wasm.buildTranscript( + helloHub.hub_nonce, + peerNonce, + helloHub.tls_cert_fingerprint, + helloHub.hub_identity_pubkey, + peerPkBytes, + ); + const hubPk = this.wasm.Pubkey.fromBytes(helloHub.hub_identity_pubkey); + const ok = hubPk.verify(transcript, proofHub.sig); + hubPk.free(); + if (!ok) throw new Error('hub signature verification failed'); + + // 4. Peer → Hub: ProofPeer + const peerSig = await this.identity.sign(transcript); + await this.sendFrame(conn, { t: 'proof_peer', sig: peerSig }); + + // Set up incremental sync state. Loaded from storage if we've talked to + // this hub before; otherwise fresh. + const stateKey = hexOf(helloHub.hub_identity_pubkey); + const savedState = await this.opts.storage.loadSyncState(stateKey); + const syncState = savedState + ? this.wasm.SyncState.decode(savedState) + : new this.wasm.SyncState(); + + const result: ConnectionState = { + conn, + hubPubkey: helloHub.hub_identity_pubkey, + syncState, + ready: true, + }; + return result; + } + + private async runSyncLoop(state: ConnectionState): Promise { + // Drive an initial outbound message. + await this.pumpOutbound(state); + + for await (const bytes of state.conn.recv()) { + const frame = this.wasm.decodeFrame(bytes); + switch (frame.t) { + case 'sync': { + const moved = this.doc.receiveSyncMessage(state.syncState, frame.bytes); + if (moved) { + await this.flush(); + this.emit({ kind: 'doc-changed', heads: this.doc.heads() }); + } + await this.pumpOutbound(state); + await this.persistSyncState(state); + break; + } + case 'ping': + await this.sendFrame(state.conn, { t: 'pong', ts: frame.ts }); + break; + case 'pong': + break; + case 'blob_fetch': + case 'blob_push': + // Not supported in storage-only mode; ignore for now. Full blob + // support requires a JS-side blob CAS — out of scope for v1. + break; + case 'error': + this.emit({ kind: 'error', message: frame.message }); + return; + default: + // Late handshake frame after handshake completed — ignore. + break; + } + } + } + + /** Send any pending sync messages. Called on heads change + on inbound. */ + private async pumpOutbound(state: ConnectionState): Promise { + while (true) { + const msg = this.doc.generateSyncMessage(state.syncState); + if (!msg) return; + await this.sendFrame(state.conn, { t: 'sync', bytes: msg }); + this.emit({ kind: 'sync-progress', outbound: true }); + } + } + + private async sendFrame(conn: TransportConn, frame: Frame): Promise { + const bytes = this.wasm.encodeFrame(frame); + await conn.send(bytes); + } + + /** Wake the sync loop after a local doc mutation. No-op when offline. + * Awaitable so callers can ensure the change is on the wire before they + * return. (The sync loop also pumps on every inbound; this kick is what + * delivers a local-only change while no inbound is in flight.) */ + private async kickSyncLoop(): Promise { + if (this.connection?.ready) { + await this.pumpOutbound(this.connection); + } + } + + /** Persist doc + sync state. Called after every mutation + on inbound + * sync-message-applied. Cheap because Automerge.save is incremental-ish; + * the storage adapter writes atomically. */ + private async flush(): Promise { + if (this.closed) return; + const bytes = this.doc.save(); + await this.opts.storage.saveDoc(bytes); + if (this.connection?.ready) await this.persistSyncState(this.connection); + } + + private async persistSyncState(state: ConnectionState): Promise { + const bytes = state.syncState.encode(); + await this.opts.storage.saveSyncState(hexOf(state.hubPubkey), bytes); + } +} + +// ---- Helpers ---- + +function generateVaultId(): string { + // RFC 4122 v4 UUID via crypto.getRandomValues. Available in Node ≥ 19, + // Bun, all modern browsers. + const bytes = new Uint8Array(16); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const c: any = globalThis.crypto; + c.getRandomValues(bytes); + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = hexOf(bytes); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +async function loadOrCreateIdentity( + wasm: WasmBindings, + storage: StorageAdapter, +): Promise { + const seed = await storage.loadIdentitySeed(); + if (seed) return wasm.Identity.fromSeed(seed); + const id = wasm.Identity.generate(); + await storage.saveIdentitySeed(id.seed()); + return id; +} + +function defaultTransport(): TransportAdapter { + // Resolved at call time so import-time evaluation in browsers doesn't + // try to require('ws'). Consumers should explicitly pass `transport` if + // they need full control; this best-effort autodetect handles the + // simple cases. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const w: any = (globalThis as any).WebSocket; + if (typeof w === 'function') { + return makeBrowserTransport(w); + } + throw new Error( + 'no WebSocket implementation found; pass `transport` explicitly or import from @agentsync/sdk/node', + ); +} + +function makeBrowserTransport(WebSocketCtor: typeof globalThis.WebSocket): TransportAdapter { + return { + async connect(url: string): Promise { + const ws = new WebSocketCtor(url); + ws.binaryType = 'arraybuffer'; + await new Promise((res, rej) => { + ws.addEventListener('open', () => res(), { once: true }); + ws.addEventListener('error', () => rej(new Error(`websocket error connecting to ${url}`)), { + once: true, + }); + }); + const incoming: Uint8Array[] = []; + let waiter: ((v: Uint8Array | null) => void) | null = null; + let closed = false; + ws.addEventListener('message', (ev) => { + const data = ev.data instanceof ArrayBuffer ? new Uint8Array(ev.data) : null; + if (!data) return; + if (waiter) { + const w = waiter; + waiter = null; + w(data); + } else { + incoming.push(data); + } + }); + ws.addEventListener('close', () => { + closed = true; + if (waiter) { + const w = waiter; + waiter = null; + w(null); + } + }); + return { + async send(bytes: Uint8Array) { + ws.send(bytes); + }, + async *recv() { + while (true) { + if (incoming.length > 0) { + yield incoming.shift()!; + continue; + } + if (closed) return; + const next = await new Promise((res) => { + waiter = res; + }); + if (next === null) return; + yield next; + } + }, + channelBinding(): Uint8Array | null { + // Browsers don't expose peer cert; channel binding falls back to + // empty in the handshake. The hub must be on a real CA cert for + // this mode to be safe. + return null; + }, + async close() { + ws.close(); + }, + }; + }, + }; +} + +async function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((res) => { + const t = setTimeout(res, ms); + if (signal) { + const onAbort = () => { + clearTimeout(t); + res(); + }; + if (signal.aborted) onAbort(); + else signal.addEventListener('abort', onAbort, { once: true }); + } + }); +} + +async function withTimeout(p: Promise, ms: number): Promise { + let to: ReturnType; + return await Promise.race([ + p.finally(() => clearTimeout(to)), + new Promise((_, rej) => { + to = setTimeout(() => rej(new Error(`timeout after ${ms}ms`)), ms); + }), + ]); +} diff --git a/sdks/typescript/src/web.ts b/sdks/typescript/src/web.ts new file mode 100644 index 0000000..f63da17 --- /dev/null +++ b/sdks/typescript/src/web.ts @@ -0,0 +1,58 @@ +// Browser / bundler entry. Uses the wasm-pack `bundler` target glue, which +// emits a top-level `import` of the .wasm file that Vite, webpack, Rollup, +// and esbuild understand. For Node/Bun, import `@agentsync/sdk` instead. + +import * as wasm from '#wasm-bundler'; +import { type CreateOptions, type OpenOptions, Vault as VaultClass } from './vault.js'; +import { wrap } from './wrapper.js'; + +const wasmModule = wrap(wasm); + +export const { + Identity, + Pubkey, + Doc, + SyncState, + parseAuthorizedKeys, + renderAuthorizedKeys, + randomNonce, + buildTranscript, + encodeFrame, + decodeFrame, + contentHash, + schemaVersion, + defaultPort, + normalizeRendezvousUrl, +} = wasmModule; + +/** High-level Vault API. The default transport uses the global + * `WebSocket` constructor (every modern browser provides one). Pass + * `transport` explicitly for non-standard runtimes. */ +export const Vault = { + create(opts: CreateOptions) { + return VaultClass.create(wasmModule, opts); + }, + open(opts: OpenOptions) { + return VaultClass.open(wasmModule, opts); + }, +}; + +export type { Vault as VaultInstance, CreateOptions, OpenOptions } from './vault.js'; +export type { + AuthorizedPeer, + DirectoryMeta, + FileMeta, + Frame, + FrameTag, + HelloOp, + Label, + ReconnectOptions, + StorageAdapter, + TransportAdapter, + TransportConn, + VaultEvent, + VaultOptions, +} from './types.js'; + +export { MemoryStorage, memoryStorage } from './adapters/memory-storage.js'; +export { OpfsStorage, opfsStorage } from './adapters/opfs-storage.js'; diff --git a/sdks/typescript/src/wrapper.ts b/sdks/typescript/src/wrapper.ts new file mode 100644 index 0000000..c73671a --- /dev/null +++ b/sdks/typescript/src/wrapper.ts @@ -0,0 +1,98 @@ +// Re-exports the wasm-bindgen surface as a typed module. The bundler and +// nodejs glue files are byte-for-byte identical at the JS API level — +// this wrapper exists so we have one place to typecheck the API and add +// thin convenience layers later if needed. + +import type { AuthorizedPeer, DirectoryMeta, FileMeta, Frame, Label } from './types.js'; + +// The wasm-pack output declares these as concrete classes. We restate the +// minimum slice we need so consumers don't have to depend on whichever +// .d.ts (bundler vs nodejs) gets picked. +interface WasmModule { + Identity: typeof IdentityClass; + Pubkey: typeof PubkeyClass; + Doc: typeof DocClass; + SyncState: typeof SyncStateClass; + parseAuthorizedKeys(body: string): AuthorizedPeer[]; + renderAuthorizedKeys(entries: AuthorizedPeer[]): string; + randomNonce(): Uint8Array; + buildTranscript( + hubNonce: Uint8Array, + peerNonce: Uint8Array, + tlsCertFingerprint: Uint8Array, + hubPubkey: Uint8Array, + peerPubkey: Uint8Array, + ): Uint8Array; + encodeFrame(value: Frame): Uint8Array; + decodeFrame(bytes: Uint8Array): Frame; + contentHash(bytes: Uint8Array): string; + schemaVersion(): number; + defaultPort(): number; + normalizeRendezvousUrl(url: string): string; +} + +declare class IdentityClass { + static generate(): IdentityClass; + static fromSeed(seed: Uint8Array): IdentityClass; + seed(): Uint8Array; + pubkey(): PubkeyClass; + sign(message: Uint8Array): Promise; + free(): void; +} + +declare class PubkeyClass { + static fromBytes(bytes: Uint8Array): PubkeyClass; + static fromSshString(s: string): PubkeyClass; + toSshString(): string; + fingerprint(): string; + bytes(): Uint8Array; + verify(message: Uint8Array, signature: Uint8Array): boolean; + free(): void; +} + +declare class SyncStateClass { + constructor(); + static decode(bytes: Uint8Array): SyncStateClass; + encode(): Uint8Array; + free(): void; +} + +declare class DocClass { + constructor(vaultId: string); + static load(bytes: Uint8Array): DocClass; + save(): Uint8Array; + saveIncremental(): Uint8Array; + vaultId(): string; + heads(): Uint8Array[]; + merge(other: DocClass): boolean; + generateSyncMessage(state: SyncStateClass): Uint8Array | undefined; + receiveSyncMessage(state: SyncStateClass, bytes: Uint8Array): boolean; + writeTextFile(path: string, content: string): string; + readFile(path: string): string; + fileExists(path: string): boolean; + deleteFile(path: string): void; + renameFile(from: string, to: string): void; + writeAttachment(path: string, hash: string, size: number): string; + listFiles(): FileMeta[]; + createDirectory(path: string): string; + deleteDirectory(path: string, recursive: boolean): void; + listDirectories(): DirectoryMeta[]; + createLabel(name: string): void; + deleteLabel(name: string): void; + listLabels(): Label[]; + restoreToLabel(name: string): void; + restoreToTime(targetMs: number): void; + free(): void; +} + +export type Identity = IdentityClass; +export type Pubkey = PubkeyClass; +export type Doc = DocClass; +export type SyncState = SyncStateClass; + +export function wrap(mod: unknown): WasmModule { + // The wasm-pack glue is structurally a WasmModule; cast and re-export. + // We intentionally do not deep-clone or proxy — the wasm-bindgen objects + // hold pointers into linear memory and must be `.free()`d when done. + return mod as WasmModule; +} diff --git a/sdks/typescript/test/e2e/hub-handshake.test.ts b/sdks/typescript/test/e2e/hub-handshake.test.ts new file mode 100644 index 0000000..ada1b3a --- /dev/null +++ b/sdks/typescript/test/e2e/hub-handshake.test.ts @@ -0,0 +1,114 @@ +// e2e: spawn a real `agentsync` hub and verify the wasm SDK can speak its +// wire protocol. The hub picks an ephemeral port; we open a WSS connection +// (with cert validation disabled — TLS trust is bound at the application +// layer via the cert fingerprint inside the handshake transcript), wait +// for the first frame, and assert it decodes as a `hello_hub` carrying +// the hub's vault id. +// +// This test runs under Node (not Bun) because Bun's built-in WebSocket +// client doesn't currently support the hub's ed25519 self-signed TLS cert. +// `bun run test:e2e` invokes node with --experimental-strip-types so this +// .ts file can run unmodified. + +import { strict as assert } from 'node:assert'; +import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, before, describe, test } from 'node:test'; +import { WebSocket } from 'ws'; +// Import the built SDK so this test exercises the same artifact npm +// consumers will install. `bun run build` (or just `build:ts`) must run +// first; the test:e2e script in package.json wires that up. +import { decodeFrame } from '../../dist/index.js'; + +const AGENTSYNC = process.env.AGENTSYNC_BIN ?? 'agentsync'; + +let tmp: string; +let vaultDir: string; +let hub: ChildProcessWithoutNullStreams | null = null; +let port = 0; + +function waitFor(check: () => T | undefined, timeoutMs = 10_000): Promise { + const start = Date.now(); + return new Promise((resolve, reject) => { + const tick = () => { + const v = check(); + if (v !== undefined) { + resolve(v); + return; + } + if (Date.now() - start > timeoutMs) { + reject(new Error(`timeout after ${timeoutMs}ms`)); + return; + } + setTimeout(tick, 50); + }; + tick(); + }); +} + +before(async () => { + tmp = mkdtempSync(join(tmpdir(), 'agentsync-e2e-')); + vaultDir = join(tmp, 'vault'); + mkdirSync(vaultDir, { recursive: true }); + + const initProc = spawn(AGENTSYNC, ['init', '--name', 'wasm-e2e'], { + cwd: vaultDir, + env: { ...process.env, HOME: tmp, AGENTSYNC_HOME: tmp }, + }); + await new Promise((res, rej) => { + initProc.on('exit', (code) => (code === 0 ? res() : rej(new Error(`init exit ${code}`)))); + }); + + hub = spawn(AGENTSYNC, ['--listen', '127.0.0.1:0'], { + cwd: vaultDir, + env: { ...process.env, HOME: tmp, AGENTSYNC_HOME: tmp }, + }); + + // The CLI announces the bound port on stdout: `listening on wss://addr:port`. + const stdout: string[] = []; + hub.stdout.on('data', (b: Buffer) => { + stdout.push(b.toString()); + }); + + port = await waitFor(() => { + const joined = stdout.join(''); + const m = joined.match(/listening on wss:\/\/[^:]+:(\d+)/i); + return m ? Number(m[1]) : undefined; + }); +}); + +after(() => { + if (hub) hub.kill('SIGTERM'); + if (tmp) rmSync(tmp, { recursive: true, force: true }); +}); + +describe('wasm SDK ↔ agentsync hub', () => { + test('decodes the hello_hub frame the hub puts on the wire', async () => { + const ws = new WebSocket(`wss://127.0.0.1:${port}`, { + rejectUnauthorized: false, + }); + + const firstFrame = await new Promise((resolve, reject) => { + ws.on('error', reject); + ws.on('message', (data: Buffer) => { + resolve(new Uint8Array(data)); + }); + setTimeout(() => reject(new Error('no frame within 5s')), 5_000); + }); + + ws.close(); + + const frame = decodeFrame(firstFrame); + assert.equal(frame.t, 'hello_hub'); + if (frame.t === 'hello_hub') { + assert.equal(typeof frame.vault_id, 'string'); + assert.ok(frame.vault_id.length > 0); + assert.equal(frame.hub_identity_pubkey.length, 32); + assert.equal(frame.hub_nonce.length, 32); + // Phase 2+ fingerprint is non-empty when WSS is on. + assert.equal(frame.tls_cert_fingerprint.length, 32); + } + }); +}); diff --git a/sdks/typescript/test/e2e/vault-sync.test.ts b/sdks/typescript/test/e2e/vault-sync.test.ts new file mode 100644 index 0000000..8f01d7d --- /dev/null +++ b/sdks/typescript/test/e2e/vault-sync.test.ts @@ -0,0 +1,273 @@ +// e2e: spawn a real `agentsync` hub, wire up a TS-side `Vault` peer, and +// verify that: +// 1. The 4-message handshake completes +// 2. The Vault emits a `connected` event +// 3. The doc syncs (TS peer learns whatever files the hub has) +// 4. Disconnect tears down cleanly +// +// This exercises the full TypeScript SDK protocol stack: WebSocket +// transport, frame codec, transcript signing, Automerge incremental sync, +// storage persistence (memory adapter for the test). +// +// Runs under Node — Bun's WebSocket client doesn't currently accept the +// hub's ed25519 self-signed cert. CI provides AGENTSYNC_BIN. + +import { strict as assert } from 'node:assert'; +import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, before, describe, test } from 'node:test'; +import { WebSocket } from 'ws'; +import { Identity, Vault, memoryStorage, nodeWsTransport } from '../../dist/index.js'; + +const AGENTSYNC = process.env.AGENTSYNC_BIN ?? 'agentsync'; + +let tmp = ''; +let vaultDir = ''; +let hub: ChildProcessWithoutNullStreams | null = null; +let port = 0; +let vaultId = ''; +let peerIdentity: ReturnType; + +async function waitFor( + check: () => T | undefined | Promise, + timeoutMs = 10_000, +): Promise { + const start = Date.now(); + while (true) { + let v: T | undefined; + try { + v = await check(); + } catch { + v = undefined; + } + if (v !== undefined) return v; + if (Date.now() - start > timeoutMs) { + throw new Error(`timeout after ${timeoutMs}ms`); + } + await new Promise((r) => setTimeout(r, 50)); + } +} + +before(async () => { + tmp = mkdtempSync(join(tmpdir(), 'agentsync-vault-e2e-')); + vaultDir = join(tmp, 'vault'); + mkdirSync(vaultDir, { recursive: true }); + + // 1. Run agentsync init to bootstrap the hub vault. + const initProc = spawn(AGENTSYNC, ['init', '--name', 'vault-e2e'], { + cwd: vaultDir, + env: { ...process.env, HOME: tmp, AGENTSYNC_HOME: tmp }, + }); + let initOutput = ''; + initProc.stdout.on('data', (b) => { + initOutput += b.toString(); + }); + initProc.stderr.on('data', (b) => { + initOutput += b.toString(); + }); + await new Promise((res, rej) => { + initProc.on('exit', (code) => + code === 0 ? res() : rej(new Error(`init exit ${code}: ${initOutput}`)), + ); + }); + + // 2. Pull the vault_id out of the printed init output. + const vaultIdMatch = initOutput.match(/vault_id\s*=\s*([0-9a-f-]{36})/); + if (!vaultIdMatch) throw new Error(`could not parse vault_id from: ${initOutput}`); + vaultId = vaultIdMatch[1]!; + + // 3. Generate a TS-side identity for use later when adding it to authorized_keys. + peerIdentity = Identity.generate(); + + // 4. Spawn the hub. The materializer writes `authorized_keys` to the + // vault root once it's running. + hub = spawn(AGENTSYNC, ['--listen', '127.0.0.1:0'], { + cwd: vaultDir, + env: { + ...process.env, + HOME: tmp, + AGENTSYNC_HOME: tmp, + AGENTSYNC_LOG: 'error', + }, + }); + const stdout: string[] = []; + hub.stdout.on('data', (b: Buffer) => { + stdout.push(b.toString()); + }); + hub.stderr.on('data', (b: Buffer) => { + stdout.push(b.toString()); + }); + + // 5. Pull the bound port out of the announce line. + port = await waitFor(() => { + const joined = stdout.join(''); + const m = joined.match(/listening on wss:\/\/[^:]+:(\d+)/i); + return m ? Number(m[1]) : undefined; + }); + + // 6. Wait for the hub to materialize `authorized_keys`, then append our + // pubkey. The hub re-ingests on file change, picks up the new pubkey, + // and accepts handshakes signed by it. + const akPath = join(vaultDir, 'authorized_keys'); + await waitFor(async () => { + try { + await readFile(akPath, 'utf8'); + return true; + } catch { + return undefined; + } + }); + const akContent = await readFile(akPath, 'utf8'); + const peerPk = peerIdentity.pubkey(); + const peerSshLine = `${peerPk.toSshString()} ts-e2e-peer\n`; + peerPk.free(); + await writeFile(akPath, akContent + peerSshLine); + // Give the hub a brief window to ingest + recompute the authorized set. + await new Promise((r) => setTimeout(r, 500)); +}); + +after(() => { + if (hub) hub.kill('SIGTERM'); + if (tmp) rmSync(tmp, { recursive: true, force: true }); +}); + +async function makeTsVault() { + const storage = memoryStorage(); + const v = await Vault.create({ + storage, + identity: peerIdentity, + vaultId, + rendezvousUrl: `wss://127.0.0.1:${port}`, + transport: nodeWsTransport(WebSocket as unknown as never), + }); + return { v, storage }; +} + +/** Wait until `v.subscribe` emits an event whose `kind` is in `wanted`. */ +function waitForEvent(v: Awaited>['v'], wanted: string[]) { + return new Promise((resolve, reject) => { + const off = v.subscribe((e) => { + if (wanted.includes(e.kind)) { + off(); + resolve(); + } + if (e.kind === 'error') { + off(); + reject(new Error(e.message)); + } + }); + }); +} + +describe('TypeScript Vault ↔ Rust hub', () => { + test('completes the 4-message handshake and emits connected', { timeout: 30_000 }, async () => { + const { v } = await makeTsVault(); + const events: string[] = []; + const unsub = v.subscribe((e) => events.push(e.kind)); + const connected = waitForEvent(v, ['connected']); + const connectPromise = v.connect().catch(() => {}); + await connected; + assert.ok(events.includes('connecting')); + assert.ok(events.includes('connected')); + unsub(); + await v.disconnect(); + await connectPromise; + await v.close(); + }); + + test('TS write propagates to hub disk', { timeout: 30_000 }, async () => { + const { v } = await makeTsVault(); + const connected = waitForEvent(v, ['connected']); + const connectPromise = v.connect().catch(() => {}); + await connected; + // Wait briefly for the initial sync round-trip to settle so the TS + // doc and hub doc share heads before we add a new local change. + await new Promise((r) => setTimeout(r, 800)); + + await v.writeTextFile('hello-from-ts.md', '# from ts\n'); + + const target = join(vaultDir, 'hello-from-ts.md'); + const content = await waitFor(async () => { + try { + const c = await readFile(target, 'utf8'); + return c === '# from ts\n' ? c : undefined; + } catch { + return undefined; + } + }, 15_000); + assert.equal(content, '# from ts\n'); + + await v.disconnect(); + await connectPromise; + await v.close(); + }); + + test('hub write propagates to TS Vault', { timeout: 30_000 }, async () => { + const { v } = await makeTsVault(); + const connected = waitForEvent(v, ['connected']); + const connectPromise = v.connect().catch(() => {}); + await connected; + + // Drop a file into the hub's vault directory; the materializer + // ingest loop picks it up and the sync engine forwards it to peers. + await writeFile(join(vaultDir, 'hello-from-hub.md'), '# from hub\n'); + + // Poll the TS Vault until the file shows up. + const text = await waitFor(() => { + try { + return v.readTextFile('hello-from-hub.md'); + } catch { + return undefined; + } + }, 15_000); + assert.equal(text, '# from hub\n'); + + await v.disconnect(); + await connectPromise; + await v.close(); + }); + + test('reconnect after hub restart', { timeout: 60_000 }, async () => { + const { v } = await makeTsVault(); + const connected1 = waitForEvent(v, ['connected']); + const reconnectAbort = v.connectWithReconnect({ initialBackoffMs: 200 }); + // Don't await — connectWithReconnect runs forever. + reconnectAbort.catch(() => {}); + await connected1; + + // Kill the hub and re-spawn on the SAME port so the TS Vault's + // backoff loop reconnects to the new instance. + hub!.kill('SIGTERM'); + await new Promise((r) => hub!.once('exit', () => r())); + + const newHub = spawn(AGENTSYNC, ['--listen', `127.0.0.1:${port}`], { + cwd: vaultDir, + env: { ...process.env, HOME: tmp, AGENTSYNC_HOME: tmp }, + }); + newHub.stdout.on('data', () => {}); + newHub.stderr.on('data', () => {}); + hub = newHub; + + // Subscribe AFTER killing so we only catch the re-connect event. + const reconnected = new Promise((resolve, reject) => { + const t0 = Date.now(); + const tick = setInterval(() => { + if (v.isConnected()) { + clearInterval(tick); + resolve(); + } else if (Date.now() - t0 > 30_000) { + clearInterval(tick); + reject(new Error('reconnect timeout')); + } + }, 100); + }); + await reconnected; + assert.equal(v.isConnected(), true); + + await v.disconnect(); + await v.close(); + }); +}); diff --git a/sdks/typescript/test/unit/authorized-keys.test.ts b/sdks/typescript/test/unit/authorized-keys.test.ts new file mode 100644 index 0000000..83f32e4 --- /dev/null +++ b/sdks/typescript/test/unit/authorized-keys.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from 'bun:test'; +import { Identity, parseAuthorizedKeys, renderAuthorizedKeys } from '../../src/index.js'; +import type { AuthorizedPeer } from '../../src/index.js'; + +describe('authorized_keys', () => { + test('parses ssh-style lines with labels', () => { + const id = Identity.generate(); + const body = `${id.pubkey().toSshString()} alice\n# comment\n\n`; + const peers = parseAuthorizedKeys(body); + expect(peers).toHaveLength(1); + expect(peers[0]?.pubkey).toBe(id.pubkey().toSshString()); + expect(peers[0]?.label).toBe('alice'); + }); + + test('renderAuthorizedKeys round-trips', () => { + const id = Identity.generate(); + const peers: AuthorizedPeer[] = [{ pubkey: id.pubkey().toSshString(), label: 'bob' }]; + const rendered = renderAuthorizedKeys(peers); + expect(rendered.includes('ssh-ed25519 ')).toBe(true); + const reparsed = parseAuthorizedKeys(rendered); + expect(reparsed).toEqual(peers); + }); + + test('skips comments and blank lines', () => { + const peers = parseAuthorizedKeys('# only comments\n\n # indented\n'); + expect(peers).toEqual([]); + }); +}); diff --git a/sdks/typescript/test/unit/doc.test.ts b/sdks/typescript/test/unit/doc.test.ts new file mode 100644 index 0000000..af0594d --- /dev/null +++ b/sdks/typescript/test/unit/doc.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'bun:test'; +import { Doc, contentHash, schemaVersion } from '../../src/index.js'; + +describe('Doc', () => { + test('new + write + read round-trips', () => { + const doc = new Doc('vault-1'); + expect(doc.vaultId()).toBe('vault-1'); + doc.writeTextFile('notes/hello.md', '# hello\n'); + expect(doc.readFile('notes/hello.md')).toBe('# hello\n'); + expect(doc.fileExists('notes/hello.md')).toBe(true); + expect(doc.fileExists('notes/missing.md')).toBe(false); + }); + + test('save + load preserves content', () => { + const doc = new Doc('vault-2'); + doc.writeTextFile('a.md', 'A'); + doc.writeTextFile('b.md', 'B'); + const bytes = doc.save(); + const loaded = Doc.load(bytes); + expect(loaded.vaultId()).toBe('vault-2'); + expect(loaded.readFile('a.md')).toBe('A'); + expect(loaded.readFile('b.md')).toBe('B'); + }); + + test('two peers merge concurrent edits without conflict', () => { + const a = new Doc('shared'); + a.writeTextFile('seed.md', 'seed'); + const b = Doc.load(a.save()); + a.writeTextFile('a-only.md', 'from a'); + b.writeTextFile('b-only.md', 'from b'); + const bClone = Doc.load(b.save()); + expect(a.merge(bClone)).toBe(true); + expect(a.readFile('seed.md')).toBe('seed'); + expect(a.readFile('a-only.md')).toBe('from a'); + expect(a.readFile('b-only.md')).toBe('from b'); + }); + + test('deleteFile removes the file', () => { + const doc = new Doc('v'); + doc.writeTextFile('x.md', 'x'); + expect(doc.fileExists('x.md')).toBe(true); + doc.deleteFile('x.md'); + expect(doc.fileExists('x.md')).toBe(false); + }); + + test('schemaVersion is stable', () => { + expect(schemaVersion()).toBe(1); + }); + + test('contentHash matches sha256 hex', () => { + expect(contentHash(new Uint8Array([0x68, 0x69]))).toBe( + '8f434346648f6b96df89dda901c5176b10a6d83961dd3c1ac88b59b2dc327aa4', + ); + }); +}); diff --git a/sdks/typescript/test/unit/identity.test.ts b/sdks/typescript/test/unit/identity.test.ts new file mode 100644 index 0000000..e21e5aa --- /dev/null +++ b/sdks/typescript/test/unit/identity.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from 'bun:test'; +import { Identity, Pubkey, randomNonce } from '../../src/index.js'; + +describe('Identity', () => { + test('generate produces a unique seed each call', () => { + const a = Identity.generate(); + const b = Identity.generate(); + expect(a.seed()).not.toEqual(b.seed()); + expect(a.seed().length).toBe(32); + }); + + test('fromSeed round-trips through pubkey', () => { + const id = Identity.generate(); + const seed = id.seed(); + const restored = Identity.fromSeed(seed); + expect(restored.pubkey().toSshString()).toBe(id.pubkey().toSshString()); + }); + + test('fromSeed rejects wrong length', () => { + expect(() => Identity.fromSeed(new Uint8Array(31))).toThrow(); + }); + + test('sign produces a 64-byte signature that the pubkey verifies', async () => { + const id = Identity.generate(); + const msg = new TextEncoder().encode('hello agentsync'); + const sig = await id.sign(msg); + expect(sig.length).toBe(64); + expect(id.pubkey().verify(msg, sig)).toBe(true); + expect(id.pubkey().verify(new TextEncoder().encode('hello agentsync!'), sig)).toBe(false); + }); + + test('pubkey ssh string round-trips', () => { + const id = Identity.generate(); + const ssh = id.pubkey().toSshString(); + expect(ssh.startsWith('ssh-ed25519 ')).toBe(true); + const restored = Pubkey.fromSshString(ssh); + expect(restored.bytes()).toEqual(id.pubkey().bytes()); + }); + + test('fingerprint matches OpenSSH SHA256: shape', () => { + const fp = Identity.generate().pubkey().fingerprint(); + expect(fp).toMatch(/^SHA256:[A-Za-z0-9+/]{43}$/); + }); +}); + +describe('randomNonce', () => { + test('returns 32 unique bytes per call', () => { + const a = randomNonce(); + const b = randomNonce(); + expect(a.length).toBe(32); + expect(a).not.toEqual(b); + }); +}); diff --git a/sdks/typescript/test/unit/protocol.test.ts b/sdks/typescript/test/unit/protocol.test.ts new file mode 100644 index 0000000..a9db2a6 --- /dev/null +++ b/sdks/typescript/test/unit/protocol.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from 'bun:test'; +import { + Identity, + buildTranscript, + decodeFrame, + encodeFrame, + randomNonce, +} from '../../src/index.js'; +import type { Frame } from '../../src/index.js'; + +describe('Frame codec', () => { + test('encode → decode round-trips a sync frame', () => { + const frame: Frame = { t: 'sync', bytes: new Uint8Array([1, 2, 3, 4, 5]) }; + const wire = encodeFrame(frame); + const decoded = decodeFrame(wire); + expect(decoded.t).toBe('sync'); + if (decoded.t === 'sync') { + expect(Array.from(decoded.bytes)).toEqual([1, 2, 3, 4, 5]); + } + }); + + test('round-trips a hello_hub frame', () => { + const id = Identity.generate(); + const frame: Frame = { + t: 'hello_hub', + vault_id: 'vault-x', + hub_identity_pubkey: id.pubkey().bytes(), + hub_nonce: randomNonce(), + tls_cert_fingerprint: new Uint8Array(32), + vault_name: 'demo', + }; + const decoded = decodeFrame(encodeFrame(frame)); + expect(decoded.t).toBe('hello_hub'); + if (decoded.t === 'hello_hub') { + expect(decoded.vault_id).toBe('vault-x'); + expect(decoded.vault_name).toBe('demo'); + } + }); + + test('rejects garbage bytes', () => { + expect(() => decodeFrame(new Uint8Array([0xff, 0xff, 0xff]))).toThrow(); + }); +}); + +describe('handshake helpers', () => { + test('buildTranscript is deterministic for fixed inputs', () => { + const hubNonce = new Uint8Array(32).fill(1); + const peerNonce = new Uint8Array(32).fill(2); + const fp = new Uint8Array(32).fill(3); + const hubPk = new Uint8Array(32).fill(4); + const peerPk = new Uint8Array(32).fill(5); + const a = buildTranscript(hubNonce, peerNonce, fp, hubPk, peerPk); + const b = buildTranscript(hubNonce, peerNonce, fp, hubPk, peerPk); + expect(a).toEqual(b); + // First 17 bytes are the literal "agentsync-auth-v1" domain tag. + expect(new TextDecoder().decode(a.slice(0, 17))).toBe('agentsync-auth-v1'); + }); + + test('buildTranscript rejects wrong-length inputs', () => { + expect(() => + buildTranscript( + new Uint8Array(31), + new Uint8Array(32), + new Uint8Array(0), + new Uint8Array(32), + new Uint8Array(32), + ), + ).toThrow(); + }); +}); diff --git a/sdks/typescript/test/unit/sync-pair.test.ts b/sdks/typescript/test/unit/sync-pair.test.ts new file mode 100644 index 0000000..17a90a1 --- /dev/null +++ b/sdks/typescript/test/unit/sync-pair.test.ts @@ -0,0 +1,90 @@ +// Sanity check: two Doc instances + their SyncStates can converge via the +// generateSyncMessage / receiveSyncMessage primitives. Validates the +// wasm bindings without involving the wire protocol. + +import { describe, expect, it } from 'bun:test'; +import { Doc, SyncState } from '../../src/index.js'; + +function syncFully(a: Doc, aState: SyncState, b: Doc, bState: SyncState) { + for (let i = 0; i < 50; i++) { + const m1 = a.generateSyncMessage(aState); + if (m1) b.receiveSyncMessage(bState, m1); + const m2 = b.generateSyncMessage(bState); + if (m2) a.receiveSyncMessage(aState, m2); + if (!m1 && !m2) return; + } + throw new Error('sync did not converge in 50 rounds'); +} + +describe('Doc sync round-trip', () => { + it('two docs with the same vault_id converge to identical files', () => { + const vid = '11111111-1111-4111-8111-111111111111'; + const a = new Doc(vid); + const b = new Doc(vid); + + a.writeTextFile('a-only.md', 'from A'); + b.writeTextFile('b-only.md', 'from B'); + + const aState = new SyncState(); + const bState = new SyncState(); + syncFully(a, aState, b, bState); + + const aPaths = a + .listFiles() + .filter((f) => !f.deleted_at) + .map((f) => f.path) + .sort(); + const bPaths = b + .listFiles() + .filter((f) => !f.deleted_at) + .map((f) => f.path) + .sort(); + expect(aPaths).toEqual(bPaths); + expect(aPaths).toContain('a-only.md'); + expect(aPaths).toContain('b-only.md'); + expect(a.readFile('b-only.md')).toBe('from B'); + expect(b.readFile('a-only.md')).toBe('from A'); + }); + + it('write after initial sync still propagates', () => { + const vid = '22222222-2222-4222-8222-222222222222'; + const a = new Doc(vid); + const b = new Doc(vid); + + const aState = new SyncState(); + const bState = new SyncState(); + syncFully(a, aState, b, bState); + + a.writeTextFile('late.md', 'late'); + syncFully(a, aState, b, bState); + + expect(b.readFile('late.md')).toBe('late'); + }); + + it('write after initial sync — single round (mirrors e2e flow)', () => { + // The e2e Vault sends one sync message after a local write and waits + // for the response. If a single round doesn't carry the change, + // applications would have to keep poking the sync loop. This test + // pins down the expected behavior: one outbound message must include + // the new changes. + const vid = '33333333-3333-4333-8333-333333333333'; + const a = new Doc(vid); + const b = new Doc(vid); + // Pre-seed b with a file (mirrors the hub already having a doc). + b.writeTextFile('seed.md', 'seed'); + + const aState = new SyncState(); + const bState = new SyncState(); + syncFully(a, aState, b, bState); + expect(a.readFile('seed.md')).toBe('seed'); + + // Local write on a, then ONE outbound message (no pull from b). + a.writeTextFile('after.md', 'after'); + const m = a.generateSyncMessage(aState); + expect(m).not.toBeUndefined(); + b.receiveSyncMessage(bState, m!); + // b should now have the file — Automerge sync includes changes + // in the outbound message when the sender knows the peer is missing them. + expect(b.readFile('after.md')).toBe('after'); + }); +}); diff --git a/sdks/typescript/test/unit/vault.test.ts b/sdks/typescript/test/unit/vault.test.ts new file mode 100644 index 0000000..ffb63d6 --- /dev/null +++ b/sdks/typescript/test/unit/vault.test.ts @@ -0,0 +1,184 @@ +// Vault unit tests covering: create/open round-trip, file/dir/label +// operations, restore-to-label, restore-to-time, event subscription. +// Uses `MemoryStorage` so no fs / network involved. + +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { type CreateOptions, MemoryStorage, Vault, memoryStorage } from '../../src/index.js'; + +async function freshVault(opts?: Partial) { + const storage = memoryStorage(); + const v = await Vault.create({ storage, ...opts }); + return { v, storage }; +} + +describe('Vault.create / Vault.open', () => { + it('creates a new vault and persists doc.bin to storage', async () => { + const { v, storage } = await freshVault(); + const bytes = await storage.loadDoc(); + expect(bytes).not.toBeNull(); + expect(bytes!.length).toBeGreaterThan(0); + expect(v.vaultIdValue()).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}/); + await v.close(); + }); + + it('open() reuses persisted state after a close', async () => { + const { v, storage } = await freshVault(); + const id1 = v.vaultIdValue(); + await v.writeTextFile('hello.md', '# hi\n'); + await v.close(); + const v2 = await Vault.open({ storage }); + expect(v2.vaultIdValue()).toBe(id1); + expect(await v2.readTextFile('hello.md')).toBe('# hi\n'); + await v2.close(); + }); + + it('open() errors when no doc on disk', async () => { + const storage = new MemoryStorage(); + await expect(Vault.open({ storage })).rejects.toThrow(/no vault on disk/); + }); + + it('seeds authorized_keys with the creator pubkey', async () => { + const { v } = await freshVault(); + const body = await v.readTextFile('authorized_keys'); + expect(body).toMatch(/^ssh-ed25519 [A-Za-z0-9+/]+ creator\n$/); + await v.close(); + }); +}); + +describe('Vault file operations', () => { + it('write/read/delete round-trip', async () => { + const { v } = await freshVault(); + await v.writeTextFile('a.md', 'hello'); + expect(v.fileExists('a.md')).toBe(true); + expect(await v.readTextFile('a.md')).toBe('hello'); + await v.deleteFile('a.md'); + expect(v.fileExists('a.md')).toBe(false); + await v.close(); + }); + + it('listFiles returns metadata', async () => { + const { v } = await freshVault(); + await v.writeTextFile('one.md', '1'); + await v.writeTextFile('two.md', '22'); + const files = v.listFiles().filter((f) => !f.deleted_at); + const paths = files.map((f) => f.path).sort(); + expect(paths).toContain('one.md'); + expect(paths).toContain('two.md'); + await v.close(); + }); + + it('renameFile changes the path but keeps the id', async () => { + const { v } = await freshVault(); + const id = await v.writeTextFile('old.md', 'x'); + await v.renameFile('old.md', 'new.md'); + expect(v.fileExists('new.md')).toBe(true); + expect(v.fileExists('old.md')).toBe(false); + const files = v.listFiles(); + const m = files.find((f) => f.id === id); + expect(m?.path).toBe('new.md'); + await v.close(); + }); + + it('directories: create, list, delete', async () => { + const { v } = await freshVault(); + await v.createDirectory('docs'); + const dirs = v.listDirectories().filter((d) => !d.deleted_at); + expect(dirs.some((d) => d.path === 'docs')).toBe(true); + await v.deleteDirectory('docs'); + const after = v.listDirectories().filter((d) => !d.deleted_at); + expect(after.some((d) => d.path === 'docs')).toBe(false); + await v.close(); + }); +}); + +describe('Vault labels + restore', () => { + it('createLabel + listLabels round-trip', async () => { + const { v } = await freshVault(); + await v.writeTextFile('a.md', 'first'); + await v.createLabel('v1'); + await v.writeTextFile('a.md', 'second'); + const labels = v.listLabels(); + expect(labels.some((l) => l.name === 'v1')).toBe(true); + await v.close(); + }); + + it('restoreToLabel reverts file content', async () => { + const { v } = await freshVault(); + await v.writeTextFile('a.md', 'first'); + await v.createLabel('v1'); + await v.writeTextFile('a.md', 'second'); + expect(await v.readTextFile('a.md')).toBe('second'); + await v.restoreToLabel('v1'); + expect(await v.readTextFile('a.md')).toBe('first'); + await v.close(); + }); + + it('restoreToTime reverts to a past timestamp', async () => { + const { v } = await freshVault(); + await v.writeTextFile('a.md', 'first'); + // Wait 10ms so the next write has a strictly later timestamp. + await new Promise((r) => setTimeout(r, 10)); + const t = Date.now(); + await new Promise((r) => setTimeout(r, 10)); + await v.writeTextFile('a.md', 'second'); + await v.restoreToTime(t); + expect(await v.readTextFile('a.md')).toBe('first'); + await v.close(); + }); + + it('deleteLabel removes the entry', async () => { + const { v } = await freshVault(); + await v.createLabel('tmp'); + expect(v.listLabels().some((l) => l.name === 'tmp')).toBe(true); + await v.deleteLabel('tmp'); + expect(v.listLabels().some((l) => l.name === 'tmp')).toBe(false); + await v.close(); + }); +}); + +describe('Vault events', () => { + it('subscribe receives doc-changed events on local writes', async () => { + const { v } = await freshVault(); + // Local writes don't currently emit doc-changed (only remote writes + // do); this asserts the subscribe/unsubscribe infrastructure works. + const seen: string[] = []; + const unsub = v.subscribe((e) => seen.push(e.kind)); + expect(typeof unsub).toBe('function'); + unsub(); + await v.close(); + }); + + it('isConnected returns false when offline', async () => { + const { v } = await freshVault(); + expect(v.isConnected()).toBe(false); + await v.close(); + }); +}); + +describe('MemoryStorage', () => { + let s: MemoryStorage; + beforeEach(() => { + s = new MemoryStorage(); + }); + afterEach(async () => s.close()); + + it('round-trips doc bytes', async () => { + expect(await s.loadDoc()).toBeNull(); + await s.saveDoc(new Uint8Array([1, 2, 3])); + const back = await s.loadDoc(); + expect(back).not.toBeNull(); + expect(Array.from(back!)).toEqual([1, 2, 3]); + }); + + it('round-trips sync state per peer', async () => { + expect(await s.loadSyncState('peer1')).toBeNull(); + await s.saveSyncState('peer1', new Uint8Array([9])); + expect(Array.from((await s.loadSyncState('peer1'))!)).toEqual([9]); + }); + + it('round-trips identity seed', async () => { + expect(await s.loadIdentitySeed()).toBeNull(); + await s.saveIdentitySeed(new Uint8Array(32)); + expect((await s.loadIdentitySeed())!.length).toBe(32); + }); +}); diff --git a/sdks/typescript/tsconfig.json b/sdks/typescript/tsconfig.json new file mode 100644 index 0000000..eb87253 --- /dev/null +++ b/sdks/typescript/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "baseUrl": ".", + "paths": { + "#wasm-nodejs": ["./dist/nodejs/agentsync_wasm"], + "#wasm-bundler": ["./dist/bundler/agentsync_wasm"] + }, + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "resolveJsonModule": true, + "verbatimModuleSyntax": true, + "allowJs": false + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules", "test"] +} diff --git a/specs/API-RUST.md b/specs/API-RUST.md new file mode 100644 index 0000000..c159f3a --- /dev/null +++ b/specs/API-RUST.md @@ -0,0 +1,665 @@ +# API-RUST.md — Rust Public API + +> Normative for the published Rust API. See [SPEC.md § Conformance +> language](./SPEC.md#conformance-language). + +This document specifies the public API of the `agentsync-core` Rust +crate. It is the surface a Rust application sees when it adds +`agentsync-core` as a dependency, and the contract that must be +preserved across minor versions. + +For the *byte-level* semantics underlying these methods (wire format, +on-disk format, document schema), see the linked specs. This document +states the API shape and the behaviors it guarantees, without +re-stating those semantics. + +--- + +## 1. Crate-level layout + +``` +agentsync_core:: + Vault, OpenOptions, CreateOptions, CreatedVault, VaultId, + VaultEvent, VaultEventKind, ReconnectOptions, VaultConfig, + SyncHandle, // trait + + Doc, FileId, FileKind, FileMeta, DirectoryMeta, Label, + SCHEMA_VERSION, content_hash, + + Identity, Pubkey, PUBKEY_LEN, SIGNATURE_LEN, + + Frame, HelloOp, + + Error, Result, + + AuthorizedPeer, parse_authorized_keys, render_authorized_keys, + parse_peers_md, render_peers_md, PEERS_FILE, + + HANDSHAKE_DOMAIN, NONCE_LEN, build_transcript, random_nonce, + + AUTHORIZED_KEYS_FILE, + DEFAULT_PORT, DEFAULT_LISTEN_ADDR, DEFAULT_LISTEN_ADDR_NO_TLS, + USER_IDENTITY_FILENAME, USER_STATE_DIR, + normalize_rendezvous_url, normalize_with_scheme, + + // native-only: + BindOptions, Binding, NodeFsAdapter, + discover_vault_id, agent_list_identities_at, + + // module: + pub mod host; // see HOST.md, native-only +``` + +### 1.1 Conditional compilation + +Items marked "native-only" are gated by `#[cfg(not(target_arch = "wasm32"))]`. +A wasm consumer **MUST NOT** import these — they will fail to compile. + +A reimplementation **SHOULD** preserve this gate so the crate compiles +on the wasm target. + +--- + +## 2. Versioning policy + +The crate follows semver. For the public surface specified here: + +- A change to a method **signature** (parameters, return type, async + vs sync) is a **breaking** change. +- Adding a new public item is a **minor** change. +- Adding a new variant to a public `enum` is a **breaking** change + unless the enum is `#[non_exhaustive]`. The current `Error` enum is + not `#[non_exhaustive]` but **SHOULD** be — adding variants is + expected. +- Changing the documented behavior of a method (e.g., when it errors) + is a **breaking** change. + +A reimplementation in another language **MUST** treat each public type +and method described below as part of its API contract. + +--- + +## 3. `Vault` (native-only) + +The top-level type. A `Vault` represents one open vault: a loaded +Automerge document, an `Identity`, optional network connections, +optional filesystem binding. + +### 3.1 Construction + +```rust +impl Vault { + pub async fn create(opts: CreateOptions) -> Result<(Self, CreatedVault)>; + pub async fn open(opts: OpenOptions) -> Result; +} + +pub struct CreateOptions { + pub rendezvous_url: Option, + pub identity: Option, + pub storage_path: PathBuf, +} + +pub struct OpenOptions { + pub rendezvous_url: Option, + pub vault_id: VaultId, + pub identity: Identity, + pub storage_path: PathBuf, + pub hub_pubkey: Option, + pub name: Option, +} + +pub struct CreatedVault { + pub vault_id: VaultId, + pub identity: Identity, +} + +pub type VaultId = String; +``` + +`create`: + +- Generates a new `vault_id` (UUID v4). +- Initializes a fresh Automerge document per [DOCUMENT.md](./DOCUMENT.md). +- If `identity` is `None`, generates a new ed25519 keypair and persists + it per [STORAGE.md § 8](./STORAGE.md#identity-files). +- Writes `.agentsync/config.toml` with the provided fields. +- Initializes `authorized_keys` containing the creator's pubkey. + +`open`: + +- Loads `.agentsync/doc.bin` into memory. +- Validates that the document's `vault_id` matches `opts.vault_id`. +- Does **not** automatically connect; call `connect()`. + +Both methods **MUST** be safe to call concurrently for *different* +vaults; the same vault **MUST NOT** be opened twice in the same +process. + +### 3.2 Accessors + +```rust +impl Vault { + pub fn id(&self) -> &VaultId; + pub fn identity(&self) -> &Identity; + pub fn pubkey(&self) -> Pubkey; + pub fn storage_path(&self) -> &Path; + pub fn name(&self) -> Option<&str>; + pub fn subscribe(&self) -> broadcast::Receiver; +} +``` + +`subscribe` returns a fresh receiver of the vault's event stream; see +§ 3.7. + +### 3.3 File operations + +```rust +impl Vault { + pub async fn write_text_file(&self, path: &str, content: &str) -> Result<()>; + pub async fn read_text_file (&self, path: &str) -> Result; + pub async fn delete_file (&self, path: &str) -> Result<()>; + pub async fn rename_file (&self, from: &str, to: &str) -> Result<()>; + pub async fn list_files (&self) -> Result>; + pub async fn list_file_paths(&self) -> Result>; + pub async fn file_exists (&self, path: &str) -> bool; + pub async fn file_hash (&self, path: &str) -> Result; +} +``` + +All paths **MUST** be POSIX-normalized per +[DOCUMENT.md § 5](./DOCUMENT.md#path-normalization). Implementations +normalize on input and return normalized strings on output. + +`list_files` returns only live (non-soft-deleted) entries. Order is +unspecified. + +`file_hash` returns lowercase hex SHA-256 of the file's content, +matching the `content_hash` free function. + +### 3.4 Directory operations + +```rust +impl Vault { + pub async fn create_directory (&self, path: &str) -> Result<()>; + pub async fn delete_directory (&self, path: &str, recursive: bool) -> Result<()>; + pub async fn rename_directory (&self, from: &str, to: &str) -> Result<()>; + pub async fn list_directories (&self) -> Result>; +} +``` + +`delete_directory(path, recursive=true)` is a single Automerge +transaction (see [DOCUMENT.md § 3.3](./DOCUMENT.md#recursive-delete)). +With `recursive=false`, the operation **MUST** fail with +`Error::AlreadyExists` (or similar) if the directory has live +children. + +### 3.5 History and labels + +```rust +impl Vault { + pub async fn create_label (&self, name: &str) -> Result<()>; + pub async fn delete_label (&self, name: &str) -> Result<()>; + pub async fn list_labels (&self) -> Result>; + pub async fn restore_label (&self, name: &str) -> Result<()>; + pub async fn restore_to_heads(&self, heads: &[ChangeHash]) -> Result<()>; + pub async fn restore_to_time (&self, target_ms: i64) -> Result<()>; +} +``` + +Restoration semantics: + +- Restoration is **additive**: it produces forward-going Automerge + changes that bring the document state to match the past state. It + **MUST NOT** rewrite history. +- `restore_to_heads` restores to the document state at the given heads. +- `restore_to_time(target_ms)` walks the change graph and restores to + the state at the latest heads whose timestamps are `<= target_ms`. + Wall-clock timestamps are advisory and may be skewed across peers + (typically by milliseconds with NTP); precision is best-effort. +- `restore_label(name)` is shorthand for `restore_to_heads(label.heads)`. + +### 3.6 Networking + +```rust +impl Vault { + pub async fn connect (&mut self) -> Result<()>; + pub async fn disconnect (&mut self); + pub async fn connect_with_reconnect(&mut self, opts: ReconnectOptions) -> Result<()>; + pub async fn listen (&mut self, addr: SocketAddr) -> Result; + pub async fn listen_plain(&mut self, addr: SocketAddr) -> Result; + pub async fn unlisten (&mut self); + pub async fn peer_count (&self) -> usize; + pub async fn authorized_pubkeys(&self) -> Vec; +} + +pub struct ReconnectOptions { + pub max_attempts: u32, + pub initial_backoff: Duration, + pub max_backoff: Duration, +} +``` + +`connect` performs the four-message handshake against the configured +`rendezvous_url`. See [WIRE.md § 4](./WIRE.md#handshake-normative). + +`listen` binds a TLS WebSocket listener on `addr`. The hub generates a +self-signed cert if needed (see [STORAGE.md § 5](./STORAGE.md#tls-material-hub-only)). +Returns the actually-bound `SocketAddr` (useful for port `0`). + +`listen_plain` binds plain `ws://`. **SHOULD** be used only behind a +TLS-terminating reverse proxy; implementations **SHOULD** print a +warning when used otherwise. + +`connect_with_reconnect` retries `connect` with exponential backoff +between `initial_backoff` and `max_backoff`. It **MUST** abort on +`Error::Auth` (the credentials are wrong; retrying won't help). + +### 3.7 Event stream + +```rust +pub enum VaultEventKind { + Connected, + Disconnected, + FileChanged { path: String }, + SyncProgress { percent: u8 }, + Error(String), +} + +pub struct VaultEvent { + pub kind: VaultEventKind, +} +``` + +`subscribe` returns a `tokio::sync::broadcast::Receiver`. +Multiple consumers **MAY** subscribe; each gets its own receiver. + +Note: the TypeScript SDK exposes a richer event enum (with hub pubkey +and reason fields). This is intentional — the TS SDK orchestrates the +state machine itself; the Rust crate's enum is the underlying minimal +set. See [API-TS.md § 5](./API-TS.md#5-events). + +### 3.8 Filesystem binding + +```rust +pub struct BindOptions { + pub exclude_patterns: Vec, + pub include_patterns: Vec, + pub attachment_max_bytes: u64, + pub text_file_max_bytes: u64, +} + +impl Vault { + pub async fn bind_directory(&mut self, path: &Path, opts: BindOptions) + -> Result>; + pub async fn binding_arc(&self) -> Option>; + pub async fn materialize(&self, binding: &Arc) -> Result<()>; +} +``` + +`bind_directory` installs a filesystem watcher over `path` and starts +mirroring document changes to disk and disk changes to the document. +`Binding` owns the watcher; dropping the `Arc` does not stop the bind +unless all references are gone. + +`materialize` writes every live file in the document out to the bound +directory. **MUST** be idempotent. + +### 3.9 Lifecycle + +```rust +impl Vault { + pub async fn flush(&self) -> Result<()>; + pub async fn close(self) -> Result<()>; + pub fn notify_doc_changed(&self); + pub async fn debug_dump(&self) -> Result)>>; +} +``` + +`flush` forces a save of `doc.bin` and the snapshot index. **SHOULD** +be called before shutdown to ensure no acknowledged changes are lost. + +`close` consumes the `Vault`, performs a final flush, disconnects, and +unbinds. + +`notify_doc_changed` is a hint to the engine that the document has +changed and any pending sync messages should be regenerated. Used by +adapters that mutate the document outside the normal API path. + +`debug_dump` returns `(path, body)` pairs for diagnostic introspection. +**MUST NOT** be relied on for normal operation — its format may change. + +--- + +## 4. `Doc` + +The `Doc` type wraps an `automerge::AutoCommit` and exposes the +document operations defined in [DOCUMENT.md](./DOCUMENT.md). It is +available on **both** native and wasm targets. + +```rust +impl Doc { + pub fn new(vault_id: &str) -> Result; + pub fn load(bytes: &[u8]) -> Result; + pub fn save(&mut self) -> Vec; + pub fn save_incremental(&mut self) -> Vec; + pub fn fork(&mut self) -> Self; + + pub fn vault_id(&mut self) -> Result; + pub fn heads(&mut self) -> Vec; + + pub fn merge(&mut self, other: &mut Doc) -> Result; + pub fn generate_sync_message(&mut self, state: &mut amsync::State) + -> Option>; + pub fn receive_sync_message(&mut self, state: &mut amsync::State, msg: &[u8]) + -> Result; +} +``` + +File operations on `Doc`: + +```rust +impl Doc { + pub fn write_text_file(&mut self, path: &str, content: &str) -> Result; + pub fn read_file (&mut self, path: &str) -> Result; + pub fn file_exists (&mut self, path: &str) -> bool; + pub fn file_hash (&mut self, path: &str) -> Result; + pub fn delete_file (&mut self, path: &str) -> Result<()>; + pub fn rename_file (&mut self, from: &str, to: &str) -> Result<()>; + pub fn list_files (&mut self) -> Result>; + pub fn list_file_paths(&mut self) -> Result>; + pub fn find_file_by_path(&mut self, path: &str) -> Result>; + pub fn read_file_meta (&mut self, fid: &str) -> Result>; + pub fn write_attachment(&mut self, path: &str, hash: &str, size: i64) + -> Result; +} +``` + +Directory operations on `Doc`: same shape as on `Vault`, prefixed +without the `_directory`/`_file` modifier (see source for exact names). + +Label operations on `Doc`: + +```rust +impl Doc { + pub fn create_label (&mut self, label: &str) -> Result<()>; + pub fn delete_label (&mut self, label: &str) -> Result<()>; + pub fn list_labels (&mut self) -> Result>; + pub fn get_label_heads (&mut self, label: &str) -> Result>; + pub fn restore_to_heads(&mut self, heads: &[ChangeHash]) -> Result<()>; + pub fn restore_to_time (&mut self, target_ms: i64) -> Result<()>; +} +``` + +`Doc` is the layer used by the wasm bridge (`agentsync-wasm`) and is +available without any of the OS-touching machinery (`Vault`, `host`, +`fs`, `net`). + +--- + +## 5. `Identity` and `Pubkey` + +```rust +pub struct Identity { /* opaque */ } + +impl Identity { + pub fn generate() -> Self; + pub fn from_seed(seed: [u8; 32]) -> Self; + pub fn seed(&self) -> Result<[u8; 32]>; + pub fn pubkey(&self) -> Pubkey; + pub async fn sign(&self, message: &[u8]) -> Result<[u8; 64]>; +} +``` + +There are two construction modes (file-backed and ssh-agent-backed), +exposed via free functions in the crate root and via +`Identity::load_from_file` / similar (see source). Both produce an +`Identity` whose `sign(...)` produces a standard ed25519 signature. + +```rust +pub struct Pubkey([u8; 32]); + +impl Pubkey { + pub fn from_bytes(bytes: &[u8]) -> Result; + pub fn from_ssh_string(s: &str) -> Result; + pub fn to_ssh_string(&self) -> String; + pub fn fingerprint_sha256(&self) -> String; + pub fn as_bytes(&self) -> &[u8; 32]; + pub fn verify(&self, message: &[u8], sig: &[u8]) -> bool; +} + +pub const PUBKEY_LEN: usize = 32; +pub const SIGNATURE_LEN: usize = 64; +``` + +The SSH wire format is fixed at `ssh-ed25519 ` per +[STORAGE.md § 7.4](./STORAGE.md#ssh-wire-format). + +--- + +## 6. Wire types + +```rust +pub enum HelloOp { Join, Create } + +pub enum Frame { + HelloHub { vault_id: String, hub_identity_pubkey: Vec, + hub_nonce: Vec, tls_cert_fingerprint: Vec, + vault_name: Option }, + HelloPeer { peer_identity_pubkey: Vec, peer_nonce: Vec, + op: HelloOp }, + ProofHub { sig: Vec }, + ProofPeer { sig: Vec }, + Sync { bytes: Vec }, + BlobFetch { hash: String }, + BlobPush { hash: String, bytes: Vec }, + Ping { ts: i64 }, + Pong { ts: i64 }, + Error { message: String }, +} +``` + +Encoding is MessagePack with named tags; see [WIRE.md § 2](./WIRE.md#frame-format). + +```rust +pub const HANDSHAKE_DOMAIN: &[u8] = b"agentsync-auth-v1"; // 17 bytes +pub const NONCE_LEN: usize = 32; + +pub fn random_nonce() -> [u8; 32]; +pub fn build_transcript( + hub_nonce: &[u8; 32], + peer_nonce: &[u8; 32], + tls_cert_fingerprint: &[u8], // 0 or 32 bytes + hub_pubkey: &[u8; 32], + peer_pubkey: &[u8; 32], +) -> Vec; +``` + +These helpers are exposed for clients implementing custom transports. + +--- + +## 7. `authorized_keys` parsing + +```rust +pub struct AuthorizedPeer { + pub pubkey: Pubkey, + pub label: String, +} + +pub fn parse_authorized_keys(content: &str) -> Vec; +pub fn render_authorized_keys(peers: &[AuthorizedPeer]) -> String; +``` + +Parser semantics: see [STORAGE.md § 7.2](./STORAGE.md#parser-rules). + +The deprecated `peers.md` format is also exposed for migration: + +```rust +pub const PEERS_FILE: &str = "peers.md"; +pub fn parse_peers_md(content: &str) -> Vec; +pub fn render_peers_md(peers: &[AuthorizedPeer]) -> String; +``` + +A reimplementation **MAY** omit the `peers.md` helpers; new vaults +**MUST NOT** create a `peers.md`. + +--- + +## 8. Error type + +```rust +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("io: {0}")] Io(#[from] std::io::Error), + #[error("automerge: {0}")] Automerge(#[from] automerge::AutomergeError), + #[error("automerge load: {0}")] AutomergeLoad(#[from] automerge::LoadChangeError), + #[error("invalid path: {0}")] InvalidPath(String), + #[error("not found: {0}")] NotFound(String), + #[error("already exists: {0}")] AlreadyExists(String), + #[error("auth failed: {0}")] Auth(String), + #[error("config: {0}")] Config(String), + #[error("protocol: {0}")] Protocol(String), + #[error("network: {0}")] Network(String), + #[cfg(not(target_arch = "wasm32"))] + #[error("notify: {0}")] Notify(#[from] notify::Error), + #[error("serde json: {0}")] SerdeJson(#[from] serde_json::Error), + #[error("msgpack encode: {0}")] MsgpackEncode(#[from] rmp_serde::encode::Error), + #[error("msgpack decode: {0}")] MsgpackDecode(#[from] rmp_serde::decode::Error), + #[error("websocket: {0}")] WebSocket(String), + #[error("vault: {0}")] Vault(String), + #[error("size limit exceeded: {0}")] TooLarge(String), + #[error("invalid utf8")] InvalidUtf8, + #[error("{0}")] Other(String), +} + +pub type Result = std::result::Result; +``` + +Variant semantics: + +| Variant | When | +|---|---| +| `Io` | underlying file/socket I/O failure | +| `Automerge` | Automerge operation failed (e.g., wrong type at a path) | +| `AutomergeLoad` | `doc.bin` was unparseable | +| `InvalidPath` | path failed normalization (see DOCUMENT.md § 5) | +| `NotFound` | file/label does not exist | +| `AlreadyExists` | create-when-exists, or non-recursive directory delete on non-empty | +| `Auth` | signature invalid, peer not authorized, agent refused | +| `Config` | malformed `config.toml` | +| `Protocol` | wire-protocol violation (unexpected frame, bad encoding) | +| `Network` | TCP/TLS/DNS issue distinct from I/O | +| `WebSocket` | tungstenite-level failure | +| `TooLarge` | exceeded `attachment_max_bytes` or `text_file_max_bytes` | +| `Vault` | vault-level invariant violation | +| `Other` | catch-all; **SHOULD** be replaced by a specific variant when possible | + +Callers **MUST** be prepared for new variants; this enum is not +`#[non_exhaustive]` today but **SHOULD** be treated as if it were. New +variants will be added in minor versions. + +--- + +## 9. `SyncHandle` trait + +```rust +#[async_trait] +pub trait SyncHandle: Send + Sync { + async fn register_peer( + &self, + out: mpsc::UnboundedSender, + pubkey: Option, + ) -> Result; + async fn unregister_peer(&self, peer_id: u64); + async fn generate_sync_message(&self, peer_id: u64) -> Result>>; + async fn receive_sync_message(&self, peer_id: u64, bytes: &[u8]) -> Result<()>; + async fn read_blob (&self, hash: &str) -> Result>; + async fn write_blob(&self, hash: &str, bytes: &[u8]) -> Result<()>; + async fn wait_doc_changed(&self); + async fn authorized_pubkeys(&self) -> Vec; + async fn authorized_peers(&self) -> Vec; + async fn disconnect_unauthorized_peers(&self, authorized: &[Pubkey]); +} +``` + +`SyncHandle` is the contract a network layer (the `net::client` and +`net::server` modules in the reference) talks to. It is exposed +publicly so a third party can build a custom transport while reusing +the engine. + +A reimplementation **SHOULD** treat this trait as advanced — most +consumers should use `Vault` directly. + +--- + +## 10. Constants and helpers + +```rust +pub const SCHEMA_VERSION: i64 = 1; + +pub const AUTHORIZED_KEYS_FILE: &str = "authorized_keys"; + +pub const DEFAULT_PORT: u16 = 443; +pub const DEFAULT_LISTEN_ADDR: &str = "0.0.0.0:443"; +pub const DEFAULT_LISTEN_ADDR_NO_TLS: &str = "0.0.0.0:80"; + +pub const USER_STATE_DIR: &str = ".agentsync"; +pub const USER_IDENTITY_FILENAME: &str = "id_ed25519"; + +pub fn normalize_rendezvous_url(url: &str) -> String; +pub fn normalize_with_scheme(url: &str) -> String; + +pub fn content_hash(bytes: &[u8]) -> String; // lowercase hex SHA-256 +``` + +These are normative — a reimplementation **MUST** match them on the +wire and on disk. See [WIRE.md § 10](./WIRE.md#10-constants). + +--- + +## 11. `host` module + +The `host` module re-exports the trait surface specified in +[HOST.md](./HOST.md): + +```rust +pub mod host { + pub use crypto::{Rng, Signer, TlsCert, TlsCertProvider}; + pub use filesystem::{DirEntry, FilesystemAdapter, FsEvent, Watcher}; + pub use runtime::{Clock, SpawnHandle, SpawnHandleImpl, Spawner}; + pub use storage::{BlobStorage, DocStorage, SnapshotEntry, SnapshotStorage}; + pub use transport::{Acceptor, Conn, ConnectOpts, Listener, TlsConfig, Transport}; + // pub use native::native_host; // factory +} +``` + +Available only when `cfg!(not(target_arch = "wasm32"))`. + +--- + +## 12. Stability summary + +| API | Stability | +|---|---| +| `Vault` methods listed above | stable; semver-protected | +| `Doc` methods listed above | stable; semver-protected | +| `Identity`, `Pubkey`, `Frame`, `HelloOp` | stable; semver-protected | +| `Error` enum | stable shape, expect new variants in minor versions | +| `host` traits | stable; semver-protected (see HOST.md) | +| `SyncHandle` trait | stable but advanced; **SHOULD NOT** be implemented externally without coordination | +| `debug_dump` | unstable; format may change | +| Internal modules (`net::*` types not listed here) | unstable; not part of the public API | + +A reimplementation in another language **MUST** preserve method names, +parameter order, and error semantics for items marked stable. + +--- + +## 13. Cross-references + +- [API-TS.md](./API-TS.md) — corresponding TypeScript surface. +- [DOCUMENT.md](./DOCUMENT.md) — schema operated on by `Doc`. +- [WIRE.md](./WIRE.md) — protocol used by `Vault::connect` / + `Vault::listen`. +- [HOST.md](./HOST.md) — traits in the `host` module. +- [STORAGE.md](./STORAGE.md) — what `Vault::storage_path` contains. diff --git a/specs/API-TS.md b/specs/API-TS.md new file mode 100644 index 0000000..970680d --- /dev/null +++ b/specs/API-TS.md @@ -0,0 +1,572 @@ +# API-TS.md — TypeScript SDK Public API + +> Normative for the published `@agentsync/sdk` API. See [SPEC.md § +> Conformance language](./SPEC.md#conformance-language). + +This document specifies the public API of the TypeScript SDK at +`sdks/typescript/`, published as `@agentsync/sdk`. It covers the +high-level `Vault` class, the wasm boundary it sits on, and the +JavaScript-side adapter interfaces a consumer can implement (storage, +transport). + +For byte-level semantics underlying these methods, see the linked +specs. + +--- + +## 1. Package layout + +The SDK has two entrypoints that target different runtimes: + +| Entrypoint | File | Wasm target | Built-in adapters | +|---|---|---|---| +| `@agentsync/sdk` | `src/index.ts` | `nodejs` | `MemoryStorage`, `NodeFsStorage`, `nodeWsTransport` | +| `@agentsync/sdk/web` | `src/web.ts` | `bundler` | `OpfsStorage` | + +A consumer **MUST** import from the entrypoint matching the target +runtime. Importing the wrong one will load a wasm module the runtime +cannot execute. + +The two entrypoints export the same `Vault`, `Doc`, `Identity`, +`Pubkey`, `SyncState`, frame codec, and type definitions. They differ +only in the bundled adapters. + +--- + +## 2. Top-level exports (both entrypoints) + +```ts +// Wasm-backed primitives (re-exported): +export { Identity, Pubkey, Doc, SyncState }; + +export function parseAuthorizedKeys(body: string): AuthorizedPeer[]; +export function renderAuthorizedKeys(entries: AuthorizedPeer[]): string; + +export function randomNonce(): Uint8Array; +export function buildTranscript( + hubNonce: Uint8Array, + peerNonce: Uint8Array, + tlsCertFingerprint: Uint8Array, + hubPubkey: Uint8Array, + peerPubkey: Uint8Array, +): Uint8Array; + +export function encodeFrame(frame: Frame): Uint8Array; +export function decodeFrame(bytes: Uint8Array): Frame; + +export function contentHash(bytes: Uint8Array): string; // lowercase hex SHA-256 +export function schemaVersion(): number; // == 1 +export function defaultPort(): number; // == 443 +export function normalizeRendezvousUrl(url: string): string; + +// High-level Vault: +export const Vault: { + create(opts: CreateOptions): Promise; + open (opts: OpenOptions): Promise; +}; + +// Type re-exports: +export type { + AuthorizedPeer, FileMeta, DirectoryMeta, Label, + Frame, FrameTag, HelloOp, + StorageAdapter, TransportAdapter, TransportConn, + VaultEvent, VaultOptions, ReconnectOptions, + CreateOptions, OpenOptions, VaultInstance, +}; +``` + +These exports correspond 1-to-1 with items in [API-RUST.md](./API-RUST.md) +where applicable. The wire-related helpers (`encodeFrame`, +`buildTranscript`, etc.) cross the wasm boundary; their byte-level +behavior is fully specified by [WIRE.md](./WIRE.md). + +### 2.1 Adapters bundled with `@agentsync/sdk` (Node/Bun) + +```ts +export class MemoryStorage implements StorageAdapter {} +export const memoryStorage: () => MemoryStorage; + +export class NodeFsStorage implements StorageAdapter {} +export const nodeFsStorage: (dir: string) => NodeFsStorage; + +export function nodeWsTransport(): TransportAdapter; +``` + +### 2.2 Adapters bundled with `@agentsync/sdk/web` + +```ts +export class OpfsStorage implements StorageAdapter {} +export const opfsStorage: () => OpfsStorage; +``` + +The web entrypoint does **not** ship a built-in transport; the +browser's global `WebSocket` is used directly. + +--- + +## 3. The `Vault` class + +The `Vault` class is the high-level entry point. Unlike the Rust +`Vault`, the TypeScript SDK implements the connection state machine in +JavaScript on top of wasm primitives — the wasm crate currently does +not expose a top-level `Vault` (see [HOST.md § 7.3](./HOST.md#73-current-state-of-the-reference)). + +### 3.1 Construction + +```ts +export interface VaultOptions { + storage: StorageAdapter; + identity?: Identity; + vaultId?: string; + rendezvousUrl?: string; + hubPubkey?: Uint8Array; // 32 bytes + name?: string; + transport?: TransportAdapter; // default: WebSocket +} + +export interface CreateOptions extends VaultOptions {} +export interface OpenOptions extends VaultOptions {} + +export class Vault { + static async create(opts: CreateOptions): Promise; + static async open (opts: OpenOptions): Promise; +} +``` + +`storage` is required for both `create` and `open` — the SDK has no +default storage. In the browser, pass `opfsStorage()`. In Node, pass +`nodeFsStorage(path)` or `memoryStorage()`. + +`identity` may be passed explicitly. If omitted: + +- For `create`: the SDK generates a fresh keypair and persists its seed + via `storage.saveIdentitySeed`. +- For `open`: the SDK loads the seed via `storage.loadIdentitySeed`. If + no seed exists, the call **MUST** error. + +`hubPubkey`, when set, pins the hub identity (TOFU). On a handshake +mismatch the connection **MUST** fail. See [AUTH.md § Hub trust](./AUTH.md#hub-trust). + +`transport`, when set, replaces the default. See § 6. + +### 3.2 Accessors + +```ts +class Vault { + vaultIdValue(): string; + identityRef(): Identity; + isConnected(): boolean; +} +``` + +### 3.3 File operations + +```ts +class Vault { + writeTextFile(path: string, content: string): Promise; + readTextFile (path: string): Promise; + fileExists (path: string): boolean; + deleteFile (path: string): Promise; + renameFile (from: string, to: string): Promise; + listFiles (): FileMeta[]; +} +``` + +`writeTextFile` returns the file's UUID (as a string). Path +normalization rules from [DOCUMENT.md § 5](./DOCUMENT.md#path-normalization) +apply. + +`fileExists` and `listFiles` are synchronous because they read in-memory +document state; the others are async because they may trigger +persistence. + +### 3.4 Directory operations + +```ts +class Vault { + createDirectory(path: string): Promise; + deleteDirectory(path: string, recursive?: boolean): Promise; + listDirectories(): DirectoryMeta[]; +} +``` + +`recursive` defaults to `false`. Setting it `true` performs the +atomic recursive delete specified in +[DOCUMENT.md § 3.3](./DOCUMENT.md#recursive-delete). + +### 3.5 Labels and history + +```ts +class Vault { + createLabel (name: string): Promise; + deleteLabel (name: string): Promise; + listLabels (): Label[]; + restoreToLabel(name: string): Promise; + restoreToTime(targetMs: number): Promise; +} +``` + +Label semantics match the Rust API. See +[API-RUST.md § 3.5](./API-RUST.md#35-history-and-labels). + +### 3.6 Connection management + +```ts +class Vault { + connect(): Promise; + connectWithReconnect(opts?: ReconnectOptions): Promise; + disconnect(): Promise; +} + +export interface ReconnectOptions { + maxAttempts?: number; + initialBackoffMs?: number; + maxBackoffMs?: number; +} +``` + +`connect` performs the four-message handshake (see +[WIRE.md § 4](./WIRE.md#handshake-normative)). On `Auth` failure the +promise **MUST** reject with an error whose message identifies the +failure; the connection **MUST NOT** be silently retried by `connect` +itself. + +`connectWithReconnect` retries on transport errors with exponential +backoff between `initialBackoffMs` (default 250) and `maxBackoffMs` +(default 30 000). It **MUST** stop on auth failure (the credentials +are wrong) and on the supplied `AbortSignal` if any. + +### 3.7 Lifecycle + +```ts +class Vault { + close(): Promise; +} +``` + +`close` flushes pending writes through `storage`, disconnects, and +releases the underlying wasm `Doc`. After `close`, all other methods +on the instance **MUST** throw. + +--- + +## 4. Wasm-backed primitives + +These are re-exported from `agentsync-wasm`. Their signatures match +the wasm-bindgen output. + +### 4.1 `Identity` + +```ts +class Identity { + static generate(): Identity; + static fromSeed(seed: Uint8Array): Identity; + seed(): Uint8Array; // 32 bytes + pubkey(): Pubkey; + sign(message: Uint8Array): Promise; // 64 bytes + free(): void; +} +``` + +Note: `sign` is async to mirror the Rust `Signer` trait, even though +file-backed identities sign synchronously. A consumer **SHOULD** +always `await` it. + +### 4.2 `Pubkey` + +```ts +class Pubkey { + static fromBytes(bytes: Uint8Array): Pubkey; // 32 bytes + static fromSshString(s: string): Pubkey; + toSshString(): string; // "ssh-ed25519 " + fingerprint(): string; // hex SHA-256 + bytes(): Uint8Array; + verify(message: Uint8Array, signature: Uint8Array): boolean; + free(): void; +} +``` + +### 4.3 `Doc` + +```ts +class Doc { + constructor(vaultId: string); + static load(bytes: Uint8Array): Doc; + save(): Uint8Array; + saveIncremental(): Uint8Array; + vaultId(): string; + heads(): Uint8Array[]; // each is 32 bytes + + merge(other: Doc): boolean; + generateSyncMessage(state: SyncState): Uint8Array | undefined; + receiveSyncMessage (state: SyncState, bytes: Uint8Array): boolean; + + // file ops (path is POSIX-normalized; see DOCUMENT.md) + writeTextFile(path: string, content: string): string; // returns FileId + readFile (path: string): string; + fileExists (path: string): boolean; + deleteFile (path: string): void; + renameFile (from: string, to: string): void; + writeAttachment(path: string, hash: string, size: number): string; + listFiles(): FileMeta[]; + + createDirectory(path: string): string; + deleteDirectory(path: string, recursive: boolean): void; + listDirectories(): DirectoryMeta[]; + + createLabel (name: string): void; + deleteLabel (name: string): void; + listLabels (): Label[]; + restoreToLabel(name: string): void; + restoreToTime(targetMs: number): void; + + free(): void; +} +``` + +`Doc` is a wasm-bindgen-managed object. Each method call crosses the +wasm boundary. Consumers **MUST** call `free()` when done with a `Doc` +unless they're using the high-level `Vault` (which manages its own +`Doc` lifetime). + +### 4.4 `SyncState` + +```ts +class SyncState { + constructor(); + static decode(bytes: Uint8Array): SyncState; + encode(): Uint8Array; + free(): void; +} +``` + +A `SyncState` is one peer's view of the Automerge sync protocol state +machine. The high-level `Vault` maintains one per active peer; a +consumer using `Doc` directly is responsible for managing them. + +--- + +## 5. Events + +```ts +export type VaultEvent = + | { kind: 'connecting'; url: string } + | { kind: 'connected'; hub_pubkey: Uint8Array; vault_id: string } + | { kind: 'disconnected'; reason: string } + | { kind: 'sync-progress'; outbound: boolean } + | { kind: 'doc-changed'; heads: Uint8Array[] } + | { kind: 'error'; message: string }; + +class Vault { + subscribe(listener: (e: VaultEvent) => void): () => void; + events(): AsyncIterableIterator; +} +``` + +`subscribe` returns an unsubscribe function. `events()` is the +`AsyncIterableIterator` form, suitable for `for await ... of`. + +The TypeScript event shape is **richer** than the Rust crate's +`VaultEventKind` enum (see [API-RUST.md § 3.7](./API-RUST.md#37-event-stream)). +This is intentional: the SDK orchestrates the connection state machine +in JS and has more state available to surface. + +--- + +## 6. Adapter interfaces + +A reimplementation of the SDK targeting a new runtime implements these +interfaces. + +### 6.1 `StorageAdapter` + +```ts +export interface StorageAdapter { + loadDoc(): Promise; + saveDoc(bytes: Uint8Array): Promise; + loadSyncState(peerKey: string): Promise; + saveSyncState(peerKey: string, bytes: Uint8Array): Promise; + loadIdentitySeed(): Promise; + saveIdentitySeed(seed: Uint8Array): Promise; + loadSnapshots(): Promise; + saveSnapshots(bytes: Uint8Array): Promise; + close(): Promise; +} +``` + +Contract: + +- `loadDoc` returns the saved `doc.bin` bytes, or `null` if no save has + ever occurred. +- `saveDoc(bytes)` replaces the saved document atomically. +- `loadSyncState(peerKey)` / `saveSyncState(peerKey, bytes)` persist + the per-peer Automerge sync state. `peerKey` is an opaque + implementation-chosen string (typically the hex pubkey). The adapter + **MUST NOT** parse `peerKey`. +- `loadIdentitySeed` / `saveIdentitySeed` persist the local + ed25519 seed (32 bytes). On Node, `nodeFsStorage` uses the format + defined in [STORAGE.md § 8.2](./STORAGE.md#private-file-format) so + the same identity can be used by the Rust binary. +- `loadSnapshots` / `saveSnapshots` persist `snapshots/index.json` + bytes — see [STORAGE.md § 3](./STORAGE.md#snapshotsindexjson). +- `close` releases any underlying handles (file locks, IndexedDB + handles, etc.). + +All saves **MUST** be atomic in the all-or-nothing sense (see +[STORAGE.md § 9](./STORAGE.md#atomic-write-summary)). + +### 6.2 `TransportAdapter` + +```ts +export interface TransportAdapter { + connect(url: string, opts?: TransportConnectOpts): Promise; +} + +export interface TransportConnectOpts { + pinnedCertFingerprint?: Uint8Array; // 32 bytes +} + +export interface TransportConn { + send(bytes: Uint8Array): Promise; + recv(): AsyncIterable; + channelBinding(): Uint8Array | null; + close(): Promise; +} +``` + +This mirrors the Rust `Transport` / `Conn` traits in +[HOST.md § 4](./HOST.md#transport-transport-conn-listener-acceptor). + +`channelBinding()` returns the SHA-256 of the hub's TLS cert DER if +the transport can recover it, else `null`. Returning `null` puts the +connection in degraded channel-binding mode. Browser `WebSocket` +**MUST** return `null` (the API does not expose the cert). + +`pinnedCertFingerprint` is informational. The transport **MAY** use +it to short-circuit a known mismatch before the handshake; the engine +also enforces it independently. + +### 6.3 Built-in adapters + +`MemoryStorage` keeps everything in JS-side `Map`s. Useful for tests +and ephemeral browser tabs. + +`NodeFsStorage(dir)` writes to a real `.agentsync/` directory at `dir`, +matching the layout in [STORAGE.md](./STORAGE.md). A `Vault` opened +with `NodeFsStorage(d)` is interoperable with a Rust binary opened on +the same `d`. + +`OpfsStorage()` writes to the browser's Origin Private File System, +keyed under a fixed root chosen by the SDK. Format details: +implementation-defined (no Rust reader is expected to load OPFS). + +`nodeWsTransport()` returns a `TransportAdapter` backed by the Node +`ws` package. The browser's global `WebSocket` is used as the implicit +default in `web.ts`. + +--- + +## 7. Type definitions + +```ts +export interface AuthorizedPeer { + pubkey: string; // "ssh-ed25519 AAAA..." + label: string; +} + +export interface FileMeta { + id: string; + path: string; + kind: 'Text' | 'Attachment'; + size: number; + created_at: number; + updated_at: number; + deleted_at?: number | null; + binary_hash?: string | null; +} + +export interface DirectoryMeta { + id: string; + path: string; + created_at: number; + deleted_at?: number | null; +} + +export interface Label { + name: string; + heads_b64: string; // base64 no-pad of N*32 bytes + created_at_ms: number; +} + +export type FrameTag = + | 'hello_hub' | 'hello_peer' | 'proof_hub' | 'proof_peer' + | 'sync' | 'blob_fetch' | 'blob_push' + | 'ping' | 'pong' | 'error'; + +export type HelloOp = 'join' | 'create'; + +export type Frame = + | { t: 'hello_hub'; vault_id: string; + hub_identity_pubkey: Uint8Array; hub_nonce: Uint8Array; + tls_cert_fingerprint: Uint8Array; vault_name?: string | null } + | { t: 'hello_peer'; peer_identity_pubkey: Uint8Array; + peer_nonce: Uint8Array; op: HelloOp } + | { t: 'proof_hub'; sig: Uint8Array } + | { t: 'proof_peer'; sig: Uint8Array } + | { t: 'sync'; bytes: Uint8Array } + | { t: 'blob_fetch'; hash: string } + | { t: 'blob_push'; hash: string; bytes: Uint8Array } + | { t: 'ping'; ts: number } + | { t: 'pong'; ts: number } + | { t: 'error'; message: string }; +``` + +Type-level note: `FileMeta.kind` is the capitalized form +(`'Text'` / `'Attachment'`) at the TS API surface, even though the +Automerge document stores it as the lowercase `"text"` / `"attachment"` +string. The wasm boundary maps between the two. A reimplementation +**MAY** expose either form on its API surface, but **MUST** persist the +lowercase form in the document. + +--- + +## 8. Error semantics + +The SDK propagates errors as plain `Error` instances with a string +message. There is no exception class hierarchy in v1. + +A reimplementation **SHOULD** expose error messages with stable +prefixes so consumers can pattern-match (e.g., `"auth: ..."`, `"protocol: ..."`). +The reference does not currently formalize this. + +--- + +## 9. Symmetry with the Rust API + +The TypeScript `Vault` is **not** a strict mirror of the Rust `Vault`: + +| Capability | Rust | TS | +|---|---|---| +| `connect`, `disconnect`, `connectWithReconnect` | yes | yes | +| `listen` (hub mode) | yes | **no** | +| `bind_directory` (filesystem watch) | yes | **no** | +| `materialize` | yes | **no** | +| `peer_count`, `authorized_pubkeys` accessors | yes | **no** | +| Adapter-pluggable storage | **no** (filesystem only) | yes | +| Adapter-pluggable transport | **no** | yes | +| Async iterator events (`events()`) | **no** (broadcast::Receiver) | yes | + +A reimplementation in another language **MAY** include or omit +features per its target environment but **MUST** document the gaps. + +--- + +## 10. Cross-references + +- [API-RUST.md](./API-RUST.md) — the Rust surface this layers on top + of (via `agentsync-wasm`). +- [HOST.md](./HOST.md) — the trait factoring the wasm `Host` is + expected to conform to in a future iteration. +- [WIRE.md](./WIRE.md) — protocol used by `Vault.connect`. +- [DOCUMENT.md](./DOCUMENT.md) — schema operated on by `Doc`. +- [STORAGE.md](./STORAGE.md) — formats `StorageAdapter` persists. diff --git a/specs/AUTH.md b/specs/AUTH.md new file mode 100644 index 0000000..1400cc9 --- /dev/null +++ b/specs/AUTH.md @@ -0,0 +1,400 @@ +# AUTH.md — Authentication and Authorization + +> Normative spec. See [SPEC.md § Conformance language](./SPEC.md#conformance-language) +> for RFC 2119 keyword usage. + +This document specifies the trust model: who can connect to a vault, +how the connection is authenticated, and what an attacker is and is +not prevented from doing. It cross-references [WIRE.md](./WIRE.md) for +byte-level handshake details and [STORAGE.md](./STORAGE.md) for the +file formats involved. + +--- + +## 1. Model + +agentsync's authentication model is SSH-shaped: + +- Each device has its own ed25519 identity keypair. There is no + shared secret across the vault. +- The vault has an `authorized_keys` file at its root listing the + public keys allowed to connect. This file is itself synced through + the vault. +- One peer in the vault runs as the *hub* (the peer with `--listen`). + The hub is a normal vault participant that happens to listen — it + has full plaintext access to the vault. +- Connecting peers pin the hub's identity pubkey on first connect + (TOFU), stored in their local `config.toml`. Subsequent mismatches + are refused. + +There is **no** concept of users, accounts, roles, or per-file +permissions in v1. Possession of an authorized device key grants full +read/write access to the entire vault. + +There is **no** end-to-end encryption above TLS. The hub holds vault +contents in plaintext on its filesystem. The hub **MUST** be a +trusted machine the operator controls. + +--- + +## 2. Identity keys + +### 2.1 Algorithm + +All identity keys are **ed25519** (RFC 8032). No other algorithms are +supported in v1. A reimplementation that allows other algorithms is +non-conformant. + +| Constant | Value | +|---|---| +| `PUBKEY_LEN` | 32 bytes | +| `SIGNATURE_LEN` | 64 bytes | +| Seed length | 32 bytes | + +### 2.2 Storage + +The default storage location is `~/.agentsync/id_ed25519` with +`0600` mode. The on-disk format is: + +``` +agentsync-identity-v1 +``` + +See [STORAGE.md § 8](./STORAGE.md#identity-files) for the full file +format and override paths. + +A reimplementation **MAY** also support agent-backed identities (see +§ 2.4). + +### 2.3 Wire format + +When a public key appears in `authorized_keys` or in `hub_pubkey`, it +is in OpenSSH's `authorized_keys` line format: + +``` +ssh-ed25519 [] +``` + +The base64 portion decodes to the SSH wire encoding: + +``` +u32_be(11) || "ssh-ed25519" (11 bytes) || u32_be(32) || pubkey (32 bytes) +``` + +(Total: 51 bytes.) See [STORAGE.md § 7.4](./STORAGE.md#ssh-wire-format) +for the parser contract. + +### 2.4 Agent-backed identities + +A peer **MAY** delegate identity signing to an external agent over +the SSH-agent protocol ([draft-miller-ssh-agent][ssh-agent-draft]). +This supports hardware-backed identities (Secretive, 1Password, +gpg-agent, YubiKey-Agent, OpenSSH `ssh-agent`). + +[ssh-agent-draft]: https://datatracker.ietf.org/doc/html/draft-miller-ssh-agent + +Configuration: + +```toml +[identity] +agent_socket = "/path/to/agent.sock" +agent_pubkey = "ssh-ed25519 AAAA..." # selects which key in the agent +``` + +Discovery order: + +1. `--identity-agent ` flag. +2. `[identity] agent_socket` in `config.toml`. +3. `$SSH_AUTH_SOCK` environment variable. + +If multiple keys are advertised by the agent, `agent_pubkey` selects +which one to use. If unset, the implementation **MAY** default to the +first ed25519 key in the agent's identity list. + +The agent **MUST** be asked to sign the same 177-byte transcript +defined in [WIRE.md § 4.2](./WIRE.md#42-transcript). A reimplementation +**MUST NOT** modify the transcript before sending to the agent. + +--- + +## 3. Handshake + +The complete handshake is specified in +[WIRE.md § 4](./WIRE.md#handshake-normative). This section names the +properties it provides and the failure modes a reimplementation +**MUST** handle. + +### 3.1 Mutual authentication + +Both sides sign and verify the transcript. After a successful +handshake, each side has cryptographic proof that: + +- the other side controls the corresponding ed25519 private key, and +- the same TLS channel was observed by both (channel binding — + see [WIRE.md § 4.5](./WIRE.md#45-channel-binding)). + +### 3.2 Authorization (hub side) + +After receiving `HelloPeer`, the hub **MUST** look up the connecting +peer's `peer_identity_pubkey` in the vault's `authorized_keys` file. If +absent, the hub **MUST**: + +1. Send `Frame::Error{message}` identifying the rejection reason. +2. Close the WebSocket cleanly. +3. **MUST NOT** send `ProofHub` or any subsequent frame. + +This means an unauthorized peer never sees a hub signature. The hub's +public identity is therefore not leaked to unauthorized scanners +beyond the cert (which is visible at TLS time anyway). + +### 3.3 Hub trust (peer side) + +The connecting peer pins the hub's identity in its local +`.agentsync/config.toml`: + +```toml +[vault] +hub_pubkey = "ssh-ed25519 AAAA..." +``` + +Behavior: + +- **Unset on connect:** the peer **SHOULD** prompt the operator with + the presented hub fingerprint and persist the answer. +- **Match:** the connection proceeds silently. +- **Mismatch:** the connection **MUST** be aborted before any sync + data flows, and a clear warning **MUST** be presented identifying + both keys. + +The TOFU prompt (reference UX): + +``` +The hub at wss://hub.example.com:443 has identity: + ssh-ed25519 AAAA...xyz + SHA256:7p/Q3F... + +This is the first time connecting. Trust this hub? [y/N] +``` + +The mismatch warning: + +``` +WARNING: HUB IDENTITY HAS CHANGED! +Stored: ssh-ed25519 AAAA...xyz +Presented: ssh-ed25519 AAAA...abc + +Either the hub's identity key was rotated, or someone is +impersonating it. Refusing to connect. To accept the new key, run: + agentsync hub trust ssh-ed25519 AAAA...abc +``` + +A reimplementation **MAY** offer: + +- `--accept-hub-key ` for non-interactive setups (equivalent + to pre-populating `hub_pubkey`). +- An out-of-band mechanism to clear the pin (the reference uses + `agentsync hub forget`). + +### 3.4 Channel binding + +If the peer's transport can observe the hub's TLS certificate, it +**MUST** verify that the SHA-256 of the cert DER matches the +`tls_cert_fingerprint` advertised in `HelloHub` and signed in the +transcript. Mismatch indicates a relayed MITM; the peer **MUST** +abort. + +Browser `WebSocket` cannot observe the cert. In that case, the hub +**MUST** still advertise the fingerprint (so non-browser peers can +verify), and the browser peer **MUST** treat its missing +channel-binding capability as degraded mode and **SHOULD** warn. + +### 3.5 Cipher agility / version + +There is no in-band protocol negotiation. The 17-byte transcript +prefix `agentsync-auth-v1` is a domain-separation tag. Future +protocol changes bump the tag (`agentsync-auth-v2`) and constitute a +coordinated break — old and new clients cannot interoperate. + +A reimplementation **MUST NOT** introduce a version-negotiation +phase without coordinating with the spec. + +--- + +## 4. `authorized_keys` + +### 4.1 Where it lives + +`authorized_keys` is a regular text file at the root of the synced +vault. It is stored inside the Automerge document like any other +text file (see [DOCUMENT.md § 6](./DOCUMENT.md#authorized_keys)) and +materialized to disk as `/authorized_keys`. + +A reimplementation **MUST NOT** invent a different location. + +### 4.2 Format + +See [STORAGE.md § 7](./STORAGE.md#authorized_keys-synced) for the +parser and renderer contract. Summary: + +- One `ssh-ed25519 [label]` line per authorized key. +- `#`-prefixed lines are comments. +- Blank lines are ignored. +- Legacy `- \`ssh-ed25519 ...\`` bullet lines are accepted (read-only). +- Unparseable lines are silently skipped — the file is also a UI for + humans to edit, and a single typo must not lock everyone out. + +### 4.3 Mutation + +A reimplementation **MAY** offer programmatic helpers +(`agentsync key add`, etc.). At the protocol level, all that matters +is that the file's contents change in the synced document. + +The hub **MUST** re-read `authorized_keys` whenever the document +changes and disconnect any currently-connected peer no longer +authorized. The reference implementation does this on every +doc-change notification. Removal is therefore eventually consistent — +see § 6.2. + +### 4.4 Bootstrap + +A new vault **SHOULD** be initialized with `authorized_keys` +containing the creator's pubkey, so the creator can immediately +connect from the same device they used to initialize. The reference +does this in `Vault::create`. + +To add a second device: + +1. On the new device, generate an identity (`agentsync key generate` + or `agentsync init` for a fresh vault). +2. On any device that already has the vault, append the new device's + pubkey as a line in `authorized_keys`. +3. The hub picks up the change (file watcher), re-parses, and the new + device can connect. + +There is no online "request access" flow in v1. Adding a peer is +strictly out-of-band. + +--- + +## 5. TLS + +### 5.1 Self-signed certs + +The hub generates a self-signed ed25519 X.509 certificate on first +run and persists it under `/.agentsync-server/` +(see [STORAGE.md § 5](./STORAGE.md#tls-material-hub-only)). Connecting +clients **MUST NOT** validate hostname, expiry, or chain — TLS trust +is delegated to the application-layer channel binding. + +This means agentsync works on hostname-less LAN deployments and behind +NATs without ACME / Let's Encrypt / public CAs. + +### 5.2 Reverse-proxy TLS termination + +A hub running behind a TLS-terminating reverse proxy (Fly.io, +Railway, Caddy) **MUST** be started with `--no-tls` (or equivalent), +which: + +- binds plain `ws://`, expecting the proxy to terminate TLS, +- emits a 32-byte zero `tls_cert_fingerprint` in `HelloHub`, +- puts every connection in degraded channel-binding mode. + +In this configuration: + +- The reverse proxy **MUST** be configured to refuse non-TLS inbound + traffic. +- Clients **SHOULD** still connect with `wss://` URLs (so the proxy + hop is encrypted). +- Channel binding is unavailable — peers rely solely on TLS + validation against the proxy's cert (handled by the underlying + TLS stack) plus identity-pubkey pinning. + +### 5.3 Cert rotation + +There is no automated cert rotation in v1. To rotate: + +1. Stop the hub. +2. Delete `/.agentsync-server/tls.crt` and `tls.key`. +3. Restart the hub. A fresh cert is generated. + +Existing connecting peers **MAY** see channel-binding mismatches on +reconnect; in degraded-binding mode (browser peer / `--no-tls` hub) +the rotation is invisible to clients. + +--- + +## 6. Threat model + +### 6.1 Defended + +| Attacker | How | +|---|---| +| Passive eavesdropper on the network | TLS 1.3 confidentiality + forward secrecy | +| Active MITM relaying the handshake | Channel binding (signature covers TLS cert fingerprint) | +| Active MITM running an impostor hub | TOFU `hub_pubkey` pinned in `config.toml` | +| Stolen `authorized_keys` by an outsider | No-op — pubkeys are public information | +| Compromised peer (private key extracted) | Bounded — that peer can sync until removed from `authorized_keys` | + +### 6.2 Not defended + +| Attacker | Reason | +|---|---| +| **Compromised hub (host root)** | Hub holds plaintext content. Out of scope. | +| **Compromised peer until convergence** | Removal from `authorized_keys` is eventually consistent; a peer with a stale copy **MAY** still accept the removed peer until convergence. | +| **Disk theft / backup theft** | Files on disk are plaintext (markdown by design). Use full-disk encryption on every machine running agentsync. | +| **Quantum-capable adversary** | TLS 1.3 + ed25519 are not PQ-safe. A future spec version **MAY** introduce hybrid PQ key exchange via the `agentsync-auth-v1` → `v2` tag bump. | +| **Denial of service** | The hub does not rate-limit handshakes. A reimplementation **SHOULD** apply standard transport-layer rate limits (proxy-level or OS-level). | + +### 6.3 Operational guidance + +- Treat the hub as a trusted compute node. Run it in a controlled + environment (your own VM, or a managed service whose operator you + trust). +- Use full-disk encryption on every device with a vault checkout. +- Rotate identity keys by removing the old line from + `authorized_keys` (eventually consistent) and adding a new one. +- Monitor `authorized_keys` for unexpected additions; this is the + audit trail. + +--- + +## 7. Cryptographic primitives + +| Primitive | Algorithm | +|---|---| +| Identity signature | Ed25519 (RFC 8032) | +| Hash for cert fingerprint | SHA-256 | +| Hash for blob naming | SHA-256 | +| Hash for Automerge change IDs | SHA-256 (Automerge internal) | +| RNG for nonces and seeds | OS CSPRNG (`OsRng` in the reference) | +| TLS suite | TLS 1.3 with rustls defaults | + +A reimplementation **MUST NOT** substitute weaker primitives. It +**MAY** add post-quantum hybrid layers under a new transcript domain +tag in a future spec version. + +--- + +## 8. Migration from the symmetric-key model + +Earlier drafts of agentsync used a single shared `vault_key` and an +HMAC-derived auth token. That model is **removed**. agentsync is +pre-release; there is no migration tooling. Existing vaults using the +old model **MUST** be re-initialized. + +A reimplementation interoperating with current code **MUST NOT** look +for an `AGENTSYNC_KEY` environment variable, a `--vault-key` flag, or +a `[key]` section in `config.toml`. The current model is identity-key +based throughout. + +--- + +## 9. Cross-references + +- [WIRE.md § 4](./WIRE.md#handshake-normative) — byte-level handshake. +- [STORAGE.md § 7](./STORAGE.md#authorized_keys-synced) — file format. +- [STORAGE.md § 8](./STORAGE.md#identity-files) — identity file format. +- [STORAGE.md § 5](./STORAGE.md#tls-material-hub-only) — TLS files. +- [DOCUMENT.md § 6](./DOCUMENT.md#authorized_keys) — `authorized_keys` + representation in the Automerge document. +- [HOST.md § 3.2](./HOST.md#32-signer) — `Signer` trait. diff --git a/specs/DOCUMENT.md b/specs/DOCUMENT.md new file mode 100644 index 0000000..22ea49d --- /dev/null +++ b/specs/DOCUMENT.md @@ -0,0 +1,436 @@ +# DOCUMENT.md — Automerge Document Schema + +> Normative spec. See [SPEC.md § Conformance language](./SPEC.md#conformance-language) +> for RFC 2119 keyword usage. + +This document specifies the logical schema of the single Automerge +document that is the source of truth for a vault. The on-disk byte +encoding of the document is Automerge's native columnar format and is +out of scope for this spec — see [STORAGE.md § doc.bin](./STORAGE.md#docbin) +for persistence rules. + +--- + +## 1. Top-level keys + +The Automerge document's root **MUST** be an Automerge map. It **MUST** +have these keys, with these types, and **MUST NOT** have any other +top-level keys (a reimplementation **SHOULD** ignore unknown keys for +forward compatibility, but **MUST NOT** create them). + +| Key | Automerge type | Cardinality | Purpose | +|---|---|---|---| +| `schema_version` | scalar `int` | exactly one | always `1` for this spec | +| `vault_id` | scalar `str` | exactly one | UUID identifying this vault | +| `directories` | map | exactly one | directory entries keyed by UUID | +| `files` | map | exactly one | file entries keyed by UUID | +| `labels` | map | exactly one | named recovery points keyed by label name | + +### 1.1 `schema_version` + +`schema_version` **MUST** equal `1`. A reimplementation that loads a +document with a different value **MUST** error out and **MUST NOT** +attempt to interpret the rest of the document. + +### 1.2 `vault_id` + +A UUID (canonical lowercase 8-4-4-4-12 form) chosen at vault creation. +A reimplementation **MUST NOT** mutate `vault_id` after creation. + +When a vault is opened, the implementation **SHOULD** verify that +`vault_id` in the loaded document matches the `vault_id` in the local +`config.toml` (if set). Mismatch indicates a vault swap and **SHOULD** +error. + +### 1.3 `directories`, `files`, `labels` + +Each is an Automerge map. `directories` and `files` are keyed by UUID; +`labels` is keyed by the label name. Their value schemas are specified +below. + +### 1.4 Genesis actor + +When creating a new document, the implementation **SHOULD** set the +Automerge actor ID to a stable value derived from `vault_id`. The +reference uses a SHA-256-based derivation so all peers initialising the +same vault produce comparable history. + +This is not strictly required for correctness — Automerge merges +regardless of actor — but using a deterministic genesis actor avoids +spurious history-divergence on multi-peer fresh setups. + +--- + +## 2. Files + +### 2.1 Map shape + +`root.files` is a flat Automerge map. Each entry's *key* is a freshly +generated UUID (the `FileId`). Each entry's *value* is an Automerge map +with the fields specified below. + +A reimplementation **MUST NOT** key `files` by file path. Path is +metadata; it changes on rename. The UUID key is the stable identity. + +### 2.2 File entry fields + +``` +files[""] = { + "meta": , + "content": , // text files only + "binary_hash": , // attachments only +} +``` + +`meta` is **required** for every file entry. + +Exactly one of `content` or `binary_hash` **MUST** be present, depending +on `meta.kind`: + +- `meta.kind == "text"` → `content` MUST be an Automerge Text object; + `binary_hash` MUST be absent. +- `meta.kind == "attachment"` → `binary_hash` MUST be a scalar string; + `content` MUST be absent. + +### 2.3 `meta` field schema + +``` +files[""]["meta"] = { + "id": , // UUID, equal to the parent map key + "path": , // POSIX-normalized path; see § 5 + "kind": , // "text" or "attachment" + "size": , // bytes + "created_at": , // ms since Unix epoch + "updated_at": , // ms since Unix epoch + "deleted_at": ?, // optional; ms since Unix epoch + "binary_hash": ?, // optional; mirror of entry.binary_hash for attachments +} +``` + +Fields: + +- `id` **MUST** equal the UUID used as the parent map key. (This + duplication is for convenience — a reimplementation **MUST** keep them + in sync.) +- `path` **MUST** be a non-empty POSIX path normalised per § 5. +- `kind` **MUST** be exactly the string `"text"` or `"attachment"`. +- `size` is the file's byte length. For text files it is the UTF-8 byte + length of the Text content; for attachments it is the size of the + blob. +- `created_at`, `updated_at` are wall-clock millisecond timestamps. They + are advisory — Automerge's own change graph is the authoritative + ordering — but they MUST be monotonic per file (i.e., `updated_at >= + created_at`). +- `deleted_at` indicates a soft-deleted file. When present, the entry + **MUST** be excluded from `list_files()` results but **MUST** remain + in the map (so renames and history are preserved). When absent (or + null), the file is live. +- `binary_hash` (in `meta`) is only set for attachments. It is + redundant with the entry's top-level `binary_hash` field; the two + **MUST** be kept in sync if both are written. + +### 2.4 Text body + +For `kind == "text"`, the file's content is an Automerge `Text` +object at `files[""]["content"]`. The Text's full string contents +are the body of the file. + +A reimplementation **MUST** edit the body via Automerge `splice_text` +(or equivalent) operations rather than wholesale replacement, so +concurrent edits CRDT-merge. A wholesale replacement is a valid Text +operation but defeats merge. + +### 2.5 Attachment body + +For `kind == "attachment"`, `files[""]["binary_hash"]` is a +scalar string equal to the lowercase hexadecimal SHA-256 of the +attachment's bytes. The actual bytes live in +`.agentsync/blobs/` (see [STORAGE.md § Blob store](./STORAGE.md#blob-store)) +and are exchanged via [WIRE.md § 7](./WIRE.md#7-blob-exchange). + +A reimplementation **MUST NOT** store attachment bytes inside the +Automerge document. The blob hash is the only document-side reference. + +### 2.6 Lookup by path + +A reimplementation **MUST** implement "find file by path" as a linear +scan over the live (non-deleted) entries comparing `meta.path` to the +normalized query path. There is no path index in v1. + +If two live entries share the same `path`, an implementation **MAY** +return either one (concurrent creates can produce this; CRDT semantics +do not enforce path uniqueness). A reimplementation **MAY** flag the +duplicate to the user. + +--- + +## 3. Directories + +### 3.1 Map shape + +`root.directories` mirrors `root.files`: a flat map keyed by UUID, each +value an Automerge map carrying directory metadata. + +``` +directories[""] = { + "id": , // UUID, equal to parent map key + "path": , // POSIX-normalized path + "created_at": , // ms since Unix epoch + "deleted_at": ?, // optional soft-delete marker +} +``` + +### 3.2 Implicit vs explicit directories + +A directory entry **MUST** exist in `directories` if any of the +following is true: + +- the user explicitly created it (e.g., to model an empty directory), +- it is the parent of a live file or directory, *and* an implementation + is configured to materialize parent dirs explicitly. + +The reference creates ancestor directory entries lazily when files are +written. A reimplementation **MAY** keep `directories` empty unless the +user explicitly creates one — but on filesystem materialization it +**MUST** create any missing ancestor directories on disk regardless. + +A directory entry's absence does not mean the directory does not exist +on disk; a directory always exists if any live file lists it as an +ancestor. + +### 3.3 Recursive delete + +`delete_directory(path, recursive=true)` **MUST** be a single Automerge +transaction that soft-deletes: + +- the directory entry, and +- every descendant file entry, and +- every descendant directory entry. + +This atomicity is a feature of one-Automerge-doc-per-vault; a +reimplementation **MUST** preserve it (otherwise concurrent peers may +observe inconsistent partial deletion states). + +--- + +## 4. Labels + +### 4.1 Map shape + +`root.labels` is an Automerge map keyed by label name. Each value is +either: + +- **(canonical, v1)** an Automerge map: the *object form*, or +- **(legacy)** a scalar `bytes`: the *bytes form*. + +A reimplementation **MUST** read both. It **MUST** write only the object +form. + +### 4.2 Object form + +``` +labels[""] = { + "heads": , // see § 4.4 for encoding + "created_at": , // ms since Unix epoch +} +``` + +`heads` is required. `created_at` is required for new labels; readers +**MUST** treat absence as `0` (epoch) for forward compatibility. + +### 4.3 Legacy bytes form + +A label whose value is a scalar `bytes` directly (no enclosing object) +**MUST** be interpreted as `heads` only, with `created_at` defaulting +to `0`. This form is preserved for backward compat with vaults created +by earlier reference versions. + +A reimplementation **SHOULD NOT** create new labels in the legacy form. +Whether to *upgrade* legacy labels in place to the object form is an +implementation choice; the reference does not. + +### 4.4 `heads` encoding + +The `heads` value encodes a set of Automerge `ChangeHash` values, each +of which is exactly 32 bytes. The encoding is **byte concatenation**: + +``` +heads = changeHash[0] || changeHash[1] || ... || changeHash[N-1] +``` + +A reimplementation **MUST** reject a `heads` value whose length is not +a multiple of 32. The order of hashes is *not* significant — Automerge +treats heads as a set — but readers **SHOULD** preserve order on +round-trip. + +The same encoding is used for the on-disk `snapshots/index.json` cache +([STORAGE.md § 3](./STORAGE.md#snapshotsindexjson)), where the bytes +are then base64-encoded for JSON. + +### 4.5 Label invariants + +- Label names are arbitrary UTF-8 strings. There is no length limit in + v1, but a reimplementation **SHOULD** reject names containing control + characters or excessive length (>= 1024 bytes). +- Two peers concurrently creating labels with the same name CRDT-merge + to one of the two values; this is acceptable. Concurrent + delete-create on the same name is also CRDT-resolved. + +### 4.6 Sync to local cache + +The on-disk `snapshots/index.json` is a derived view of `root.labels`. +After every label-affecting operation (create, delete, restore), the +implementation **SHOULD** rewrite the index from the document. The +reference does this synchronously. + +--- + +## 5. Path normalization + +All `path` fields in the document **MUST** be POSIX-normalized. +Specifically: + +- Use forward slashes only. On Windows, the implementation **MUST** + translate backslashes before storage. +- No leading slash. Paths are *vault-relative*, e.g., `"notes/todo.md"`, + not `"/notes/todo.md"`. +- No `.` or `..` segments. +- No empty segments (no `"a//b"`). +- Unicode strings **MUST** be normalised to NFC. +- Paths are case-sensitive on the wire and in the document. (Local + filesystem case-folding is the materialization layer's concern.) + +A reimplementation **MUST** validate paths at ingest (any API that +takes a path) and reject malformed inputs with a clear error. It +**MUST NOT** silently coerce. + +The reference's `path::normalize` function is the canonical implementation. + +--- + +## 6. `authorized_keys` + +`authorized_keys` is **a regular file** in the document — not a special +top-level key. It lives at: + +``` +files[""] = { + "meta": { "path": "authorized_keys", "kind": "text", ... }, + "content": , +} +``` + +The format of the Text body is specified in +[STORAGE.md § authorized_keys](./STORAGE.md#authorized_keys-synced). + +A reimplementation that needs to enumerate authorized peers **MUST**: + +1. Look up the file entry whose `meta.path == "authorized_keys"`. +2. Read its `content` Text as a string. +3. Parse using the rules from STORAGE.md § 7. + +A reimplementation **MUST NOT** invent an alternative location for the +authorized list. The choice of "synced through the document, like any +other file" is foundational to the trust model — see +[AUTH.md](./AUTH.md). + +A vault **SHOULD** be initialized with `authorized_keys` containing +the creator's pubkey, so the creator can connect immediately. The +reference does this in `Vault::create`. + +--- + +## 7. Soft deletes + +Both `files` and `directories` use a `deleted_at` timestamp for +soft-delete. Hard-deleting an entry from the map is **forbidden** in v1 +because: + +- Renames are tracked via stable UUIDs; deleting an entry breaks + history reconstruction. +- Point-in-time recovery to a moment when the file was live must still + return its content. +- Concurrent recreation at the same path must be distinguishable from + the original. + +Visibility rules: + +- `list_files()` **MUST** filter out entries where `deleted_at` is + present (non-null). +- `read_file(path)` **MUST** treat a deleted entry as not-found. +- `restore_to_heads(...)` may produce a state where an entry's + `deleted_at` is unset; the listing then becomes visible again. + +Filesystem materialization **MUST** delete the on-disk file when an +entry transitions from live to soft-deleted, and create it when the +reverse happens. + +--- + +## 8. ID generation + +All UUIDs in the document (`FileId`, `DirId`) **MUST** be generated by +a process that produces values which are unique with overwhelming +probability across all peers. The reference uses UUID v4 +(122 random bits via OS RNG). + +A reimplementation **MUST NOT** use sequential or path-derived IDs; +that breaks rename semantics and CRDT identity. + +--- + +## 9. Schema invariants (summary) + +A document is *well-formed* if all of the following hold at every +moment: + +1. `root.schema_version == 1`. +2. `root.vault_id` is a non-empty string and stable across all changes. +3. `root.directories`, `root.files`, `root.labels` are all map objects. +4. Every entry in `root.files` has a `meta` map with `id`, `path`, + `kind`, `size`, `created_at`, `updated_at`. +5. For every file entry, exactly one of `content` or `binary_hash` + is present (consistent with `meta.kind`). +6. Every entry in `root.directories` has `id`, `path`, `created_at`. +7. Every label value is either an object with `heads` or a `bytes` + scalar (legacy). +8. Every `path` is POSIX-normalized per § 5. + +A reimplementation **MAY** validate these invariants on load and +**SHOULD** report violations as a corrupt-document error rather than +silently coercing. + +CRDT merges **MAY** transiently create states that violate uniqueness +expectations (e.g., two live files at the same path). These are not +spec violations — they are valid CRDT outcomes the user must resolve. + +--- + +## 10. Conformance vectors + +The following vectors live under `specs/vectors/document/`: + +- **`vectors/document/empty.bin`** — a fresh document immediately after + `Doc::new(vault_id)` with no files, directories, or labels. +- **`vectors/document/with-files.bin`** — a document with a small set + of text files and one attachment, paired with a JSON manifest naming + the expected `list_files()` output. +- **`vectors/document/with-labels.bin`** — a document with one + object-form label and one legacy bytes-form label, plus expected + `list_labels()` output. +- **`vectors/document/with-deleted.bin`** — a document with both live + and soft-deleted entries, plus expected listing output. + +These are scaffolded in [vectors/README.md](./vectors/README.md). + +--- + +## 11. Cross-references + +- [STORAGE.md](./STORAGE.md) — how the document persists to disk and + how `authorized_keys` parses. +- [WIRE.md](./WIRE.md) — sync protocol that propagates document + changes; blob exchange for attachment bodies. +- [AUTH.md](./AUTH.md) — semantic interpretation of `authorized_keys`. +- [API-RUST.md](./API-RUST.md), [API-TS.md](./API-TS.md) — the + programmatic interfaces that read and write this schema. diff --git a/specs/HOST.md b/specs/HOST.md new file mode 100644 index 0000000..675df7f --- /dev/null +++ b/specs/HOST.md @@ -0,0 +1,558 @@ +# HOST.md — Platform-Abstraction Contract + +> Normative spec. See [SPEC.md § Conformance language](./SPEC.md#conformance-language) +> for RFC 2119 keyword usage. + +This document specifies the `Host` trait surface — the contract between +the agentsync engine and the platform it runs on. It is the integration +seam for porting agentsync to a new runtime (browser, Deno, Tauri, +Obsidian plugin, embedded device). + +The traits described here are exposed by the reference Rust crate as +`agentsync_core::host::*`. A reimplementation in another language +**SHOULD** preserve the same factoring even if the trait names differ, +because the engine's portability story depends on every "platform" +concern being injected through these traits rather than imported as a +global. + +--- + +## 1. Overview + +A `Host` is the bundle of platform capabilities a `Vault` needs. There +are nine sub-capabilities: + +| Capability | Trait | Required? | +|---|---|---| +| Async runtime | `Spawner` | yes | +| Wall-clock + timers | `Clock` | yes | +| Cryptographic RNG | `Rng` | yes | +| Outbound transport | `Transport` | yes | +| Inbound listener | `Listener` | hub only (`Option`) | +| Document storage | `DocStorage` | yes | +| Blob storage | `BlobStorage` | yes | +| Snapshot index storage | `SnapshotStorage` | yes | +| Filesystem watch + I/O | `FilesystemAdapter` | optional (storage-only mode supported) | +| TLS cert provisioning | `TlsCertProvider` | hub-with-TLS only (`Option`) | +| Identity signing | `Signer` | yes (per-vault, not on `Host`) | + +The `Host` itself is `Send + Sync + 'static`. All sub-traits are +`Send + Sync + 'static` unless noted. + +```rust +pub trait Host: Send + Sync + 'static { + fn spawner(&self) -> &dyn Spawner; + fn clock(&self) -> &dyn Clock; + fn rng(&self) -> &dyn Rng; + fn transport(&self) -> &dyn Transport; + fn listener(&self) -> Option<&dyn Listener>; + fn doc_storage(&self) -> &dyn DocStorage; + fn blob_storage(&self) -> &dyn BlobStorage; + fn snapshot_storage(&self) -> &dyn SnapshotStorage; + fn filesystem(&self) -> Option<&dyn FilesystemAdapter>; + fn tls(&self) -> Option<&dyn TlsCertProvider>; +} +``` + +A reimplementation of any of these traits **MUST** satisfy every +contract clause in the relevant section below. + +--- + +## 2. Runtime: `Spawner`, `Clock` + +### 2.1 `Spawner` + +```rust +pub trait Spawner: Send + Sync + 'static { + fn spawn(&self, fut: BoxFuture<'static, ()>) -> SpawnHandle; +} +``` + +Contract: + +- `spawn` **MUST** schedule the future for execution. It **MUST NOT** + block the caller. +- The returned `SpawnHandle` **MAY** be dropped without affecting the + task; the task continues to completion. +- `SpawnHandle` **MUST** support both `abort()` and `join()`: + - `abort()` cancels the task. Cancellation is best-effort (the future + may have already completed). + - `join()` resolves when the task completes (whether normally or via + abort). It **MUST NOT** panic. + +The reference impl is `TokioSpawner` wrapping `tokio::spawn`. A wasm +impl **SHOULD** wrap `wasm_bindgen_futures::spawn_local` (in which +case the `?Send` bound on returned futures matters). + +### 2.2 `Clock` + +```rust +pub trait Clock: Send + Sync + 'static { + fn now_ms(&self) -> i64; + fn sleep(&self, d: Duration) -> BoxFuture<'static, ()>; +} +``` + +Contract: + +- `now_ms()` returns wall-clock milliseconds since the Unix epoch + (1970-01-01T00:00:00Z). Negative values **MUST** be supported (the + spec is total) but are not produced under normal operation. +- `now_ms()` is used for `created_at` / `updated_at` timestamps and + TLS cert validity windows. It is **not** authoritative for ordering + — Automerge's change graph is. +- `sleep(d)` resolves after approximately `d` of real time. It **MAY** + fire late on a busy runtime; callers **MUST NOT** rely on tight + tolerance. + +The reference impl is `SystemClock` using `std::time::SystemTime` and +`tokio::time::sleep`. + +--- + +## 3. Crypto: `Rng`, `Signer`, `TlsCertProvider` + +### 3.1 `Rng` + +```rust +pub trait Rng: Send + Sync + 'static { + fn fill_bytes(&self, buf: &mut [u8]); +} +``` + +Contract: + +- `fill_bytes` **MUST** fill `buf` with cryptographically secure random + bytes. A non-CSPRNG **MUST NOT** be used for this trait. +- Used for: nonce generation in the handshake, identity seed + generation, UUID generation. + +Reference impl: `OsRngProvider` wrapping `rand_core::OsRng`. A wasm +impl **SHOULD** delegate to `crypto.getRandomValues()`. + +### 3.2 `Signer` + +```rust +#[async_trait(?Send)] +pub trait Signer: Send + Sync + 'static { + async fn sign(&self, msg: &[u8]) -> Result<[u8; 64]>; + fn pubkey(&self) -> Pubkey; +} +``` + +Contract: + +- `sign(msg)` returns an ed25519 signature over `msg` using the + signer's private key. Returns exactly 64 bytes. +- `pubkey()` returns the corresponding ed25519 public key (32 bytes). +- The returned signature **MUST** verify against `pubkey()` for the + same `msg` under standard ed25519 (RFC 8032). +- `sign` is async to accommodate external signers (ssh-agent, hardware + tokens, WebAuthn). File-backed signers complete synchronously and + **MAY** return immediately resolved futures. +- A signer **MAY** error if the user cancels (e.g., a Touch ID prompt). + The returned `Error::Auth(message)` **SHOULD** identify the failure + mode. + +A `Signer` is **per-vault**, not part of `Host` — different vaults may +use different identities. The reference's `IdentitySigner` wraps +either a file-backed or ssh-agent identity. + +### 3.3 `TlsCertProvider` + +```rust +#[async_trait(?Send)] +pub trait TlsCertProvider: Send + Sync + 'static { + async fn load_or_generate(&self, dir: &Path) -> Result; +} + +pub struct TlsCert { + pub cert_der: Vec, + pub key_der: Vec, +} +``` + +Contract: + +- If `/tls.crt` and `/tls.key` both exist, load and return + them. +- Otherwise, generate a fresh self-signed ed25519 keypair (10-year + validity), persist atomically (write-tmp + rename), and return. + See [STORAGE.md § 5](./STORAGE.md#tls-material-hub-only) for file + format and mode requirements. +- Returned `cert_der` and `key_der` **MUST** be DER-encoded. + `key_der` **MUST** be PKCS#8 (so rustls can consume it). + +This trait is hub-only. `Host::tls()` returns `None` for browser hosts +where TLS is the underlying socket's concern. + +--- + +## 4. Transport: `Transport`, `Conn`, `Listener`, `Acceptor` + +### 4.1 `Transport` + +```rust +#[async_trait(?Send)] +pub trait Transport: Send + Sync + 'static { + async fn connect(&self, url: &str, opts: ConnectOpts) -> Result>; +} + +pub struct ConnectOpts { + pub expected_hub_pubkey: Option, +} +``` + +Contract: + +- `connect` opens a connection to `url` (`wss://...` or `ws://...`) + and returns a duplex frame channel. +- The transport **SHOULD** apply normal connect timeouts internally; + the engine does not impose one. +- `expected_hub_pubkey` is informational — the transport itself does + not enforce it; the engine's handshake does. It is provided in case + the transport wants to log or pre-pin (e.g., for diagnostic UI). + +### 4.2 `Conn` + +```rust +#[async_trait(?Send)] +pub trait Conn: Send + 'static { + async fn send(&mut self, frame: Bytes) -> Result<()>; + async fn recv(&mut self) -> Result>; + fn channel_binding(&self) -> Option<[u8; 32]>; + async fn close(self: Box) -> Result<()>; +} +``` + +Contract: + +- `send(frame)` sends a binary WebSocket frame containing exactly the + bytes of one MessagePack-encoded `Frame`. Errors are unrecoverable — + the connection is dead. +- `recv()` returns `Ok(Some(bytes))` for the next received frame, + `Ok(None)` if the peer closed cleanly, or `Err(_)` on protocol + failure. +- `channel_binding()` returns the SHA-256 of the peer's TLS cert DER + (32 bytes), or `None` if the underlying transport cannot expose it + (plain `ws://`, browser `WebSocket`). Used by the handshake to + enforce channel binding (see [WIRE.md § 4.5](./WIRE.md#45-channel-binding)). + A `Conn` that returns `None` puts the engine in degraded + channel-binding mode. +- `close()` performs a clean WebSocket close. It is best-effort; an + error means the close didn't complete cleanly but the peer **SHOULD** + still treat the connection as dead. + +### 4.3 `Listener` + +```rust +#[async_trait(?Send)] +pub trait Listener: Send + Sync + 'static { + async fn bind( + &self, + addr: SocketAddr, + tls: Option, + ) -> Result>; +} + +pub struct TlsConfig { + pub cert_der: Vec, + pub key_der: Vec, +} +``` + +Contract: + +- `bind` binds a TCP listener to `addr` and returns an `Acceptor`. +- If `tls` is `Some`, the listener performs TLS termination on each + inbound connection using the supplied cert/key. If `None`, plaintext + WebSocket only. +- Returning from `bind` **MUST** mean the listener is fully bound and + ready to accept connections — there must be no race where a peer's + immediate `connect` fails because the listener is still starting. + +`Host::listener()` returns `None` for browser hosts (browsers cannot +bind listeners). + +### 4.4 `Acceptor` + +```rust +#[async_trait(?Send)] +pub trait Acceptor: Send + 'static { + async fn accept(&mut self) -> Result>>; + fn local_addr(&self) -> SocketAddr; + async fn close(self: Box) -> Result<()>; +} +``` + +Contract: + +- `accept()` blocks until a new inbound connection's WebSocket upgrade + completes, then returns the `Conn`. Returns `Ok(None)` after + `close()` has been called. +- `local_addr()` returns the actually-bound socket address (useful + when `bind` used port `0`). +- `close()` stops accepting *new* connections. In-flight `Conn`s are + unaffected and **MUST** be torn down by their owning peer task. + +--- + +## 5. Storage: `DocStorage`, `BlobStorage`, `SnapshotStorage` + +These three traits abstract the on-disk layout specified in +[STORAGE.md](./STORAGE.md). A reimplementation that targets a +non-filesystem backend (OPFS, IndexedDB, S3) implements these traits +against that backend. + +### 5.1 `DocStorage` + +```rust +#[async_trait(?Send)] +pub trait DocStorage: Send + Sync + 'static { + async fn load(&self) -> Result>>; + async fn save(&self, bytes: &[u8]) -> Result<()>; + async fn ensure_ready(&self) -> Result<()>; +} +``` + +Contract: + +- `load()` returns the saved document bytes, or `Ok(None)` if no save + has ever happened. +- `save(bytes)` replaces the stored document atomically. After `save` + returns `Ok`, a subsequent `load()` **MUST** return `Some(bytes)` + even after a crash. +- `ensure_ready()` performs any first-time setup (creating directories, + initializing a database). It **MUST** be idempotent. + +Atomicity for the filesystem reference is write-tmp + rename of +`doc.bin` (see [STORAGE.md § 2.3](./STORAGE.md#23-atomic-write)). A +non-filesystem reimplementation **MUST** provide an equivalent +all-or-nothing replacement. + +### 5.2 `BlobStorage` + +```rust +#[async_trait(?Send)] +pub trait BlobStorage: Send + Sync + 'static { + async fn has(&self, hash: &str) -> bool; + async fn get(&self, hash: &str) -> Result>; + async fn put(&self, bytes: &[u8]) -> Result; + async fn put_with_hash(&self, hash: &str, bytes: &[u8]) -> Result<()>; + async fn ensure_ready(&self) -> Result<()>; +} +``` + +Contract: + +- All hashes are lowercase hexadecimal SHA-256 (64 chars). +- `has(hash)` returns whether the blob is locally available. +- `get(hash)` returns the blob's bytes; errors if absent. +- `put(bytes)` computes the hash, stores the blob, returns the hash. +- `put_with_hash(hash, bytes)` is for when the hash is supplied + externally (e.g., from a wire frame). The implementation **MUST** + verify `SHA-256(bytes) == hash` and reject on mismatch with + `Error::Other("blob hash mismatch...")`. +- `ensure_ready()` initializes (creates directories) idempotently. + +Writes **MUST** be atomic in the same sense as `DocStorage::save`. + +### 5.3 `SnapshotStorage` + +```rust +#[async_trait(?Send)] +pub trait SnapshotStorage: Send + Sync + 'static { + async fn read(&self) -> Result>; + async fn write(&self, entries: &[SnapshotEntry]) -> Result<()>; + async fn ensure_ready(&self) -> Result<()>; +} + +pub struct SnapshotEntry { + pub label: String, + pub heads: Vec, + pub created_at_ms: i64, +} +``` + +Contract: + +- `read()` returns the cached label index. Returns an empty `Vec` if + no index exists yet (NOT an error). +- `write(entries)` replaces the index atomically. +- The on-disk format for the filesystem reference is JSON with + base64-no-pad heads — see [STORAGE.md § 3](./STORAGE.md#snapshotsindexjson). + A non-filesystem reimplementation **MAY** use any format internally + but **MUST** preserve the data. + +This is a *cache* of the document's `labels` map. A reimplementation +that does not provide cached fast-reads **MAY** make `read()` and +`write()` no-ops (the engine is correct without the cache). + +--- + +## 6. Filesystem: `FilesystemAdapter`, `Watcher` + +```rust +#[async_trait(?Send)] +pub trait FilesystemAdapter: Send + Sync + 'static { + async fn read(&self, path: &Path) -> Result>; + async fn write(&self, path: &Path, content: &[u8]) -> Result<()>; + async fn delete(&self, path: &Path) -> Result<()>; + async fn list(&self, path: &Path) -> Result>; + async fn exists(&self, path: &Path) -> bool; + async fn hash(&self, path: &Path) -> Result; + async fn create_dir_all(&self, path: &Path) -> Result<()>; + async fn remove_dir(&self, path: &Path) -> Result<()>; + fn watch( + &self, + path: &Path, + sink: UnboundedSender, + ) -> Result>; +} + +pub struct DirEntry { + pub path: PathBuf, + pub is_dir: bool, + pub size: u64, +} + +pub trait Watcher: Send + Sync {} + +pub enum FsEvent { + Touched(PathBuf), + Removed(PathBuf), + Renamed { from: PathBuf, to: PathBuf }, +} +``` + +Contract per method: + +- `read(path)` reads the entire file into memory. +- `write(path, content)` writes the file. Reimplementations **SHOULD** + use atomic write (write-to-tmp + rename) to avoid mid-write states + visible to the watcher. +- `delete(path)` removes the file; error if it does not exist. +- `list(path)` enumerates direct children. The order is unspecified. +- `exists(path)` returns presence (file or directory). +- `hash(path)` returns lowercase hex SHA-256 of the file's content. +- `create_dir_all(path)` creates the directory and any missing + ancestors. Idempotent — succeeds if the directory already exists. +- `remove_dir(path)` removes an empty directory. Errors on a non-empty + directory. +- `watch(path, sink)` installs a recursive watcher rooted at `path`. + Events flow into `sink` until the returned `Watcher` is dropped. + +The `Watcher` trait is just a marker — its only contract is that +dropping it stops the watch. Implementations **SHOULD** implement +`Drop` to release any underlying resources. + +`FsEvent` ordering follows the underlying OS watcher's ordering. The +engine treats events as advisory and reconciles against the document. + +`Host::filesystem()` returning `None` puts the engine in *storage-only +mode* — no filesystem materialization, no watcher. This is the +expected configuration for a browser app that holds the vault in OPFS +without binding to any user directory. + +--- + +## 7. Wiring + +### 7.1 Native factory + +The reference exposes a single factory: + +```rust +pub fn native_host(storage_path: PathBuf) -> Arc; +``` + +`storage_path` is the `.agentsync/` directory. The factory wires up: + +- `TokioSpawner`, `SystemClock`, `OsRngProvider` (stateless) +- `NativeDocStorage`, `NativeBlobStorage`, `NativeSnapshotStorage` + (each holds `storage_path`) +- `NativeFilesystem` (wraps the existing `NodeFsAdapter` for now) +- `NativeTransport`, `NativeListener` (placeholders pending the + cutover from the legacy `net` module) +- `NativeTlsProvider` + +A reimplementation **SHOULD** provide an analogous single +factory-by-path entry point. + +### 7.2 Hosts that aren't native + +A wasm/browser host implements only the subset the runtime supports: + +| Trait | Browser | Node | Native | +|---|---|---|---| +| `Spawner` | `wasm_bindgen_futures` | `tokio` (or vanilla) | `tokio` | +| `Clock` | `Date.now()` + `setTimeout` | `Date.now()` + `setTimeout` | `SystemTime` + `tokio::time::sleep` | +| `Rng` | `crypto.getRandomValues` | `crypto.getRandomValues` | `OsRng` | +| `Transport` | `WebSocket` | `ws` package | `tokio-tungstenite` | +| `Listener` | None | `ws` server | `tokio-tungstenite` | +| `DocStorage` | OPFS | `node:fs` | `tokio::fs` | +| `BlobStorage` | OPFS | `node:fs` | `tokio::fs` | +| `SnapshotStorage` | OPFS | `node:fs` | `tokio::fs` | +| `FilesystemAdapter` | None or FSAA | `node:fs` + `chokidar` | `notify` + `tokio::fs` | +| `TlsCertProvider` | None | `rcgen` (if hub) | `rcgen` | + +A trait that is `None`-able on `Host` (i.e., `listener`, `filesystem`, +`tls`) **MAY** be skipped. A required trait **MUST** be provided. + +### 7.3 Current state of the reference + +As of this spec, the wasm crate (`crates/agentsync-wasm`) does **not** +yet implement `Host`. It exposes lower-level primitives (the `Doc`, +`Identity`, `Pubkey`, `SyncState` types and frame codec) directly, and +the TypeScript SDK at `sdks/typescript/` assembles those plus +JS-implemented adapters into a working `Vault` class — see +[API-TS.md](./API-TS.md). The Host trait surface is currently used only +on native builds. + +This is a transitional state. A reimplementation that builds on top of +the wasm crate today **SHOULD** plan for the wasm `Host` to become +real, at which point the TypeScript SDK will route through it. + +--- + +## 8. Async-runtime portability + +Every async trait method uses `#[async_trait(?Send)]`. The `?Send` +relaxation matters because: + +- Native futures are typically `Send` (tokio is multi-threaded). +- Wasm futures carrying `JsValue` are **not** `Send` (no real threads). + +A reimplementation that wraps non-`Send` types in any of these futures +**MUST** preserve the `?Send` relaxation. A wrapper that requires +`Send` (e.g., re-imposing `async_trait` without `?Send`) breaks wasm. + +--- + +## 9. Conformance + +A `Host` implementation is conformant if: + +1. Each implemented trait satisfies every "MUST" in its section. +2. Method signatures match exactly (ignoring async-runtime sugar that + compiles to the same thing). +3. The atomicity guarantees on `DocStorage::save`, `BlobStorage::put*`, + and `SnapshotStorage::write` are preserved. +4. Optional capabilities (`listener`, `filesystem`, `tls`) are reported + honestly via `Option`. + +A reimplementation in another language **MAY** rename or rebundle +traits, but the contract clauses must be preserved verbatim. + +--- + +## 10. Cross-references + +- [SPEC.md](./SPEC.md) — overall architecture and crate layout. +- [WIRE.md](./WIRE.md) — uses `Transport` and `Conn`. +- [STORAGE.md](./STORAGE.md) — what the storage traits persist. +- [DOCUMENT.md](./DOCUMENT.md) — what `DocStorage` ultimately stores. +- [AUTH.md](./AUTH.md) — how `Signer` is used in the handshake. +- [API-RUST.md](./API-RUST.md), [API-TS.md](./API-TS.md) — public APIs + layered on top of `Host`. diff --git a/specs/SPEC.md b/specs/SPEC.md new file mode 100644 index 0000000..5bfbb7d --- /dev/null +++ b/specs/SPEC.md @@ -0,0 +1,300 @@ +# agentsync — Specification (v1) + +> **Status:** Normative root. This document and the per-concern specs it +> indexes are authoritative for any reimplementation of agentsync. Any +> divergence between code and spec is a bug; please file it. + +agentsync is a real-time, peer-to-peer sync engine for a directory of files. +A vault is one Automerge document. Peers connect over WSS to a designated +*hub* peer (any peer running with `--listen`) and converge via Automerge's +sync protocol. Authentication is per-device ed25519, gated by an SSH-style +`authorized_keys` file synced through the vault itself. Point-in-time +recovery and named labels are first-class operations on the document's +change history. + +This document is the index. Every claim about *behavior* (what the bytes on +the wire are, what the on-disk file contains, what the API guarantees) lives +in one of the per-concern specs below. + +## Conformance language + +The keywords **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** +in this and the linked specs are to be interpreted as in [RFC 2119][rfc2119] +and [RFC 8174][rfc8174] when, and only when, they appear in all capitals. + +A statement without these keywords is descriptive — it explains rationale +or implementation behavior but is not normative. A reimplementation that +violates a "MUST" is non-conformant; a reimplementation that violates a +"SHOULD" is conformant but discouraged. + +[rfc2119]: https://www.rfc-editor.org/rfc/rfc2119 +[rfc8174]: https://www.rfc-editor.org/rfc/rfc8174 + +## Document index + +| Document | Scope | +|---|---| +| [WIRE.md](./WIRE.md) | Wire protocol: framing, handshake bytes, sync exchange, error frames, connection state machine. | +| [AUTH.md](./AUTH.md) | Authentication and authorization model: identity keys, `authorized_keys`, hub TOFU, threat model. | +| [STORAGE.md](./STORAGE.md) | On-disk formats: `.agentsync/` layout, `doc.bin`, `snapshots/index.json`, `blobs/`, `config.toml`, identity files, TLS material. | +| [DOCUMENT.md](./DOCUMENT.md) | Automerge document schema: top-level keys, file tree representation, body encoding per file kind, label entries, invariants. | +| [HOST.md](./HOST.md) | Platform-abstraction contract: the `Host` trait surface a port must implement (runtime, transport, storage, filesystem, crypto). | +| [API-RUST.md](./API-RUST.md) | Public Rust API of `agentsync-core`: `Vault`, `Doc`, `Identity`, error types, stability policy. | +| [API-TS.md](./API-TS.md) | Public TypeScript API of `@agentsync/sdk`: `Vault` class, adapter interfaces, wasm boundary. | +| [vectors/](./vectors/README.md) | Golden test vectors a reimplementation can run as a conformance harness. | + +## What v1 is + +A vault is a directory of files synced in real time across peers, with the +following first-class properties: + +1. **Per-vault Automerge document.** All file metadata, directory metadata, + file contents, and labels live inside one Automerge document per vault. +2. **Peer-to-peer over WSS.** Every peer is the same binary. One peer runs + with `--listen` and is reachable on a public address (the *hub*). Other + peers connect to the hub and exchange Automerge sync messages over a + binary WebSocket framed in MessagePack. +3. **Per-device identity.** Each device has an ed25519 keypair. The hub + gates inbound connections by checking the connecting peer's pubkey + against an SSH-style `authorized_keys` file stored *inside the vault*. +4. **Channel-bound authentication.** TLS uses a self-signed cert generated + by the hub on first run; the application-layer handshake signature + covers the SHA-256 of the cert DER, defeating active relayed MITM. +5. **History as a first-class data structure.** The Automerge document is + the log. Labels are named pointers into that history. Restoring to a + past moment is additive — it produces forward-going changes that bring + the document state to match the past state. + +## Goals (v1) + +A v1 implementation **MUST**: + +1. Sync a local directory between peers running the same binary, over WSS, + in real time (sub-second propagation under nominal network conditions). +2. Converge concurrent edits across peers via Automerge CRDTs. +3. Authenticate every connection with ed25519 per-device identities and + reject any peer not in the vault's `authorized_keys`. +4. Encrypt all peer-to-peer traffic with TLS 1.3. +5. Persist the full Automerge document — including history — to + `.agentsync/doc.bin`. +6. Expose point-in-time recovery via [Automerge's][automerge] history + primitives. +7. Expose named recovery points (labels) that themselves sync between + peers via the document. + +A v1 implementation **SHOULD**: + +8. Distribute as a single static binary under 15 MB on Linux x86_64. +9. Cold-start a 10,000-file vault in under 5 seconds. +10. Run with zero external infrastructure — the "server" is the same + binary with `--listen`. + +[automerge]: https://automerge.org/ + +## Non-goals (v1) + +These are **explicitly out of scope** for v1. A reimplementation MAY +implement them as extensions, but they are not required for conformance. + +- End-to-end encryption above TLS. The hub holds vault contents in + plaintext on its filesystem; the threat model treats the hub as a + trusted machine the operator controls. See [AUTH.md § Threat + model](./AUTH.md#threat-model). +- Multi-tenant SaaS hosting. A vault is a self-contained primitive with + no concept of users, accounts, or roles. +- Per-file or per-directory access control. `authorized_keys` grants full + read/write to the entire vault. +- Hard key revocation. Removal from `authorized_keys` is eventually + consistent — a peer with a stale copy may still accept a removed peer + until convergence. +- Off-peer durability for snapshots and history. Backups are the + operator's responsibility (`restic`, `borg`, `rclone` against + `.agentsync/`). +- Conflict-resolution UI. Automerge merges deterministically; surfacing + the merge to a human is left to higher-level tooling. +- Per-file CRDT partitioning. v1 uses one Automerge document per vault. +- Binary file delta sync. Files large enough to not fit in the document + are content-addressed and stored whole; see [STORAGE.md § Blob + store](./STORAGE.md#blob-store). +- Permission systems, RBAC, mobile clients, web UI. + +## Architecture + +``` +┌────────────────────────────────────────────────────────────┐ +│ Local Peer │ +│ │ +│ Markdown files on disk │ +│ ↕ (notify crate / fsevents / inotify) │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ agentsync-core (Rust) │ │ +│ │ │ │ +│ │ FilesystemAdapter ↔ Vault ↔ Net │ │ +│ │ │ │ │ +│ │ Automerge Document │ │ +│ │ (in memory, includes full history) │ │ +│ │ ↕ │ │ +│ │ .agentsync/ (on-disk state) │ │ +│ │ - doc.bin (saved Automerge doc) │ │ +│ │ - snapshots/index.json │ │ +│ │ - blobs/ (attachments) │ │ +│ │ - config.toml │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ ↕ wss:// (TLS 1.3) │ +└──────────────────────────│──────────────────────────────────┘ + │ 4-message handshake, then + │ Automerge sync frames + ↓ +┌──────────────────────────────────────────────────────────────┐ +│ Hub Peer (agentsync --listen) │ +│ │ +│ Identical binary. Bound to a public address. │ +│ Holds the same .agentsync/ state on its own filesystem. │ +│ Generates a self-signed TLS cert on first run. │ +│ Fans out updates from one connected peer to all others. │ +└──────────────────────────────────────────────────────────────┘ + ↑ + │ other peers connect here + ┌────────────┴────────────┐ + │ │ + Other peer Other peer +``` + +Key properties: + +- **Symmetric peer code.** The hub is just another peer running the same + binary. No separate "server" codebase, no Postgres, no S3, no Docker + Compose stack. +- **One Automerge document per vault.** Tree structure, file metadata, and + text-file contents live inside this single document; binary attachments + are referenced by content-addressed blob hash. Operations across files + are atomic Automerge transactions. +- **The document is the log.** A reimplementation MUST NOT maintain a + separate append-only log. Automerge's change DAG is the history. +- **All peer traffic is TLS 1.3.** TLS termination MAY be delegated to a + reverse proxy (Fly.io, Railway, Caddy) by running the hub with + `--no-tls` over plain `ws://` and binding to localhost; this is for + deployment convenience, not security. +- **Channel-bound auth.** The handshake signature transcript covers the + SHA-256 of the TLS cert DER. See [WIRE.md § Handshake](./WIRE.md#handshake). + +## Crates and packages (current state) + +agentsync ships as multiple components. A reimplementation MAY collapse +these into fewer artifacts. + +| Component | Role | +|---|---| +| `agentsync-core` | Rust crate — the engine. Native targets get full functionality including networking, filesystem watching, and TLS. | +| `agentsync-wasm` | Rust crate — exposes a subset of `agentsync-core` to JavaScript via `wasm-bindgen`. | +| `agentsync-cli` | The `agentsync` binary. Thin wrapper over `agentsync-core`. | +| `@agentsync/sdk` | TypeScript package at `sdks/typescript/`. Wraps `agentsync-wasm` plus injected JS-side adapters into a high-level `Vault` class. | + +See [API-RUST.md](./API-RUST.md) and [API-TS.md](./API-TS.md) for the +public surfaces. + +## Versioning and compatibility + +agentsync uses three independent version surfaces. A reimplementation +**MUST** track all three. + +### 1. Handshake domain-separation tag + +The first 17 bytes of every signed handshake transcript are the ASCII +bytes `agentsync-auth-v1`. This tag **MUST** appear verbatim. It is *not* +a negotiated version field — there is no negotiation. Any future +incompatible change to the handshake bumps this tag (e.g., +`agentsync-auth-v2`) and is a coordinated break. See [WIRE.md § +Transcript](./WIRE.md#transcript). + +### 2. Document `schema_version` + +The Automerge document carries a `schema_version` integer at its root +(currently `1`). A reimplementation reading a document **MUST** check this +field; a value it does not understand is a fatal error. See +[DOCUMENT.md § Top-level keys](./DOCUMENT.md#top-level-keys). + +### 3. `snapshots/index.json` `schema_version` + +The on-disk snapshot index has its own `schema_version` (currently `1`). +This is local-cache state; on mismatch a reimplementation **MAY** discard +the file and rebuild from the document. + +### Backward-compatibility shims + +Two specific shims exist in the current code and **MUST** be preserved by +any reimplementation that wants to interoperate with existing vaults: + +- **`Frame::HelloHub.vault_name`** is `Option` and serde-default; older + hubs may omit it. +- **Legacy label encoding.** Labels stored as `ScalarValue::Bytes` + directly (not wrapped in an object) **MUST** be readable; new labels + **MUST** be written in the object form. See [DOCUMENT.md § + Labels](./DOCUMENT.md#labels). + +## Conformance criteria + +A reimplementation is *conformant* if all of the following hold: + +1. It implements every behavior marked **MUST** in this document and the + linked specs. +2. It can complete the handshake with a reference Rust hub and exchange + sync messages such that document state converges. The conformance test + for this is [vectors/handshake](./vectors/README.md). +3. It can load a `doc.bin` produced by the reference Rust implementation, + and a `doc.bin` it produces can be loaded by the reference. The + conformance test for this is [vectors/doc-roundtrip](./vectors/README.md). +4. It rejects (with the appropriate `Frame::Error`) any peer not in + `authorized_keys`. +5. It writes `snapshots/index.json` and `config.toml` in the schema + defined by [STORAGE.md](./STORAGE.md), such that the reference + implementation can read them. + +A reimplementation that targets only a *subset* of the API (e.g., a +read-only browser client without filesystem watching) is conformant for +that subset if it satisfies the relevant items above. The test-vector +manifest declares which vectors apply to which subset. + +## Implementation references + +The reference implementation lives in this repository. When the spec is +ambiguous, the reference implementation is authoritative — and the +ambiguity is a spec bug. + +- Rust core: `crates/agentsync-core/` +- Wasm bridge: `crates/agentsync-wasm/` +- CLI: `crates/agentsync-cli/` +- TypeScript SDK: `sdks/typescript/` +- E2E tests: `tests/` +- Test vectors: `specs/vectors/` + +## Spec maintenance + +Any change to wire format, on-disk format, or document schema **MUST** +include a paired update to the relevant per-concern spec in the same +commit or pull request. Reviewers should reject changes that leave the +spec stale. + +Test vectors **MUST** be regenerated whenever the wire or storage format +changes intentionally; tools for this live in +`specs/vectors/`. See [vectors/README.md](./vectors/README.md). + +## Open issues / non-normative roadmap + +The following items are described in the original product spec and are +*planned* but not yet specified normatively. They are listed here so the +roadmap is visible to reimplementers without polluting the normative +specs. + +- **Compaction.** `agentsync compact` to drop history older than a + retention window. Currently a milestone item; no spec exists. +- **Blob garbage collection.** Blobs in `.agentsync/blobs/` are never + reclaimed in v1. A future GC pass keyed on live `binary_hash` references + is anticipated. +- **`agentsync diff `** to inspect changes + between two history points. +- **Performance budget enforcement in CI.** A regression > 20% on any + named benchmark fails CI. Benchmarks themselves are not specified + normatively. + +These are not part of the conformance bar for v1. diff --git a/specs/STORAGE.md b/specs/STORAGE.md new file mode 100644 index 0000000..e984935 --- /dev/null +++ b/specs/STORAGE.md @@ -0,0 +1,522 @@ +# STORAGE.md — On-Disk Formats + +> Normative spec. See [SPEC.md § Conformance language](./SPEC.md#conformance-language) +> for RFC 2119 keyword usage. + +This document specifies every byte agentsync writes to disk. A +reimplementation that follows this document **MUST** be able to +read a `.agentsync/` directory produced by the reference and produce +one the reference can read. + +--- + +## 1. Vault directory layout + +A vault is a directory chosen by the user. Inside it, agentsync owns +one subdirectory: `.agentsync/`. The user's files live alongside it. + +``` +my-vault/ +├── notes/ +│ ├── research.md ← user file (synced) +│ └── todo.md ← user file (synced) +├── README.md ← user file (synced) +├── authorized_keys ← synced via the document; see § 7 +└── .agentsync/ ← agentsync-managed, MUST NOT be edited by hand + ├── config.toml ← per-vault config (§ 6) + ├── doc.bin ← saved Automerge document (§ 2) + ├── snapshots/ + │ └── index.json ← labels cache (§ 3) + └── blobs/ + └── ← content-addressed binary attachments (§ 4) +``` + +A reimplementation **MUST** use exactly these paths (relative to the +vault root and `.agentsync/` subdirectory) so vaults are portable +between implementations. + +The hub additionally owns a *sibling* directory `/.agentsync-server/` +holding TLS material; see § 5. + +The user's identity key is stored *outside* any vault, by default at +`~/.agentsync/id_ed25519`. See § 8. + +--- + +## 2. `doc.bin` + +### 2.1 Format + +`.agentsync/doc.bin` is the saved Automerge document for the vault. The +byte format is whatever `automerge::AutoCommit::save()` produces — a +columnar binary encoding that includes the full change history. + +A reimplementation **MUST** load this file via Automerge's `load` +constructor and **MUST** produce the file via `save` (or +`save_incremental` after a full save). + +Logical schema of the document is specified in [DOCUMENT.md](./DOCUMENT.md). + +### 2.2 Persistence cadence + +The reference saves under either of two conditions, whichever fires +first: + +- **Time-based:** every 1 second, if the document changed since the + last save. +- **Change-based:** after 100 accumulated Automerge changes. + +A reimplementation **MAY** use different thresholds. It **MUST** save: + +- on clean shutdown, +- before disconnecting (so peers don't see acknowledged changes that are + later lost), and +- when the caller explicitly asks (e.g., a `flush()` API). + +### 2.3 Atomic write + +Every write to `doc.bin` **MUST** be atomic. The reference uses: + +1. Write the full saved bytes to `/doc.bin.tmp`. +2. `rename(doc.bin.tmp, doc.bin)`. + +The temporary file name **MUST** be `doc.bin.tmp` (in the same directory +as `doc.bin`, so the rename is on the same filesystem). A +reimplementation **MAY** call `fsync` before the rename for added +durability; the reference does not. + +If the process crashes between (1) and (2), the implementation **MUST** +recover by loading the existing `doc.bin`. Any unflushed in-memory +changes are lost; this is acceptable because they were not yet +acknowledged to peers. + +### 2.4 Save format choice + +Implementations **MAY** use Automerge's `save_incremental()` for +performance. The on-disk file is still produced by full `save()` — the +reference re-saves the whole document each cycle and does not maintain +an external append log. + +--- + +## 3. `snapshots/index.json` + +### 3.1 Purpose + +A local cache of the vault's labels (named recovery points). The +authoritative copy lives inside the Automerge document — see +[DOCUMENT.md § Labels](./DOCUMENT.md#labels). This file is for fast +reads without having to load the document. + +### 3.2 Schema + +```json +{ + "schema_version": 1, + "labels": [ + { + "label": "", + "heads": "", + "created_at": + }, + ... + ] +} +``` + +Field requirements: + +- `schema_version` **MUST** equal `1` for v1. A reader encountering a + different value **MAY** discard the file and rebuild from the + document. +- `labels` is an array, in any order; the reference sorts by `label` + name when writing, but readers **MUST NOT** rely on order. +- `label` is the human-readable label name (a UTF-8 string). +- `heads` is the base64 (RFC 4648 "standard alphabet, no padding") + encoding of `N * 32` raw bytes, where `N` is the number of Automerge + change-hash values comprising the label. Implementations **MUST** + reject a `heads` whose decoded length is not a multiple of 32. +- `created_at` is the wall-clock millisecond timestamp at which the + label was created. + +### 3.3 Atomic write + +Same protocol as `doc.bin`: + +1. Write to `/snapshots/index.json.tmp`. +2. `rename` to `/snapshots/index.json`. + +The directory `snapshots/` **MUST** be created if absent before writing. + +### 3.4 Read semantics + +If the file is absent, a reimplementation **MUST** treat it as an empty +labels list (not as an error). This keeps fresh-clone behavior simple. + +### 3.5 Drift from document + +The on-disk index is a *cache*. The document is the source of truth. +After loading the document, an implementation **SHOULD** rewrite the +index from the document's labels map to keep them in sync. The +reference does this on every label-affecting operation. + +--- + +## 4. Blob store + +### 4.1 Layout + +``` +.agentsync/blobs/ +├── ← raw bytes of the blob +├── +└── ... +``` + +### 4.2 Naming + +Each filename **MUST** be the lowercase hexadecimal SHA-256 of the +blob's bytes — exactly 64 hex characters, no extension, no prefix +sharding, no separator. + +Example: `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` +is the empty blob. + +### 4.3 Atomic write + +To write a new blob: + +1. Compute `hash = lowercase_hex(SHA-256(bytes))`. +2. If `/blobs/` already exists, the write is a no-op. +3. Otherwise, write `bytes` to `/blobs/.tmp.`. +4. `rename(.tmp., )`. + +The temp filename **MUST** start with `.tmp.` so it is distinguishable +from a real blob if the rename fails partway and a stale file is left +behind (a reimplementation **MAY** sweep `.tmp.*` files at startup). + +### 4.4 Verified write + +When a blob arrives over the wire (via `blob_push` — see +[WIRE.md § 7](./WIRE.md#7-blob-exchange)), the implementation **MUST** +re-verify the SHA-256 before persisting. The reference's `put_with_hash` +function performs this check. + +### 4.5 What goes in blobs vs the document + +This decision is made at the *binding* layer (the filesystem-watch +adapter), not by the storage layer. The reference rule is: + +- Files whose extension is in `[sync] extensions` are stored as + Automerge `Text` *inside* the document. They are subject to + `[sync] text_file_max_bytes` (default 1 MiB). +- All other allowed files are stored as blobs and referenced by hash + in the document. They are subject to `[sync] attachment_max_bytes` + (default 10 MiB). + +Both limits are enforced *before* writing to the document or the blob +store; oversized files **MUST** be rejected with a clear error. + +### 4.6 Garbage collection + +Blobs are **never** garbage-collected in v1. A blob persists once +written, even if no live document state references it. A future GC +mechanism is anticipated; reimplementations **MAY** implement one but +**MUST NOT** assume the reference does so. + +--- + +## 5. TLS material (hub only) + +The hub stores its self-signed TLS keypair in a sibling of `.agentsync/`: + +``` +/.agentsync-server/ +├── tls.crt ← X.509 cert in DER form +└── tls.key ← ed25519 private key in PKCS#8 DER form, mode 0600 +``` + +(The reference computes this directory as `/../.agentsync-server` +where `` is the `.agentsync/` directory.) + +### 5.1 Generation + +If `tls.crt` and `tls.key` both exist on hub start, they **MUST** be +loaded as-is. + +If either is missing, the hub **MUST** generate a fresh self-signed +ed25519 keypair, valid for 10 years from the current wall-clock time, +and write both files atomically (write-tmp + rename) before serving any +connections. The reference uses `rcgen` for generation. + +### 5.2 File modes + +`tls.key` **MUST** be created with mode `0600` (owner read/write only) +on POSIX systems. A reimplementation on Windows **MAY** rely on +filesystem ACLs. + +`tls.crt` has no mode requirement. + +### 5.3 Cert content + +The cert's Common Name, SAN list, and validity dates are not normatively +constrained beyond the 10-year lifetime SHOULD. Connecting peers +**MUST NOT** validate any of these (see [WIRE.md § 1.2](./WIRE.md#12-tls)). + +--- + +## 6. `config.toml` + +`.agentsync/config.toml` holds per-vault configuration. The format is +TOML. + +### 6.1 Schema + +```toml +[vault] +id = "" # optional locally; the doc carries the canonical vault_id +name = "" # optional +rendezvous_url = "wss://hub.example:443" # optional +hub_pubkey = "ssh-ed25519 AAAA..." # optional, set on TOFU + +[identity] +path = "" # optional; default is ~/.agentsync/id_ed25519 +agent_socket = "" # optional, mutually exclusive with path +agent_pubkey = "ssh-ed25519 AAAA..." # optional, selects which key in the agent + +[sync] +extensions = ["md", "markdown"] # which extensions to treat as text +include = [] # additional glob patterns (relative to vault root) +attachment_max_bytes = 10485760 # 10 MiB +text_file_max_bytes = 1048576 # 1 MiB +log_retention_days = 30 # reserved; not yet enforced +``` + +### 6.2 Defaults + +| Field | Default if absent | +|---|---| +| `vault.id` | (none — derived from `doc.bin`) | +| `vault.name` | (none) | +| `vault.rendezvous_url` | (none — vault is local-only) | +| `vault.hub_pubkey` | (none — peer prompts on first connect) | +| `identity.path` | `~/.agentsync/id_ed25519` | +| `identity.agent_socket` | (none — file-backed identity) | +| `identity.agent_pubkey` | (none) | +| `sync.extensions` | `["md", "markdown"]` | +| `sync.include` | `[]` | +| `sync.attachment_max_bytes` | `10485760` (10 MiB) | +| `sync.text_file_max_bytes` | `1048576` (1 MiB) | +| `sync.log_retention_days` | `30` | + +A reimplementation **MUST** treat any absent section or field as its +default. Unknown fields **SHOULD** be ignored to permit forward-compat +extensions. + +### 6.3 Mutual exclusion + +`identity.path` and `identity.agent_socket` **MUST NOT** both be set +non-empty. If both are present, an implementation **MUST** error out +during config load. + +### 6.4 Atomic write + +The reference currently writes `config.toml` directly via `fs::write`, +which is *not* atomic. A reimplementation **SHOULD** use the +write-tmp + rename pattern for `config.toml` as well, especially for +TOFU updates (where `hub_pubkey` is appended after first connect). + +--- + +## 7. `authorized_keys` (synced) + +The vault's authorized peers list lives at `authorized_keys` *at the +root of the vault directory* (i.e., alongside the user's files), not +under `.agentsync/`. It is synced through the document like any other +text file. See [DOCUMENT.md § authorized_keys](./DOCUMENT.md#authorized-keys) +for its representation in the Automerge document. + +### 7.1 Format + +``` +# agentsync authorized_keys +# +# One ssh-ed25519 public key per line. +# Lines starting with '#' are comments. + +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... chris-macbook +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... chris-iphone +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... homelab-nas +``` + +### 7.2 Parser rules + +A reimplementation **MUST** accept: + +- Lines starting with `#` — comments, ignored. +- Empty lines — ignored. +- Lines of the form `ssh-ed25519 [label]` where: + - `ssh-ed25519` is the literal key type. No other algorithms are + supported in v1. + - `` is the OpenSSH wire format of the public key (a 32-byte + ed25519 public key wrapped in the standard SSH framing — see § 7.4 + below). + - `label` is everything after ``, trimmed. May be empty. +- Leading whitespace **MUST** be trimmed before parsing. + +A reimplementation **MUST** also accept the legacy bullet form +`- \`ssh-ed25519 ...\` — label` for backward compatibility with vaults +that used the deprecated `peers.md` format. Backticks and the leading +`- ` or `* ` **MUST** be stripped before normal parsing. + +Unparseable lines **MUST** be silently skipped (not error). This is +because `authorized_keys` is also the user-facing UI for adding peers, +and a typo on one line should not lock everyone else out. + +### 7.3 Render format + +When agentsync writes `authorized_keys` programmatically, the output +**MUST** be: + +``` +# agentsync authorized_keys +# +# One ssh-ed25519 public key per line. Lines starting with '#' are +# comments. Paste `agentsync key show` output from any device you +# want to authorize. + + + +... +``` + +Each key line is `ssh-ed25519 ` or `ssh-ed25519