diff --git a/.changeset/delete-ack-durability.md b/.changeset/delete-ack-durability.md new file mode 100644 index 0000000000..54fc64dccf --- /dev/null +++ b/.changeset/delete-ack-durability.md @@ -0,0 +1,10 @@ +--- +"@electric-ax/durable-streams-server-rust": patch +--- + +fix: an acked DELETE is now durable before the 204 — the data-file and +sidecar unlinks (plus a parent-directory fsync) previously ran on a +detached background task, so a crash right after the ack resurrected the +stream with all its data on the next boot. Soft deletes (fork-referenced +streams) likewise persist the flag before acking. Found by the +crash/fault simulation (seed 20387; details in `CRASH_SIM_FINDINGS.md`). diff --git a/.changeset/replicated-durability.md b/.changeset/replicated-durability.md new file mode 100644 index 0000000000..9aa48439e6 --- /dev/null +++ b/.changeset/replicated-durability.md @@ -0,0 +1,14 @@ +--- +"@electric-ax/durable-streams-server-rust": minor +--- + +feat: `--durability replicated` — Kafka-style durability via quorum +replication (openraft) instead of local fsync. An append acks once a quorum +of replicas has committed and applied it; no fsync anywhere on the hot path. +Any node accepts writes (forward-to-leader with read-your-writes), every node +serves reads, log-first apply keeps replicas byte-identical through +fail-over. Includes restart-rejoin (durable vote + manifest snapshots with +mesh byte-fetch), live membership change (`/_repl/add-learner`, +`/_repl/change-membership`), `REPL_STATS`/`/_repl/status` observability, and +a 3-node deploy kit (`deploy/replicated/`: local cluster, docker compose, +k8s, smoke incl. leader-kill and restart-rejoin). diff --git a/.changeset/wal-1m-cardinality.md b/.changeset/wal-1m-cardinality.md new file mode 100644 index 0000000000..00ec1a9396 --- /dev/null +++ b/.changeset/wal-1m-cardinality.md @@ -0,0 +1,9 @@ +--- +"@electric-ax/durable-streams-server-rust": patch +--- + +perf: fix the high-stream-cardinality (200k–1M streams) write cliff — O(1) WAL +checkpoint drain, checkpoint fully off the async runtime with concurrent shards +and a resident tails map, meta sidecar flush moved from per-append to the +checkpoint, single registry lookup per append. 1M streams now sustains 1.11M +ops/s on 16 vCPU (was 862k, with 405 ms worst-case latency now 150 ms). diff --git a/.changeset/wal-multi-segment-recovery.md b/.changeset/wal-multi-segment-recovery.md new file mode 100644 index 0000000000..ad289405ec --- /dev/null +++ b/.changeset/wal-multi-segment-recovery.md @@ -0,0 +1,12 @@ +--- +"@electric-ax/durable-streams-server-rust": patch +--- + +fix: WAL recovery no longer loses acked data when the WAL spans multiple +segments. Boot re-preallocated the first segment unconditionally, so a sealed +(exactly-packed) `1.wal` grew a zero tail that replay mis-read as the end of +the durable log — dropping every later segment's acked records and truncating +the per-stream files to the stale frontier — and a checkpoint-recycled `1.wal` +was recreated empty, making replay recover nothing. Boot now opens existing +segments non-destructively. Found by the new seeded crash/fault simulation +(`src/wal/sim_tests.rs`, findings in `CRASH_SIM_FINDINGS.md`). diff --git a/.changeset/wal-quiet-stream-torn-tail.md b/.changeset/wal-quiet-stream-torn-tail.md new file mode 100644 index 0000000000..02bfda841c --- /dev/null +++ b/.changeset/wal-quiet-stream-torn-tail.md @@ -0,0 +1,13 @@ +--- +"@electric-ax/durable-streams-server-rust": patch +--- + +fix: recovery now truncates a torn, never-acked tail on streams with no +surviving WAL record and no checkpoint tails entry (e.g. a stream created +after the last checkpoint whose only in-flight append was torn by power +loss) — previously the torn fragment became reader-visible. The `.meta` +sidecar persists a `durable_tail` proof (riding along on existing fsynced +meta writes; no new hot-path fsyncs) and recovery seeds every stream's +durable frontier from it. Old sidecars keep the previous behavior until +their next natural meta write. Found by the crash/fault simulation +(seed 20230; details in `CRASH_SIM_FINDINGS.md`). diff --git a/docs/superpowers/specs/2026-07-02-wal-crash-simulation-design.md b/docs/superpowers/specs/2026-07-02-wal-crash-simulation-design.md new file mode 100644 index 0000000000..48678160a8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-02-wal-crash-simulation-design.md @@ -0,0 +1,110 @@ +# WAL crash/recovery randomized simulation — design + +**Date:** 2026-07-02 +**Goal:** find correctness issues in the durable-streams Rust server's recovery path under +network failures (client drops), disk failures (power-loss page-cache loss, torn writes), +and arbitrary crash points — via seeded, reproducible randomized simulation. + +## Why this shape + +The existing coverage (`src/wal/e2e_tests.rs`, `src/wal/recovery.rs` unit tests) is +deterministic: each test hand-picks one crash scenario. A randomized simulator explores the +scenario space — interleavings of creates/appends/closes/forks/checkpoints, crash points, +and fault combinations — that nobody thought to write a test for. + +Approaches considered: + +1. **In-process randomized crash simulation (chosen).** A `#[cfg(test)]` module that drives + the real handler path (`handlers::handle`), simulates crashes exactly like the existing + e2e `Harness` (stop committers, drop store + WalSet, keep the data dir), injects disk + faults constrained to the documented fault model, and re-runs the real boot sequence + (`WalSet::open` → sidecar pass → `wal::recovery::recover` → `reset_after_recovery`). + Deterministic per seed; can consult internal state (durable LSN, checkpoint tails) to + keep injected faults inside the fault model. +2. Black-box process-level fuzzer (kill -9 the real binary over HTTP). Higher socket-layer + fidelity, but kill -9 preserves the page cache, so it cannot simulate power loss — the + most interesting failure class here — and reproduction is flaky. +3. cargo-fuzz on the WAL codec. Narrow; the codec already has CRC framing + unit tests. + +## Fault model (what the simulator may break) + +Grounded in ARCHITECTURE.md + recovery.rs: + +- **Process crash:** committers stop, process state vanishes, all file bytes written so far + survive (page cache == disk from the test's point of view). +- **Power loss / disk failure**, applied on top of a crash: + - _Per-stream data files_ are fsynced only at checkpoint (and at recovery repair). Any + byte beyond a stream's last checkpointed durable tail may be lost or garbage. The + simulator may truncate, zero, or scribble the region past that floor. + - _WAL segments_ are fdatasync'd up to the durable LSN (that is what releases acks). + Bytes of records with `lsn > durable_lsn` at crash time may be torn: the simulator + scans the segment with the real codec, finds the byte offset where staged-but-unacked + records start, and corrupts/zeroes a random suffix from a random point at or past it. + - `.meta` sidecars: create/close fsync them; the lazy tail flush does not. The simulator + does not corrupt sidecars in v1 (identity durability is a separate contract). +- **Network failure:** a client connection dropping mid-append is modeled by aborting the + spawned append task at a random await point (tokio cancellation), and by crashing with + appends still in flight. Such appends are **maybe-applied**: the oracle accepts their + presence or absence, but never a torn fragment of them. + +## Workload generator + +Seeded xorshift PRNG (no new dependencies; `rand` stays out of the dependency tree — the +existing crate has zero dev-deps and tests use std only). Per step, weighted choice of: + +- create stream (octet or JSON content type) +- append a self-describing record (`#|` for octet; `{"s":..,"i":..}` for + JSON) via the real POST path — sizes varied, occasionally multi-KB +- append with cancellation: spawn, then abort after a random yield count (maybe-applied) +- close a stream (real POST close path) +- fork a stream at a random offset ≤ parent tail (exercises `file_base > 0` recovery) +- delete a stream (recovery must not resurrect it) +- checkpoint a random shard (`shard.checkpoint()`), then refresh that shard's per-stream + durable-tail floors from `read_durable_tails()` (governs data-file fault legality) +- read a random range via the real GET path and check it against the oracle + +## Crash / fault / recover cycle + +Each seed runs G generations (default 4). Per generation: run K workload steps, then +crash (with 0..3 appends deliberately still in flight), then with independent +probabilities inject data-file faults and WAL-suffix faults as bounded above, then boot +the real recovery sequence and run the oracle. Subsequent generations continue the +workload on the recovered store — recovery-of-recovery bugs (stale `appender.written`, +tail/watch mismatches) only show up this way. + +## Oracle (checked after every recovery, and on every read) + +Per stream, the oracle tracks: records issued (unique tokens, in order), ack status of +each (acked / maybe / rejected), closed status, expected file_base. + +1. **No loss:** the recovered data file's bytes parse as a concatenation of whole issued + records, in issue order, containing **every acked record** — i.e. an ordered + subsequence of issued records ⊇ acked records. (Maybe-applied records may appear or + not; nothing else may.) +2. **No torn record:** the parse consumes the file exactly — no trailing fragment, no + interior garbage. For JSON streams every recovered record re-parses as JSON. +3. **Tail consistency:** `Shared.tail == file_base + file_len`, `durable_tail == tail`, + and the watch channel publishes the same tail. +4. **Closed-ness durability:** a stream whose close was acked recovers `closed == true` + (position may lawfully shrink only under power-loss faults, and never below the + checkpointed floor). +5. **No resurrection:** a deleted stream does not reappear after recovery. +6. **Read correctness:** GETs (catch-up path) return exactly the oracle's bytes for the + requested range. +7. **Recovery never panics** and never errors on in-model faults. + +On violation: print the seed, generation, step trace tail, and the diff — everything +needed to replay deterministically. + +## Placement & running + +- New module `src/wal/sim_tests.rs` (`#[cfg(test)]`, registered in `wal/mod.rs`), reusing + the `Harness` boot/crash idiom from `e2e_tests.rs` and `DurabilityGuard::wal()`. +- CI-friendly default: a small fixed seed set (fast, deterministic). +- Long-run mode via env: `DS_SIM_SEEDS=` and `DS_SIM_STEPS=` scale the exploration; + a wrapper invocation runs thousands of seeds locally to hunt for issues. + +## Out of scope (v1) + +Tiering/offload faults (S3), `.meta` corruption (byzantine tier), memory-mode recovery, +multi-process concurrency, and the HTTP socket layer itself. Each can be layered on later. diff --git a/packages/durable-streams-rust/.gitignore b/packages/durable-streams-rust/.gitignore index 9f814832e9..21cd8dd587 100644 --- a/packages/durable-streams-rust/.gitignore +++ b/packages/durable-streams-rust/.gitignore @@ -8,3 +8,5 @@ # the cross-server suite may still drop these here when run against this checkout: /bench/ benchmark-results.json +# deploy/replicated/local-cluster.sh state (data dirs, logs, pids) +/.local-cluster/ diff --git a/packages/durable-streams-rust/ARCHITECTURE.md b/packages/durable-streams-rust/ARCHITECTURE.md index 409023dd9a..174f8c70f5 100644 --- a/packages/durable-streams-rust/ARCHITECTURE.md +++ b/packages/durable-streams-rust/ARCHITECTURE.md @@ -35,10 +35,10 @@ flowchart LR W2 --> W3["encode_wire
(JSON flatten, append delimiter)"] W3 --> W4[["per-stream appender mutex"]] W4 --> W5["write_all → data file
(lands in page cache)"] - W5 --> W6["update tail + resident cache"] - W6 --> W8["publish tail
(watch channel)"] - W8 --> W7["group-commit fsync
(WAL shard committer)"] - W7 --> W9["204 / 200 — only after durable"] + W5 --> W6["advance writer tail
+ stage into WAL shard"] + W6 --> W7["group-commit fsync
(WAL shard committer)"] + W7 --> W8["publish durable tail + resident cache
(watch channel)"] + W8 --> W9["204 / 200 — only after durable"] end subgraph READ["READ · GET"] @@ -72,10 +72,10 @@ The dotted edges are the only coupling between writers and readers: publishing t 1. **Parse idempotency headers** — `Producer-Id` / `Producer-Epoch` / `Stream-Seq`. Duplicate `(producer, epoch, seq)` is acknowledged without re-appending (exactly-once producers). 2. **`encode_wire`** — turn the request body into the contiguous wire representation. In JSON mode this flattens arrays and appends the `,` delimiter so the on-disk bytes are already a valid stream fragment. 3. **Acquire the per-stream appender mutex** (`AsyncMutex`). This is the _only_ serialization point, and it's per-stream — different streams never contend. -4. **`write_wire`** — `write_all` the bytes to the data file (they land in the OS page cache immediately), advance the tail under a short `RwLock` write, update the **resident tail cache**, then **publish the new tail** on the `watch` channel. Note the order: the cache is populated _before_ the wake, so a woken subscriber reliably hits it. -5. **Durability** — in `wal` mode the append is staged into the stream's assigned WAL shard, and the response returns only after the shard's group-commit committer `fdatasync`s the segment covering this record. See [Durability](#durability) below. In `wal` mode everything above and the entire read path are identical regardless of workload. In `memory` mode binary appends take a separate socket→file splice path (zero-copy `splice(2)`, no WAL, no `fdatasync`) that bypasses `handle_append` / `encode_wire` via the engine zero-copy intercept and acks immediately after the page-cache write. +4. **`write_wire`** — `write_all` the bytes to the data file (they land in the OS page cache immediately) and advance the _writer_ tail (`Shared.tail`) under a short `RwLock` write. The reader-observable tail does **not** move yet. +5. **Durability, then visibility** — in `wal` mode the append is staged into the stream's assigned WAL shard (under the appender mutex, so per-stream LSN order matches byte order) and the handler awaits the shard's group-commit `fdatasync`. Only after the record is durable does `publish_durable_tail` advance the **reader-observable `durable_tail`** (monotonically), populate the **resident tail cache**, and **publish on the `watch` channel** — cache before wake, so a woken subscriber reliably hits it. Then the 2xx is returned. See [Durability](#durability) below. In `memory` mode there is no WAL: the page-cache write is the ack, and binary appends take a separate socket→file splice path (zero-copy `splice(2)`) that bypasses `handle_append` / `encode_wire` via the engine zero-copy intercept. -Visibility vs durability are deliberately decoupled: the bytes are in the page cache (and the tail is published) before durability resolves, so a live reader sees data with minimal latency, while the _appender_ doesn't get its 2xx until the data is durable. +Visibility is gated on durability (PROTOCOL.md §4.1): a live reader never observes bytes (or an EOF) that a crash could roll back. The writer tail runs ahead in `Shared.tail`; readers see `durable_tail`, which follows it as group commits resolve. ## Durability @@ -105,7 +105,7 @@ Every append is written to the per-stream data file (page cache, no hot fsync Per-stream files are `fdatasync`'d off the ack path at a periodic **checkpoint**, after which the bounded WAL is recycled. On boot, recovery replays the WAL from its oldest retained segment, reconciles each stream's durable tail (torn-tail repair via truncation + `fdatasync`), then resets the WAL for fresh appends. -The invariant: **visibility is never gated on durability**. Bytes land in the page cache and the tail is published before the WAL `fdatasync`, so live readers see an append at memory latency while the appender's acknowledgement waits for the WAL commit. +The invariant: **readers only ever observe durable bytes** (PROTOCOL.md §4.1). Bytes land in the page cache immediately, but the reader-observable `durable_tail` (and the `watch` wake) advances only after the WAL `fdatasync` covering the record — the same barrier that releases the appender's acknowledgement. A crash therefore never rolls back anything a reader has seen. ### `memory` mode @@ -140,8 +140,8 @@ flowchart TB end A3 ==> PC[("OS page cache
— the hot tier")] - A3 ==> RC["resident tail cache
(last chunk, configurable via --tail-cache-bytes,
in heap)"] - A3 -.-> TW[/"tail watch channel
(one notify per append — before fsync)"/] + A4 ==> RC["resident tail cache
(last chunk, configurable via --tail-cache-bytes,
in heap)"] + A4 -.-> TW[/"tail watch channel
(one notify per append — after the group commit)"/] subgraph FAN["② Fan-out"] direction TB @@ -159,7 +159,7 @@ The techniques, each with its mechanism and payoff: 1. **Contiguous wire-byte storage.** The file _is_ the response. Reads are byte ranges with no reframing and no per-message copy — and this is what makes `sendfile` zero-copy possible at all. 2. **Group-commit coalesced fsync.** The durability contract ("return after fsync") is the expensive part of an append. Concurrent appenders share a single in-flight barrier fsync, so throughput scales with the _batch size_ per fsync rather than one fsync per message. This is why unbatched appends hit ~30k/s where a per-append-fsync server (Node) does ~130/s. 3. **Per-stream single writer, lock-free reads.** One async mutex orders a stream's appends; there is no global lock (streams live in a `DashMap`). Reads take a brief tail snapshot and do positioned reads — they never block the writer and never wait on each other. -4. **Pre-fsync visibility.** Appended bytes are in the page cache and the tail is published before the fsync resolves, so live readers see data at memory latency; only the appender's acknowledgement waits for durability. +4. **Durable-gated visibility, group-commit-amortized.** The reader-observable tail is published only after the record's group-commit fsync (readers never see bytes a crash could roll back — PROTOCOL.md §4.1). Because N concurrent appends share one barrier fsync, the visibility latency cost is one amortized group commit, not one fsync per append. 5. **`watch`-channel wakeups.** Live readers park on a per-stream `watch`; an append is one `send_replace` that wakes all of them. No polling loop, no timer churn. 6. **Resident tail cache (fan-out de-duplication).** Without it, N caught-up SSE/long-poll subscribers each re-read (and re-encode) the _same_ just-appended bytes — N× duplicated work that grows with audience size. The cache keeps the last chunk in the heap so all N share one read (and SSE encodes once per subscriber off that shared buffer). For small hot reads it's also fewer syscalls than `sendfile` and skips the read-offload pool hop. 7. **Zero-copy egress.** `FileRange` reads are served with `sendfile` (page cache → socket, no userspace copy → ~5× less CPU per byte than a buffered copy). The **`--read-offload`** strategy keeps a cold backfill's disk fault off the async workers so one slow read can't stall unrelated requests. diff --git a/packages/durable-streams-rust/CARDINALITY_1M.md b/packages/durable-streams-rust/CARDINALITY_1M.md new file mode 100644 index 0000000000..1c1d5012fd --- /dev/null +++ b/packages/durable-streams-rust/CARDINALITY_1M.md @@ -0,0 +1,54 @@ +# 1M-stream cardinality fixes — findings + results (2026-07-02) + +Follow-up to `WRITE_BOTTLENECKS_1M.md` (bottleneck #2: stream cardinality) and +`CONTENTION_INVESTIGATION.md`. Server commit: `662b0c845` on +`perf/combined-t1a-t1c-t2a`. **Outcome: 1M streams reaches 1,114,644 ops/s on a +16 vCPU `c4d-standard-16-lssd` (ladder unsaturated), and the 500k→1M degradation at +equal load is −17% (was a cliff).** + +## Root causes (evidence-first, local Linux repro at 20k→400k streams) + +The key mechanism: **at high cardinality, ops/stream/checkpoint-interval drops below +1, so every "amortized once-per-stream-per-interval" cost becomes a per-op cost.** + +| # | cost | evidence | fix | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Checkpoint drain did O(touched) capture (`shared.read()` + Arc clone per stream) **while holding the shard `dirty` mutex** — the same mutex every append's epoch-transition takes; at 400k streams every append is a transition | `WAL_CKPT drain_us` 25–140 ms/tick; `dirty_wait_load` 0.01→0.30 cores as streams 20k→400k; p99 ≈ max drain | O(1) critical section: take Vec + bump epoch under the lock, capture after release | +| 2 | Checkpoint capture / cumulative tails-file re-read+re-sort+rewrite / recycle ran **on async runtime threads**, serially across shards | `WAL_CKPT` capture 31 ms + tails 24 ms per tick per shard on runtime threads | whole checkpoint body in one `spawn_blocking`; tails map memory-resident; shards checkpoint concurrently (`JoinSet`) | +| 3 | **Per-append meta sidecar flush**: with inter-append gap > the 100 ms debounce (always, at high cardinality) every producer append did JSON + `File::create(.meta.tmp)` + `rename` → all workers spin on the **data-dir inode rwsem** | perf: `osq_lock`+`rwsem_spin_on_owner` under `write_meta_sync` = **38–46% of ALL server CPU at every cardinality** | WAL-staged appends only mark `meta_dirty`; checkpoint writes sidecars for drained streams after recycle (memory-mode keeps the debounced flush). Producer/access staleness bound: 100 ms debounce → checkpoint cadence (contract already allows lag) | +| 4 | Two registry lookups per append (`handle_append` metric label + `_inner`) — 2× SipHash + cold DashMap walk at 1M keys | code inspection | `_inner` returns `is_json` | + +## Local A/B (contention-repro-linux.sh, 6 srv cores, conn=256, shards=6) + +| streams | before | after | p99 | +| ------- | ------------- | ---------- | ------------ | +| 20k | ~43–46k ops/s | **80.4k** | 41 → 7.7 ms | +| 200k | 32.3k | **50.6k** | 53 → 17.9 ms | +| 400k | 16.0k | **36–44k** | 144 → ~28 ms | + +Correctness: 95 crate tests + 326 conformance tests pass. New telemetry: `WAL_CKPT` +per-shard checkpoint phase line (`--wal-stats`), and the repro script grew +`--tmpfs` / `--wal-stats` knobs + WAL_CKPT summarizing. + +## Remote validation (GKE, 16 vCPU, pool client, 256 B, batch 1) + +Suite `ds-bench/suites/run-durable-cpu16-1m-card.json`, image +`durable-streams:combined-card@sha256:d74840bd…`; full detail + caveats in +`ds-bench/results/run-durable-cpu16-1m-card/FINDINGS.md`. + +| streams | pods | ops/s | p50 / p99 / max | +| ------- | ---- | ------------- | ------------------------------------------------------------ | +| 1M | 32 | 898,582 | 3.5 / 30.5 / **149.6 ms** (baseline 862k, 3.3 / 32 / 405 ms) | +| 1M | 64 | **1,114,644** | 3.4 / 60.3 / 211 ms — still climbing (+21%/+16 pods) | +| 500k | 48 | 1,110,268 | 3.1 / 42.7 / 145 ms | + +## Open follow-ups + +1. True 16 vCPU ceiling at 1M (ladder past 64 pods) and a 32 vCPU 1M run. +2. **Per-shard producer-state journal**: sidecar writes/s ≈ ops/s at full cardinality + (off the hot path now, but it stretches checkpoint cadence — locally ~1.6 s/shard + meta phase at 400k — and bounds producer-state staleness). One cumulative + per-shard file per tick, recovery overlays producers by max(epoch, seq). +3. Read+write mix at 1M streams (everything here is write-only). +4. `--wal-stats` cell on NVMe (`run-durable-cpu16-1m-card-stats.json`, never ran) to + confirm checkpoint fsync/meta phase behavior at 1M on real disks. diff --git a/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md b/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md new file mode 100644 index 0000000000..5342feedcf --- /dev/null +++ b/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md @@ -0,0 +1,106 @@ +# WAL write-saturation contention investigation + +Tracking doc for the investigation into the write-throughput ceiling reported in +`ds-bench/results/run-durable-pool2/FINDINGS.md` (server plateaus at ~80% CPU and +then _declines_ under more load — a ceiling set by the commit path, not compute). + +## Hypothesis + +Sharding, group-commit, and the network reactor are **not isolated**: they are +multiplexed over one shared work-stealing Tokio runtime, and each WAL shard is +guarded by cross-thread blocking `std::sync::Mutex`es. So a "shard" is a _lock_, +not a _core_. At saturation the cost is lock contention + committer scheduling + +a durability-wakeup thundering herd, not CPU. + +Per-append contended state (all on the shard a stream hashes to): + +- `shard.dirty` Mutex + HashMap insert — **every append** (`register_dirty`). +- `shard.inner` Mutex — **twice** per append (reserve + mark_written). +- `durable_tx` watch — `publish_durable` wakes **every** parked waiter on the shard. + +## Phase 0 — telemetry (DONE) + +Added always-on, dependency-free contention telemetry (independent of the heavy +`telemetry`/OTLP feature): + +- `ShardStats` gained per-shard counters: `inner`/`dirty` lock-wait nanos + + acquire counts, records `staged`, and durability `waiters_woken` + (`src/wal/telemetry.rs`). +- Instrumented the hot path (`src/wal/shard.rs`): `register_dirty`, + `reserve_and_stage` (both `inner` acquisitions), `publish_durable`. +- Runtime gate `--wal-stats `: arms the hot-path timing (one relaxed + atomic load when off — no clock reads in a default run) and spawns a stderr + emitter printing per-interval `WAL_CONT` lines: + + ``` + WAL_CONT staged/s=… fsync/s=… batch_avg=… inner_wait_us=… inner_wait_load=… \ + dirty_wait_us=… dirty_wait_load=… waiters_woken_avg=… + ``` + + `*_wait_load` = fraction of a core-second spent purely _waiting_ on that lock + (>1.0 ⇒ more than a whole core lost to parking on it). + +## Phase 0 — local reproduction (DONE / caveated) + +`scripts/contention-repro.sh` drives the server with the `ds-bench multi-stream` +pool client and prints throughput + CPU + steady-state `WAL_CONT`. + +**macOS caveats (why a Linux harness is also needed):** + +- `F_FULLFSYNC` is a true drive barrier (~tens of ms) and dominates the commit + path, masking the lock. Added a **bench-only** `DS_BENCH_FAST_FSYNC` env + (`src/store.rs`, `src/wal/segment.rs`) that uses plain `fsync` on macOS so a + RAM-disk data dir gives cheap fsync (the Linux+NVMe regime). NOT durable; never + set in production. +- The 10-core dev box co-locates client + server, so the _absolute_ throughput + ceiling is confounded (a flat ~1600 ops/s independent of shards/connections). + The **contention telemetry signals are valid** on macOS (use them for relative + before/after of a change); the **throughput-ceiling** comparison must run on + Linux with a tmpfs data dir and CPU isolation (`contention-repro-linux.sh`). + +Use a RAM disk for cheap fsync on macOS: + +``` +DEV=$(hdiutil attach -nomount ram://6291456 | awk '{print $1}') +diskutil erasevolume HFS+ dsram "$DEV" # → /Volumes/dsram +TMPDIR=/Volumes/dsram scripts/contention-repro.sh --shards 1 --connections 256 +``` + +## Baseline (Linux harness, 6 server cores / 4 client cores, tmpfs, conn=256) + +Reproduces the findings' signature — a hard throughput ceiling at **~80% CPU** +(480–500 of 600), barely helped by more shards, with CPU left on the table: + +| shards | ops/s | cpu% | fsync/s | batch | inner_wait_load | waiters_woken | +| ------ | ------ | ---- | ------- | ----- | --------------- | ------------- | +| 1 | 45,543 | 482 | 6,113 | 7.5 | 0.02 | 25.8 | +| 2 | 46,905 | 504 | 12,384 | 3.8 | 0.01 | 12.4 | +| 6 | 48,825 | 495 | 24,177 | 2.0 | 0.01 | 5.2 | + +Read: fsync is cheap (tmpfs), the inner lock is not yet the gate at 6 cores +(`inner_wait_load`≈0.02), so the ceiling here is the **commit + durability-wakeup +coordination machinery** burning CPU/scheduling (25.8 waiters woken per commit at +1 shard). The lock itself becomes the gate at the findings' 32-core scale; both +are targeted below. (Numbers are this dev box; use deltas, not absolutes.) + +## How to judge a candidate change + +A change is good if it **lifts the Linux throughput ceiling** AND drives the +contention metric it targets toward zero: + +- lock-free `register_dirty` → `dirty_wait_load` → ~0 +- atomic reserve → `inner_wait_load` drops +- coalesced wakeups → `waiters_woken_avg` → ~1 +- dedicated committer / io_uring → higher `fsync/s` without CPU saturation + +## Candidate architectures (Phase 1, parallel worktrees) + +- **T1a** lock-free `register_dirty` (atomic dirty bit + lock-free push on 0→1). +- **T1b** atomic reserve (packed `fetch_add` for lsn+write_pos; lock only on roll). +- **T1c** coalesced durability wakeups (wake only satisfied waiters, not broadcast). +- **T2a** dedicated committer thread(s) off the shared runtime / drop per-commit + `spawn_blocking`. +- **T2b** io_uring WAL writes + fsync (Linux). +- **T3** shared-nothing thread-per-core spike (shard→core, per-core epoll via + `SO_REUSEPORT`, no cross-core lock; SPSC handoff for the pool client's + all-shards-per-connection access pattern). diff --git a/packages/durable-streams-rust/CRASH_SIM_FINDINGS.md b/packages/durable-streams-rust/CRASH_SIM_FINDINGS.md new file mode 100644 index 0000000000..b48383cba4 --- /dev/null +++ b/packages/durable-streams-rust/CRASH_SIM_FINDINGS.md @@ -0,0 +1,142 @@ +# Crash/recovery simulation — findings (2026-07-02) + +A seeded randomized crash/fault simulation for the WAL recovery path now lives at +`src/wal/sim_tests.rs` (design: `docs/superpowers/specs/2026-07-02-wal-crash-simulation-design.md`). +Each seed drives the real handler path with a random workload (creates, appends, +client-disconnect cancellations, closes, forks, deletes, checkpoints), crashes by dropping +the generation's tokio runtime + stopping committers, injects power-loss disk faults +constrained to the documented fault model, reboots through the real recovery sequence, and +checks a no-loss/no-torn/consistency oracle across multiple generations per seed. + +Run it: + +```sh +# CI-fast deterministic smoke (4 seeds) +cargo test crash_recovery_randomized_simulation + +# long hunt +DS_SIM_SEEDS=1000 DS_SIM_SEED0=20000 DS_SIM_GENS=4 DS_SIM_STEPS=150 \ + cargo test crash_recovery_randomized_simulation -- --nocapture + +# reproduce a failure (the panic message prints the seed) +DS_SIM_SEEDS=1 DS_SIM_SEED0= cargo test crash_recovery_randomized_simulation +# forensics: DS_SIM_SNAPSHOT=1 snapshots each generation's pre-boot disk state; +# `DS_DUMP_DIR= cargo test wal_forensic_dump -- --ignored --nocapture` decodes it. +``` + +## Finding 1 (CRITICAL, fixed): boot clobbered sealed/recycled WAL segments → acked-data loss + +Found by the very first seed (89837), gen 1. `Shard::open` unconditionally ran +`FileSegment::create(1.wal, segment_size)`, which re-preallocates to full segment size. +Two on-disk states the segment roll/recycle feature (11a) introduced made that destructive: + +- **Sealed `1.wal`** — sealing truncates a rolled segment to its exactly-packed length, + and that exact length is what lets `replay_from_checkpoint` walk across the segment seam + (`off == raw.len()` → next segment). Re-preallocating grafted a `fallocate`/`set_len` + zero tail onto the sealed segment; replay decoded the zeros as `Incomplete` — the clean + end of the durable log — and **silently dropped every acked record in all later + segments**. Worse than a skip: `reconcile_tail` then **truncated the per-stream files** + back to the stale frontier, destroying acked bytes that were sitting intact in the page + cache. Pure process crash (no power loss) suffices to trigger it. +- **Recycled `1.wal`** — after a checkpoint recycles the first segment(s), boot re-created + a fresh all-zero `1.wal`. The walk visits segments in start-lsn order, decoded zero + records in the spurious `1.wal`, and stopped — **replaying nothing**: every acked append + since the last checkpoint was truncated away. + +The trigger threshold in production is one segment roll (128 MiB of WAL traffic, or any +`--wal-segment-bytes` override) between two boots — no fault injection required. + +**Fix:** boot now discovers existing `*.wal` files and opens the newest one as a +non-destructive placeholder handle (`FileSegment::open_existing`); only a genuinely fresh +shard dir creates + preallocates `1.wal`. The in-memory cursor stays a placeholder until +`reset_after_recovery` rebuilds it, per the documented boot order. +Regression tests: `e2e_multi_segment_acked_records_after_first_seal_survive_crash`, +`e2e_recycled_first_segment_acked_records_survive_crash` (both fail on the pre-fix code). + +## Finding 2 (fixed): recovery debug assert panicked on a healthy multi-boot recovery + +Seed 89840, gen 2+. `recovery.rs` asserted that a stream rebuilt purely from replayed +records (no checkpoint-persisted tail) must have its first record at `file_base`. That +premise is false across the boot cycle: after `recover` + `reset_after_recovery`, a +stream's post-boot appends produce WAL records starting at its _recovered tail_ — with no +persisted-tails entry until the next checkpoint. The durable prefix below the first +replayed record came from the previous boot's recovery reconcile (which `fdatasync`s the +repaired file), a third proof source the assert didn't model. Debug builds panicked the +recovery thread (release builds were unaffected). + +**Fix:** the assert now checks the actual invariant — no _hole_ between the file's +pre-replay logical end and the first replayed record. + +## Finding 3 (docs, fixed): ARCHITECTURE.md described pre-fsync visibility; the code gates visibility on durability + +`ARCHITECTURE.md` claimed "visibility is never gated on durability" (tail published before +the WAL fsync). The implementation deliberately does the opposite: `write_wire` advances +only the writer tail; `publish_durable_tail` advances the reader-observable `durable_tail`, +populates the tail cache, and fires the `watch` **after** `wait_durable_lsn` — readers +never observe bytes a crash could roll back (PROTOCOL.md §4.1). The doc's write-path +diagram, §Durability invariant, and fan-out technique list were updated to match the code. + +## Observation (by design, worth knowing): cancelled appends leave a lagging reader tail + +A client that disconnects mid-append can leave the handler task cancelled at +`wait_durable_lsn`: the bytes are in the data file (and staged in the WAL, becoming durable +via the group commit), `Shared.tail` is advanced, but `publish_durable_tail` never runs for +that record. The reader-observable tail lags the file until the next successful append +publishes a higher frontier (monotonic heal) or a crash-recovery exposes the frontier. +Contract-consistent (the append never acked), but operators reading `Stream-Next-Offset` +may see it hold still despite durable bytes existing past it on a quiet stream. + +## Finding 4 (fixed): WAL-quiet streams had no torn-tail proof after power loss + +Initially documented as a theoretical residual gap, then **confirmed by the 1000-seed +hunt** (seed 20230, gen 1): a stream created after the last checkpoint whose only append +was in-flight at the crash. Power loss tore the append's WAL record (never fdatasync'd — +nothing acked) while the data file kept a torn prefix of its bytes. Recovery had **no +truncation proof** — zero surviving WAL records, no checkpoint `tails` entry — so the +sidecar pass trusted `tail = file_base + file size` and exposed the torn fragment (615 +bytes of a partial record) to readers: exactly the C1 torn-tail shape the WAL exists to +prevent, on the one class of stream the existing proof chain missed. No acked data was at +risk, but torn/garbage bytes (including torn JSON) became reader-visible. + +**Fix:** the `.meta` sidecar now persists a `durable_tail` proof, captured from +`Shared.durable_tail` (which only ever advances post-fsync, so any captured value is +honest). It rides along on writes that already happen — fsynced at create/close, +refreshed by the checkpoint's meta flush and by recovery's reconcile — **zero new +hot-path fsyncs**. Recovery seeds every sidecar-recovered stream's frontier with +`max(meta proof, checkpoint tails, replayed ends)` and truncates anything past it; a +fast path skips streams whose file already sits exactly at their frontier, so 1M-stream +boots stay free of extra I/O. Recovery durably re-persists the proof before +`reset_after_recovery` wipes the WAL + tails file (else the next boot's seed would +regress below a durable frontier it can no longer re-replay). Old sidecars without the +field keep the previous trust-the-file-size behavior (`serde` default), converting to +proven on their next natural meta write. +Regression test: `e2e_wal_quiet_stream_torn_unacked_tail_truncated`. + +## Finding 5 (fixed): an acked DELETE was not durable — crash resurrected the stream + +Found by the second 1000-seed hunt (seed 20387, gen 2), with **no fault injection at +all** — a clean process crash right after an acked DELETE. `handle_delete` returned 204 +while the data-file + sidecar unlinks ran on a **detached** `spawn_blocking` task (and +with no parent-directory fsync even once they ran). A crash after the ack left both +files on disk, and the next boot's sidecar pass resurrected the stream **with all its +data** — after the client was told it was permanently gone (a correctness and +data-retention/compliance issue). The soft-delete meta flag had the same fire-and-forget +shape. + +**Fix:** DELETE now awaits `delete_or_soft_delete_durable` before the 204 — synchronous +unlinks + one parent-dir fsync (both artifacts live in the same directory), or the +durable soft-delete meta write; a failure acks 500, not 204. The expiry sweep on the hot +read path keeps the detached non-durable variant (an expired stream that resurrects +simply re-expires on next access — harmless, and documented on the method). +Regression test: `e2e_acked_delete_is_durable_no_resurrection_after_crash`. + +## Simulation results + +- Pre-fix: seed 89837 (the first default seed) hit Finding 1 in generation 1; seed 20230 + hit Finding 4 in generation 1 of the first 1000-seed hunt; seed 20387 hit Finding 5 in + generation 2 of the second hunt (clean crash, no faults). +- Post-fix: 4 default smoke seeds + 50-seed and 1000-seed hunts (4 generations × 150 steps + each, 1–3 shards, 32 KiB segments to force rolls/recycles) pass with faults enabled + (data-file truncate/scribble/zero within the un-fsynced region; WAL suffix zero/scribble + above the durable LSN; torn staged appends; client-disconnect cancellations; + in-flight-at-crash appends). diff --git a/packages/durable-streams-rust/Cargo.lock b/packages/durable-streams-rust/Cargo.lock index f448af9c2d..60475917b5 100644 --- a/packages/durable-streams-rust/Cargo.lock +++ b/packages/durable-streams-rust/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -20,12 +31,77 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyerror" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71add24cc141a1e8326f249b74c41cfd217aeb2a67c9c6cf9134d175469afd49" +dependencies = [ + "serde", +] + [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.89" @@ -34,7 +110,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -55,12 +131,33 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -70,12 +167,70 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byte-unit" +version = "5.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +dependencies = [ + "rust_decimal", + "schemars", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytes" version = "1.12.0" @@ -111,11 +266,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "core-foundation" version = "0.10.1" @@ -171,6 +374,27 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -189,19 +413,21 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "durable-streams" version = "0.1.0" dependencies = [ + "bincode", "bytes", "crc32c", "dashmap", "httparse", "libc", "object_store", + "openraft", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", @@ -215,6 +441,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.16.0" @@ -258,6 +490,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.32" @@ -314,7 +552,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -402,6 +640,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -681,6 +928,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.13.0" @@ -755,6 +1008,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -845,6 +1104,48 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openraft" +version = "0.9.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d35e2f60cdf9bcfc39a020966091017c6dc2a4b43b355a22ca3e76106f4a0a" +dependencies = [ + "anyerror", + "byte-unit", + "chrono", + "clap", + "derive_more", + "futures", + "maplit", + "openraft-macros", + "rand 0.8.6", + "serde", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-futures", + "validit", +] + +[[package]] +name = "openraft-macros" +version = "0.9.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbf0342d747a8da209c8e1d3ca8f788100966669412aaacb449409205931251" +dependencies = [ + "chrono", + "proc-macro2", + "quote", + "semver", + "syn 2.0.118", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -861,7 +1162,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror", + "thiserror 2.0.18", "tracing", ] @@ -891,7 +1192,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest 0.13.4", - "thiserror", + "thiserror 2.0.18", "tokio", "tonic", "tonic-types", @@ -929,7 +1230,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.4", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -978,7 +1279,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1011,6 +1312,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1040,7 +1350,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1052,6 +1362,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quick-xml" version = "0.37.5" @@ -1076,7 +1406,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -1097,7 +1427,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -1132,6 +1462,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.6" @@ -1200,6 +1536,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1217,6 +1573,15 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -1304,6 +1669,52 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.6", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1396,12 +1807,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "3.7.0" @@ -1458,7 +1887,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1511,6 +1940,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -1541,7 +1976,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1560,12 +1995,29 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.118" @@ -1594,7 +2046,22 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", ] [[package]] @@ -1603,7 +2070,18 @@ version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -1614,7 +2092,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1676,7 +2154,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1713,6 +2191,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.6" @@ -1829,7 +2337,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1842,6 +2350,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -1905,6 +2423,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -1923,12 +2447,43 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "validit" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efba0434d5a0a62d4f22070b44ce055dc18cb64d4fa98276aa523dadfaba0e7" +dependencies = [ + "anyerror", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2017,7 +2572,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -2093,7 +2648,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2104,7 +2659,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2287,6 +2842,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -2299,6 +2863,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yoke" version = "0.8.3" @@ -2318,7 +2891,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2339,7 +2912,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2359,7 +2932,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2399,7 +2972,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/packages/durable-streams-rust/Cargo.toml b/packages/durable-streams-rust/Cargo.toml index 654dbd6298..ae95c0181f 100644 --- a/packages/durable-streams-rust/Cargo.toml +++ b/packages/durable-streams-rust/Cargo.toml @@ -24,6 +24,15 @@ dashmap = "6" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["raw_value"] } httparse = "1" + +# Consensus for `--durability replicated` (REPLICATION.md): openraft (Raft), +# actively maintained and production-proven (Databend, CnosDB). Replaced +# OmniPaxos after our deterministic simulation caught an unpublished-fix +# consensus safety bug in the last released version and the project proved +# unmaintained (see REPLICATION.md "History: why openraft"). +openraft = { version = "0.9.21", features = ["serde", "storage-v2"] } +# Peer-message framing for the replication mesh. +bincode = "1" libc = "0.2" percent-encoding = "2" diff --git a/packages/durable-streams-rust/README.md b/packages/durable-streams-rust/README.md index 1d20d08c09..9db4f2e7d2 100644 --- a/packages/durable-streams-rust/README.md +++ b/packages/durable-streams-rust/README.md @@ -72,11 +72,12 @@ durable, single-node server on `127.0.0.1:4437` with its data dir under `$TMPDIR **Durability** — controls how appends are made durable. See [ARCHITECTURE.md › Durability modes](ARCHITECTURE.md#durability-modes). -| Flag | Default | Description | -| --------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--durability` | `wal` | `wal` (default) — durable group-commit `fdatasync`; an append acks only after its record is in the sharded WAL. `memory` (Linux-only) — no WAL; binary appends use zero-copy `socket→file` splice, ack on the page-cache write; **NOT locally crash-durable** — durability is delegated to (future) replication. Exits 2 on non-Linux. | -| `--wal-shards` | CPU cores | (`wal` mode) shard / group-commit-committer count; persisted on first run, a later run must match it | -| `--wal-segment-bytes` | `128 MiB` | (`wal` mode) per-shard WAL segment size; lower it only to force segment rolls in tests/benches | +| Flag | Default | Description | +| --------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--durability` | `wal` | `wal` (default) — durable group-commit `fdatasync`; an append acks only after its record is in the sharded WAL. `memory` (Linux-only) — no WAL; binary appends use zero-copy `socket→file` splice, ack on the page-cache write; **NOT locally crash-durable** — durability is delegated to (future) replication. Exits 2 on non-Linux. `replicated` — no WAL, no fsync; an append acks once a quorum of replicas has committed it via Raft (openraft): see [REPLICATION.md](REPLICATION.md). | +| `--wal-shards` | CPU cores | (`wal` mode) shard / group-commit-committer count; persisted on first run, a later run must match it | +| `--wal-segment-bytes` | `128 MiB` | (`wal` mode) per-shard WAL segment size; lower it only to force segment rolls in tests/benches | +| `--repl-*` | — | (`replicated` mode) `--repl-id`, `--repl-peers 1@h:p,2@h:p,…`, `--repl-listen`, `--repl-ack-timeout-ms`, `--repl-snapshot-logs`, `--repl-join` — cluster membership and tuning; see [REPLICATION.md › Configuration](REPLICATION.md#configuration) | **Read path** — performance knobs; none change protocol behaviour. Leave at defaults unless a benchmark says otherwise. diff --git a/packages/durable-streams-rust/REPLICATION.md b/packages/durable-streams-rust/REPLICATION.md new file mode 100644 index 0000000000..436ee862f4 --- /dev/null +++ b/packages/durable-streams-rust/REPLICATION.md @@ -0,0 +1,356 @@ +# Replicated durability (`--durability replicated`) + +Kafka-style durability for the durable-streams server: instead of making an +append durable with a local WAL `fdatasync`, the append is **replicated to a +quorum of servers** and acked once the replication log has _decided_ it. The +fsync leaves the hot path entirely — durability comes from independent failure +domains, exactly like a Kafka produce with `acks=all` against an in-memory +(page-cache) log. + +The consensus layer is **[openraft]** (MIT/Apache-2.0) — an actively +maintained, production-proven Raft (Databend, CnosDB, RobustMQ) with +snapshots, joint-consensus membership change, and a bring-your-own network + +storage design that lets us keep our own TCP mesh and in-memory log. It +replaced OmniPaxos — [see the history](#history-why-openraft). + +[openraft]: https://github.com/databendlabs/openraft + +- [The model](#the-model) +- [Design: log-first apply](#design-log-first-apply) +- [What is replicated](#what-is-replicated) +- [Guarantees and trade-offs](#guarantees-and-trade-offs) +- [Configuration](#configuration) +- [Deploying a cluster](#deploying-a-cluster) +- [Operations](#operations) +- [Roadmap](#roadmap) + +## The model + +The existing durability modes are two points on a spectrum: + +| mode | ack means | survives | +| ---------------- | ------------------------------------------------------ | ------------------------------------ | +| `wal` | record fdatasync'd in the sharded WAL | single-node crash & power loss | +| `memory` | record in the page cache | nothing (process restart at best) | +| **`replicated`** | **record decided by a quorum of replicas (in memory)** | **loss of any minority of replicas** | + +`replicated` mode is `memory` mode's write path plus a consensus round: +no fsync anywhere on the append path. A 3-node cluster tolerates the total +loss of any 1 node with zero data loss; what it does _not_ survive is the +simultaneous loss of a majority (e.g. a coordinated power failure of 2/3 +nodes) — the same trade Kafka makes with `acks=all` + +`log.flush.interval.messages=MAX` (flush left to the OS). + +## Design: log-first apply + +The single most important design decision: **in replicated mode, every +state-mutating operation is applied to the store only from the decided +consensus log — on every node, including the leader.** + +``` +client ── POST /s ──▶ handler (any node) + │ parse + validate (best-effort pre-checks) + │ encode_wire + ▼ + propose LogOp::Append{path, wire, producer, close} + │ (raft client_write; on a follower the op is + │ forwarded to the leader over the RPC mesh) + ▼ + ...committed by quorum... + ▼ + state machine apply (every node, log order, sharded by stream) + │ authoritative checks (closed / producer dedup / seq) + │ appender lock → write_wire → publish_durable_tail + ▼ + client_write resolves with the APPLY OUTCOME → HTTP response + (forwarded acks additionally wait for the origin node's own apply + to cover the entry — read-your-writes on the node you wrote to) +``` + +Contrast with the WAL path, where the handler writes the stream file first and +then waits for the fsync. Writing first doesn't work under replication: a +leader that writes locally, proposes, and then loses leadership before the +proposal decides is left with bytes in its file that the rest of the cluster +never saw — its offsets have diverged forever. With log-first apply the store +on every node is a deterministic function of the decided log prefix, so +replicas can never diverge, and leader fail-over needs no truncation/repair +protocol. + +Consequences: + +- **Any node accepts writes.** A non-leader node forwards the op to the + leader over the RPC mesh and relays the apply outcome; before responding it + waits until its OWN state machine has applied the entry, so the client + keeps read-your-writes on the node it wrote to. No client-side leader + routing, no 307 redirects. Writing to the leader saves one network hop. +- **Every node serves reads.** The applier calls the same + `publish_durable_tail` as the other modes, so catch-up reads, long-poll and + SSE fan-out work identically on followers (Kafka "follower fetch", for + free). Follower reads are sequentially consistent, not linearizable: a + follower may briefly lag the leader's decided frontier. +- **Authoritative checks run at apply time.** Producer dedup / epoch fencing / + `Stream-Seq` / closed-stream conflicts are evaluated inside the applier, in + log order — deterministic on every replica. The handler runs the same checks + best-effort _before_ proposing only to fail fast with 4xx without burning a + consensus round. +- **The ack is an apply outcome, not just "decided".** The proposing node + resolves the client's response with the outcome the applier computed + (offset, duplicate, conflict, …), so responses are identical to the + single-node modes. +- **Applies are sharded.** Within each state-machine apply batch, entries + are sharded by stream path (per-stream log order preserved; fork-creates + barrier and apply inline). Benchmarks showed a single sequential applier + lets one slow store write (an fs-journal stall) block every other stream's + ack — 100–450 ms tails. Dirty meta sidecars are swept every ~3 s instead of + rewritten per append (the rename churn was the stall source). +- **One benign race to know about:** the fast-fail 4xx pre-checks run against + the LOCAL store, which on a follower may lag the leader by a beat — e.g. an + append that arrives on node C right after its stream's create was acked via + node A can see a 404 (~1 in 10⁵ under round-robin load). It is retryable, + exactly like Kafka's UNKNOWN_TOPIC right after topic creation. + +### The consensus core + +One openraft instance (a single consensus group / "partition") per cluster: + +- **Proposals** are `Raft::client_write(LogOp)` calls from the HTTP handlers; + the future resolves with the apply outcome — there is no hand-rolled + pending-ack map. If nothing resolves within `--repl-ack-timeout-ms` + (election in progress, dropped forward, stalled quorum) the handler returns + 503 — the client retry is deduped by producer headers. +- **Election/heartbeat**: 100 ms heartbeats, 500–1000 ms election timeout — + fail-over behavior matches the previous incarnation (~0.5–1 s stall). +- **Networking** is a plain TCP RPC mesh: one persistent, auto-reconnecting + connection per peer, length-prefixed bincode frames with correlation ids, + carrying openraft's AppendEntries/Vote/InstallSnapshot plus our + forward-to-leader proposal RPC. openraft retries failed RPCs. +- **Batching**: openraft pipelines and batches AppendEntries under load — one + replication RTT amortizes across every append in flight, the same shape as + the WAL's group commit. +- **Log purge**: every `--repl-snapshot-logs` entries (default 5000) each + node takes a METADATA-ONLY marker snapshot and openraft purges the log + behind it, bounding the in-memory log per node — a down peer does NOT block + reclamation (an improvement over the omnipaxos trim, which needed all + nodes). The stream files remain the real storage; the consensus log is only + the replication conduit. + +### Storage + +The Raft log and vote live in memory (`replication::log_store`, vendored from +openraft's reference memstore). That is deliberate: the whole point of the +mode is that durability comes from replication, not disk. Snapshots are +metadata-only markers — building one is how the log purges; installing one is +REFUSED loudly, so a follower that fell behind the purge horizon stays behind +(fail-stop) instead of silently diverging. See the trade-offs below. + +## What is replicated + +Every mutation of stream state goes through the log: + +| op | LogOp | apply | +| ----------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------- | +| `PUT /s` (create, incl. fork) | `Create{path, config, parent, base_offset, initial wire}` | `store.create` + optional first append | +| `POST /s` (append / close) | `Append{path, wire, producer, seq, close}` | checks → `write_wire` → `publish_durable_tail` (+ closed publish) | +| `DELETE /s` | `Delete{path}` | `delete_or_soft_delete` | + +Fork-point resolution (offset scanning) happens on the proposing node; the +resolved `base_offset` is what's replicated, so the applier never does read +I/O. Reads (`GET`, `HEAD`) are local and hit no consensus. + +## Guarantees and trade-offs + +Accepted trade-offs, stated plainly: + +1. **Quorum-memory durability.** An acked append survives any minority + failure (1 of 3, 2 of 5). It does not survive simultaneous power loss of a + majority. Deploy replicas across failure domains (zones) to make that + event as unlikely as your availability target requires. +2. **Restart-rejoin and node replacement are SUPPORTED.** Two mechanisms: + the Raft **vote is durable** (fsynced to `/repl.vote`; it + changes only at election time, never on the append path), which makes a + restarted node election-safe; and snapshots are **manifests** — stream + metadata whose byte content the joining node pulls over the mesh + (`FetchStream`), which the append-only model makes exact (a `[0, tail)` + prefix is immutable). A restarted node wipes its stale stream files at + boot (the log/snapshot is the source of truth) and resyncs; a brand-new + node joins with `--repl-join` + `POST /_repl/add-learner` + + `POST /_repl/change-membership`. What remains fatal is only simultaneous + majority memory loss (trade-off 1). +3. **One consensus group.** All streams share one decided sequence — a single + ordering pipeline (like a 1-partition Kafka topic, though batching makes + the pipeline wide). Sharding streams across N groups is a natural follow-up + if the single group's decide throughput becomes the ceiling. +4. **One RTT to quorum on every ack** (plus one forward hop when the write + lands on a follower). That's the price of durability-by-replication; with + AppendEntries batching it's paid once per in-flight window, not once per + append. +5. **Latency under leader failure.** Appends stall until the election timeout + (~500 ms) elects a new leader; in-flight proposals on the old leader time + out with 503 and are safe to retry (producer dedup). +6. **Tiering (`--tier`) is not supported with `replicated`** in v1 — every + node would independently offload to S3. Run with tiering off. + +## Configuration + +``` +durable-streams-server \ + --durability replicated \ + --repl-id 1 \ + --repl-peers 1@ds1:5433,2@ds2:5433,3@ds3:5433 \ + [--repl-listen 0.0.0.0:5433] # default: the port from our own peers entry + [--repl-ack-timeout-ms 10000] # propose→commit+apply wait before 503 + [--repl-snapshot-logs 5000] # marker-snapshot/purge cadence, in log entries +``` + +- `--repl-id` — this node's id (1-based, must appear in `--repl-peers`). +- `--repl-join` — start as a JOINING node: skip the initial-membership + bootstrap and wait to be added via the admin endpoints below. +- `--repl-peers` — the **full cluster membership including this node**, as + `id@host:port` of the replication (peer) listeners. Must be identical on + every node. 3 or 5 nodes are the sensible sizes. +- The HTTP `--port` (default 4437) is unchanged and independent. + +## Deploying a cluster + +Three ways, smallest to largest. All live in [`deploy/replicated/`](deploy/replicated/). + +### 1. Local processes (development) + +``` +deploy/replicated/local-cluster.sh up # builds + starts a 3-node cluster on + # http ports 4437/4438/4439 +deploy/replicated/local-cluster.sh status # per-node /_repl/status +deploy/replicated/local-cluster.sh down +``` + +### 2. docker compose + +``` +cd deploy/replicated && docker compose up --build -d +# HTTP on localhost:4437, :4438, :4439 +``` + +### 3. Kubernetes + +``` +kubectl apply -f deploy/replicated/k8s.yaml +``` + +A 3-replica StatefulSet + headless service; each pod derives `--repl-id` from +its ordinal. Spread pods across zones with a topologySpreadConstraint for real +failure-domain independence. See the manifest comments. + +### Smoke test + +`deploy/replicated/smoke.sh` runs an end-to-end check against a local cluster: +create → append on one node → read from another → kill the leader → append +again (fail-over) → verify byte-identical streams on the survivors. + +## Operations + +- **`GET /_repl/status`** on any node returns + `{"id":1,"leader":2,"decided_idx":1234,"connected_peers":[2,3]}` — use it + for readiness checks (`leader != null`) and to find the leader for + lowest-latency writes. +- **Replacing / adding a node** (send both to the LEADER): + 1. boot the new node with `--durability replicated --repl-join --repl-id 4 +--repl-peers ,4@host:port` + 2. `POST /_repl/add-learner {"id":4,"addr":"host:port"}` — blocks until the + learner has caught up (snapshot manifest + byte fetch + log replay) + 3. `POST /_repl/change-membership {"members":[1,2,3,4]}` — joint-consensus + voter switch (also how a dead node is retired: omit it from `members`) +- **Restarting a node in place**: just start it again with the same flags and + data dir — stale stream files are wiped at boot and the durable vote makes + the rejoin election-safe (`local-cluster.sh restart ` locally; the smoke + test exercises exactly this). +- **Sizing**: replication traffic ≈ append traffic × (n-1). The consensus log + holds ≤ `--repl-snapshot-logs` + a small margin of entries per node in + memory (log RSS ≈ that × payload size — lower the flag for large payloads). +- **What to monitor**: `decided_idx` advancing on all nodes; a node whose + `decided_idx` stalls while others advance has lost its mesh links. + +## History: why openraft + +The first implementation used **OmniPaxos** (Sequence Paxos, the only serious +Rust Paxos library). Two things forced the change: + +1. Our deterministic simulation caught a consensus **safety bug** in the last + published release (0.2.2, Nov 2023): a follower's cached Promise message + was not cleared on AcceptSync, so a Promise resend after a partition + + leader change fed the new leader stale log state — the decided logs + diverged and an acked entry vanished on the new leader. The fix existed + upstream (Jan 2024) but was never published. +2. The project is effectively unmaintained, so we were pinned to a git rev of + an abandoned main branch. + +openraft is actively maintained (Databend, CnosDB in production), and the +architecture — log-first apply, apply-outcome acks, sharded appliers, the TCP +mesh, producer-dedup retry safety — carried over intact; the swap touched the +consensus adapter layer only (`types/log_store/sm/net/core`). Two behavioral +improvements came for free: log purge no longer needs every peer up, and the +ack path lost its hand-rolled pending map (`client_write` returns the apply +outcome). The omnipaxos incarnation, including the message-level +deterministic simulation that caught the bug above, lives in git history +(`git log --follow src/replication/`). + +## Validation + +- `replication/tests.rs` — 3-node in-process clusters over real loopback TCP: + convergence, follower forwarding, replicated producer dedup, close/delete, + forks, frequent snapshot/purge (`snapshot_logs: 64`). +- `deploy/replicated/smoke.sh` — end-to-end HTTP incl. leader kill-over. +- openraft brings its own extensive consensus test/chaos suites. +- ROADMAP: port the message-level deterministic simulation (drops, partition + windows, crash-stop, seeded) to openraft via a simulated `AsyncRuntime` — + the harness design is in git history and it has already paid for itself + once. + +## Benchmarks (local 3-node cluster, 10-core M-series, 2026-07-03) + +All three replicas + the ds-bench client share one machine, so absolute +numbers are conservative; the memory-mode column is the same box's +single-node ceiling. ds-bench `multi-stream` (closed loop, 200 streams) and +`sustained` (open loop, 5 min). The omnipaxos column is the previous +incarnation of this mode, same box, same workloads — kept for the record of +the consensus-layer swap. + +Peak throughput (closed loop): + +| workload | openraft | omnipaxos (prev) | single-node `memory` | +| ---------------- | -------------------------------------- | ----------------------- | -------------------- | +| 256 B, 32 conns | 36.3k ops/s, p99 2.1 ms | 40.3k ops/s, p99 2.1 ms | 96k ops/s | +| 256 B, 128 conns | 51.5k ops/s, p99 6.7 ms | 52.5k ops/s, p99 9.2 ms | 100k ops/s | +| 16 KB, 32 conns | **14.9k ops/s (244 MB/s), p99 5.7 ms** | 10.7k ops/s, p99 18 ms | 35k ops/s | + +Sustained (open loop, 5 min, zero ack timeouts, all replicas byte-identical): + +| workload | openraft | omnipaxos (prev) | +| ------------------ | -------------------------------------- | ---------------------------------- | +| 256 B × 22k/s | p99 13 ms, **RSS flat 20 MB** | p99 19 ms, RSS flat 77 MB | +| 16 KB × saturation | 5.2k/s, p99 314 ms, **RSS 116–143 MB** | 4.4k/s, p99 502 ms, RSS 0.2–0.6 GB | + +Reading: peak small-payload throughput is within a few percent (−10% at low +concurrency, −2% near saturation) with equal-or-better tails; large payloads +are **+18–39% faster with 3–4× lower memory** — openraft's entry-count purge +bounds the log at `--repl-snapshot-logs` entries regardless of rate, where +the previous time-based trim held rate × seconds of payloads. **Sizing rule: +log RSS ≈ `--repl-snapshot-logs` × payload size per node; lower the flag for +large payloads.** Watch `REPL_STATS` (`--repl-stats N`): `window` bounded ⇒ +purge keeping up; `apply_max_us` large with idle CPU ⇒ store stalls; +`timeouts` climbing ⇒ elections or a dead quorum. + +## Roadmap + +- **Deterministic simulation** — port the seeded fault-injection harness + (git history: `replication/sim_tests.rs`) to openraft via a simulated + `AsyncRuntime`. +- **Sharded consensus groups** — hash streams across N Raft groups when a + single group's commit pipeline saturates. +- **Large-payload copy reduction** — `Arc<[u8]>` payloads + `serde_bytes`; + 16 KB appends are still copied several times per node. +- **Streaming snapshot install** — install currently rebuilds the store + stream-by-stream before reporting completion; fine at bench scale, + parallel fetch + resumability would help huge stores. +- **Bench integration** — a `replicated` deploy mode in ds-bench + (`scripts/bench`) to put numbers on the wal / memory / replicated triangle + on real (separate-machine) hardware. diff --git a/packages/durable-streams-rust/deploy/replicated/Dockerfile b/packages/durable-streams-rust/deploy/replicated/Dockerfile new file mode 100644 index 0000000000..b7b59cc0bc --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/Dockerfile @@ -0,0 +1,18 @@ +# Replicated-mode image. Build context is the CRATE dir (packages/durable-streams-rust): +# docker build -f deploy/replicated/Dockerfile -t durable-streams:replicated . +# Same shape as the bench image; no extra features needed (tiering is not +# supported with --durability replicated). +ARG RUST_VERSION=1.86 +FROM rust:${RUST_VERSION}-bookworm AS builder +WORKDIR /src +COPY . . +RUN cargo build --release +RUN cp target/release/durable-streams-server /durable-streams-server + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /durable-streams-server /usr/local/bin/durable-streams-server +# 4437 = HTTP (protocol default), 5437 = replication mesh +EXPOSE 4437 5437 +ENTRYPOINT ["durable-streams-server"] diff --git a/packages/durable-streams-rust/deploy/replicated/docker-compose.yml b/packages/durable-streams-rust/deploy/replicated/docker-compose.yml new file mode 100644 index 0000000000..faaa2b0296 --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/docker-compose.yml @@ -0,0 +1,60 @@ +# 3-node replicated durable-streams cluster. +# cd deploy/replicated && docker compose up --build -d +# curl -s localhost:4437/_repl/status # (or :4438 / :4439) +# Any node accepts writes; the leader (see /_repl/status) is lowest-latency. +x-node: &node + build: + context: ../.. + dockerfile: deploy/replicated/Dockerfile + image: durable-streams:replicated + restart: 'no' # v1 is fail-stop: a crashed replica must not auto-rejoin (REPLICATION.md) + +services: + ds1: + <<: *node + command: + - --host + - 0.0.0.0 + - --port + - '4437' + - --data-dir + - /data + - --durability + - replicated + - --repl-id + - '1' + - --repl-peers + - 1@ds1:5437,2@ds2:5437,3@ds3:5437 + ports: ['4437:4437'] + ds2: + <<: *node + command: + - --host + - 0.0.0.0 + - --port + - '4437' + - --data-dir + - /data + - --durability + - replicated + - --repl-id + - '2' + - --repl-peers + - 1@ds1:5437,2@ds2:5437,3@ds3:5437 + ports: ['4438:4437'] + ds3: + <<: *node + command: + - --host + - 0.0.0.0 + - --port + - '4437' + - --data-dir + - /data + - --durability + - replicated + - --repl-id + - '3' + - --repl-peers + - 1@ds1:5437,2@ds2:5437,3@ds3:5437 + ports: ['4439:4437'] diff --git a/packages/durable-streams-rust/deploy/replicated/k8s.yaml b/packages/durable-streams-rust/deploy/replicated/k8s.yaml new file mode 100644 index 0000000000..25fef0a918 --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/k8s.yaml @@ -0,0 +1,95 @@ +# 3-replica durable-streams cluster (--durability replicated). +# +# kubectl apply -f deploy/replicated/k8s.yaml +# +# Notes: +# - Each pod derives --repl-id from its StatefulSet ordinal (+1: ids are 1-based). +# - The headless service gives stable per-pod DNS for the peer mesh. +# - podManagementPolicy: Parallel — consensus needs a quorum up before it can +# elect a leader; ordered (one-by-one, readiness-gated) startup would deadlock +# if readiness required a leader. Readiness below is just "process is up". +# - The topologySpreadConstraint spreads replicas across zones: quorum-memory +# durability is only as good as the failure-domain independence of the +# replicas (REPLICATION.md "Guarantees"). +# - v1 is fail-stop: a restarted pod comes back empty and MUST NOT rejoin a +# running cluster past a trimmed log. Hence restartPolicy is not overridden +# here (k8s always restarts StatefulSet pods) — treat a pod crash as "replace +# the cluster during a maintenance window" until snapshot catch-up lands. +# For benchmarking, fresh clusters per run make this moot. +apiVersion: v1 +kind: Service +metadata: + name: durable-streams-repl + labels: { app: durable-streams-repl } +spec: + clusterIP: None # headless: per-pod DNS for the peer mesh + selector: { app: durable-streams-repl } + ports: + - { name: http, port: 4437 } + - { name: repl, port: 5437 } +--- +apiVersion: v1 +kind: Service +metadata: + name: durable-streams-repl-client + labels: { app: durable-streams-repl } +spec: + # Client-facing entry point; any node accepts writes and serves reads. For + # lowest write latency, point hot producers at the leader (/_repl/status). + selector: { app: durable-streams-repl } + ports: + - { name: http, port: 4437 } +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: durable-streams-repl +spec: + serviceName: durable-streams-repl + replicas: 3 + podManagementPolicy: Parallel + selector: + matchLabels: { app: durable-streams-repl } + template: + metadata: + labels: { app: durable-streams-repl } + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # hard-require in production + labelSelector: + matchLabels: { app: durable-streams-repl } + containers: + - name: server + image: durable-streams:replicated # replace with your registry ref + command: + - /bin/sh + - -c + # Ordinal → 1-based replica id; FQDNs via the headless service. + - | + ORD="${HOSTNAME##*-}" + exec durable-streams-server \ + --host 0.0.0.0 --port 4437 \ + --data-dir /data \ + --durability replicated \ + --repl-id "$((ORD + 1))" \ + --repl-listen 0.0.0.0:5437 \ + --repl-peers "1@durable-streams-repl-0.durable-streams-repl:5437,2@durable-streams-repl-1.durable-streams-repl:5437,3@durable-streams-repl-2.durable-streams-repl:5437" + ports: + - { name: http, containerPort: 4437 } + - { name: repl, containerPort: 5437 } + readinessProbe: + httpGet: { path: /health, port: 4437 } + initialDelaySeconds: 1 + periodSeconds: 2 + resources: + requests: { cpu: '2', memory: 4Gi } + limits: { memory: 8Gi } + volumeMounts: + - { name: data, mountPath: /data } + volumes: + # Stream files are page-cache-backed scratch in this mode (durability is + # the quorum, not this disk); emptyDir keeps the deploy self-contained. + - name: data + emptyDir: {} diff --git a/packages/durable-streams-rust/deploy/replicated/local-cluster.sh b/packages/durable-streams-rust/deploy/replicated/local-cluster.sh new file mode 100755 index 0000000000..7aa08f66a3 --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/local-cluster.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# local-cluster.sh — run a 3-node replicated durable-streams cluster locally. +# +# deploy/replicated/local-cluster.sh up build + start nodes 1..3 +# deploy/replicated/local-cluster.sh status per-node /_repl/status +# deploy/replicated/local-cluster.sh down stop and clean up +# +# Ports: HTTP 4437/4438/4439, replication mesh 5437/5438/5439 (loopback only). +# State: ./.local-cluster/{node/data,node.log,node.pid} +# Env: NODES=3 HTTP_BASE=4437 REPL_BASE=5437 PROFILE=release +# MODE=replicated|memory|wal (memory/wal: single-node baseline, no mesh) +# REPL_STATS_SECS=5 SNAPSHOT_LOGS=5000 EXTRA_ARGS="..." +set -euo pipefail +cd "$(dirname "$0")/../.." + +MODE="${MODE:-replicated}" +NODES="${NODES:-3}" +[ "$MODE" = replicated ] || NODES=1 +HTTP_BASE="${HTTP_BASE:-4437}" +REPL_BASE="${REPL_BASE:-5437}" +PROFILE="${PROFILE:-release}" +REPL_STATS_SECS="${REPL_STATS_SECS:-5}" +SNAPSHOT_LOGS="${SNAPSHOT_LOGS:-5000}" +STATE=".local-cluster" + +peers() { + local out="" + for i in $(seq 1 "$NODES"); do + out+="${out:+,}${i}@127.0.0.1:$((REPL_BASE + i - 1))" + done + echo "$out" +} + +up() { + if [ "$PROFILE" = release ]; then + cargo build --release + BIN=target/release/durable-streams-server + else + cargo build + BIN=target/debug/durable-streams-server + fi + mkdir -p "$STATE" + for i in $(seq 1 "$NODES"); do + local http=$((HTTP_BASE + i - 1)) repl=$((REPL_BASE + i - 1)) + local data="$STATE/node$i/data" + mkdir -p "$data" + local mode_args=() + if [ "$MODE" = replicated ]; then + mode_args=( + --durability replicated + --repl-id "$i" + --repl-peers "$(peers)" + --repl-listen "127.0.0.1:$repl" + --repl-snapshot-logs "$SNAPSHOT_LOGS" + ) + [ "$REPL_STATS_SECS" != 0 ] && mode_args+=(--repl-stats "$REPL_STATS_SECS") + else + mode_args=(--durability "$MODE") + fi + # shellcheck disable=SC2086 + "$BIN" \ + --host 127.0.0.1 --port "$http" \ + --data-dir "$data" \ + "${mode_args[@]}" ${EXTRA_ARGS:-} \ + >"$STATE/node$i.log" 2>&1 & + echo $! >"$STATE/node$i.pid" + echo "node $i: http://127.0.0.1:$http ($MODE, pid $!)" + done + if [ "$MODE" != replicated ]; then + sleep 0.3 + curl -sf "http://127.0.0.1:$HTTP_BASE/health" >/dev/null && echo "single-node $MODE server up" + return 0 + fi + # Wait for a leader (election timeout is ~500 ms). + for _ in $(seq 1 50); do + leader="$(curl -sf "http://127.0.0.1:$HTTP_BASE/_repl/status" 2>/dev/null \ + | sed -n 's/.*"leader":\([0-9null]*\).*/\1/p')" || true + if [ -n "${leader:-}" ] && [ "$leader" != null ]; then + echo "cluster up — leader is node $leader" + return 0 + fi + sleep 0.2 + done + echo "WARNING: no leader observed yet; check $STATE/node*.log" >&2 + return 1 +} + +status() { + for i in $(seq 1 "$NODES"); do + local http=$((HTTP_BASE + i - 1)) + printf 'node %s: ' "$i" + curl -sf "http://127.0.0.1:$http/_repl/status" || printf 'DOWN' + echo + done +} + +down() { + for f in "$STATE"/node*.pid; do + [ -f "$f" ] || continue + kill "$(cat "$f")" 2>/dev/null || true + rm -f "$f" + done + rm -rf "$STATE" + echo "cluster down" +} + +# Restart one node in place (same data dir: streams are wiped at boot and +# resynced from the cluster; the durable vote file is what makes the rejoin +# election-safe). Usage: local-cluster.sh restart +restart_node() { + local i="$1" + [ "$MODE" = replicated ] || { echo "restart is for MODE=replicated" >&2; exit 2; } + local http=$((HTTP_BASE + i - 1)) repl=$((REPL_BASE + i - 1)) + local data="$STATE/node$i/data" + [ -d "$data" ] || { echo "node $i has no state dir" >&2; exit 2; } + if [ -f "$STATE/node$i.pid" ]; then + kill "$(cat "$STATE/node$i.pid")" 2>/dev/null || true + sleep 0.3 + fi + BIN=target/release/durable-streams-server + [ "$PROFILE" = release ] || BIN=target/debug/durable-streams-server + local mode_args=( + --durability replicated + --repl-id "$i" + --repl-peers "$(peers)" + --repl-listen "127.0.0.1:$repl" + --repl-snapshot-logs "$SNAPSHOT_LOGS" + ) + [ "$REPL_STATS_SECS" != 0 ] && mode_args+=(--repl-stats "$REPL_STATS_SECS") + # shellcheck disable=SC2086 + "$BIN" \ + --host 127.0.0.1 --port "$http" \ + --data-dir "$data" \ + "${mode_args[@]}" ${EXTRA_ARGS:-} \ + >>"$STATE/node$i.log" 2>&1 & + echo $! >"$STATE/node$i.pid" + echo "node $i restarted (pid $!)" +} + +case "${1:-}" in + up) up ;; + status) status ;; + down) down ;; + restart) restart_node "${2:?usage: $0 restart }" ;; + *) echo "usage: $0 {up|status|down|restart }" >&2; exit 2 ;; +esac diff --git a/packages/durable-streams-rust/deploy/replicated/monitor.sh b/packages/durable-streams-rust/deploy/replicated/monitor.sh new file mode 100755 index 0000000000..70186bc44d --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/monitor.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# monitor.sh — sample per-node RSS/CPU + /_repl/status of a local cluster to CSV. +# +# deploy/replicated/monitor.sh [interval_secs] > monitor.csv (Ctrl-C to stop) +# +# Columns: ts,node,rss_mb,cpu_pct,leader,decided,log_window,pending,timeouts +# (repl columns are empty when the node is not in replicated mode) +set -euo pipefail +cd "$(dirname "$0")/../.." +INTERVAL="${1:-1}" +HTTP_BASE="${HTTP_BASE:-4437}" +STATE=".local-cluster" + +echo "ts,node,rss_mb,cpu_pct,leader,decided,log_window,pending,timeouts" +while true; do + ts="$(date +%s)" + for f in "$STATE"/node*.pid; do + [ -f "$f" ] || continue + i="$(basename "$f" .pid | sed 's/node//')" + pid="$(cat "$f")" + read -r rss cpu <<<"$(ps -o rss=,pcpu= -p "$pid" 2>/dev/null || echo '0 0')" + s="$(curl -sf --max-time 1 "http://127.0.0.1:$((HTTP_BASE + i - 1))/_repl/status" 2>/dev/null || true)" + leader="$(sed -n 's/.*"leader":\([0-9null]*\).*/\1/p' <<<"$s")" + decided="$(sed -n 's/.*"decided_idx":\([0-9]*\).*/\1/p' <<<"$s")" + window="$(sed -n 's/.*"log_window":\([0-9]*\).*/\1/p' <<<"$s")" + pending="$(sed -n 's/.*"pending":\([0-9]*\).*/\1/p' <<<"$s")" + timeouts="$(sed -n 's/.*"timeouts":\([0-9]*\).*/\1/p' <<<"$s")" + echo "$ts,$i,$((rss / 1024)),$cpu,$leader,$decided,$window,$pending,$timeouts" + done + sleep "$INTERVAL" +done diff --git a/packages/durable-streams-rust/deploy/replicated/smoke.sh b/packages/durable-streams-rust/deploy/replicated/smoke.sh new file mode 100755 index 0000000000..f6210d9689 --- /dev/null +++ b/packages/durable-streams-rust/deploy/replicated/smoke.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# smoke.sh — end-to-end check of a replicated cluster, including leader fail-over. +# +# Boots a local 3-node cluster (via local-cluster.sh), then: +# 1. PUT a stream on node 1 +# 2. POST appends on nodes 2 and 3 (any node accepts writes) +# 3. GET from every node — byte-identical +# 4. kill the LEADER, append again on a survivor (fail-over), read it back +# +# Exits non-zero on the first failed expectation. Cleans up on exit. +set -euo pipefail +cd "$(dirname "$0")" + +HTTP_BASE="${HTTP_BASE:-4437}" +STREAM="/smoke-$$" + +fail() { echo "FAIL: $*" >&2; exit 1; } +pass() { echo " ok: $*"; } + +cleanup() { ./local-cluster.sh down >/dev/null 2>&1 || true; } +trap cleanup EXIT + +echo "== booting 3-node cluster" +./local-cluster.sh up + +url() { echo "http://127.0.0.1:$((HTTP_BASE + $1 - 1))$2"; } + +echo "== create + append via different nodes" +code="$(curl -s -o /dev/null -w '%{http_code}' -X PUT \ + -H 'content-type: application/octet-stream' "$(url 1 "$STREAM")")" +[ "$code" = 201 ] || fail "PUT on node 1 → $code (want 201)" +pass "PUT on node 1 → 201" + +code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'content-type: application/octet-stream' --data-binary 'hello ' "$(url 2 "$STREAM")")" +[ "$code" = 204 ] || fail "POST on node 2 → $code (want 204)" +pass "POST on node 2 (follower or leader — both accept) → 204" + +code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'content-type: application/octet-stream' --data-binary 'world' "$(url 3 "$STREAM")")" +[ "$code" = 204 ] || fail "POST on node 3 → $code (want 204)" +pass "POST on node 3 → 204" + +echo "== every node serves identical bytes" +for i in 1 2 3; do + # Followers apply decided entries asynchronously — poll briefly. + for _ in $(seq 1 50); do + body="$(curl -sf "$(url "$i" "$STREAM")")" || body="" + [ "$body" = "hello world" ] && break + sleep 0.1 + done + [ "$body" = "hello world" ] || fail "node $i read: $(printf %q "$body")" + pass "node $i reads 'hello world'" +done + +echo "== leader fail-over" +leader="$(curl -sf "$(url 1 /_repl/status)" | sed -n 's/.*"leader":\([0-9]*\).*/\1/p')" +[ -n "$leader" ] || fail "no leader in /_repl/status" +kill "$(cat ../../.local-cluster/node"$leader".pid)" # state dir lives at the crate root +pass "killed leader (node $leader)" +survivor=$(( leader == 1 ? 2 : 1 )) + +# Appends stall until re-election (~500 ms); retry until the survivor acks. +ok="" +for _ in $(seq 1 100); do + code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'content-type: application/octet-stream' --data-binary '!' "$(url "$survivor" "$STREAM")")" + [ "$code" = 204 ] && { ok=1; break; } + sleep 0.2 +done +[ -n "$ok" ] || fail "append after leader kill never succeeded (last: $code)" +pass "append on node $survivor acked after fail-over → 204" + +for i in 1 2 3; do + [ "$i" = "$leader" ] && continue + for _ in $(seq 1 50); do + body="$(curl -sf "$(url "$i" "$STREAM")")" || body="" + [ "$body" = "hello world!" ] && break + sleep 0.1 + done + [ "$body" = "hello world!" ] || fail "node $i post-failover read: $(printf %q "$body")" + pass "survivor node $i reads 'hello world!'" +done + +echo "== restart-rejoin (durable vote + snapshot/log resync)" +./local-cluster.sh restart "$leader" +ok="" +for _ in $(seq 1 100); do + body="$(curl -sf "$(url "$leader" "$STREAM")")" || body="" + [ "$body" = "hello world!" ] && { ok=1; break; } + sleep 0.2 +done +[ -n "$ok" ] || fail "restarted node $leader never caught up (read: $(printf %q "$body"))" +pass "restarted node $leader rejoined and reads 'hello world!'" + +code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'content-type: application/octet-stream' --data-binary '?' "$(url "$leader" "$STREAM")")" +[ "$code" = 204 ] || fail "append on restarted node → $code (want 204)" +pass "restarted node accepts writes again → 204" + +echo "SMOKE OK — writes on all nodes, leader loss, restart-rejoin, byte-identical everywhere" diff --git a/packages/durable-streams-rust/scripts/contention-repro-linux.sh b/packages/durable-streams-rust/scripts/contention-repro-linux.sh new file mode 100755 index 0000000000..31f4586fb2 --- /dev/null +++ b/packages/durable-streams-rust/scripts/contention-repro-linux.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# contention-repro-linux.sh — faithful Linux reproduction of the WAL write +# saturation ceiling, for the lock-contention investigation. +# +# Why Linux (vs the native macOS contention-repro.sh): on Linux the WAL fsync is +# a real `fdatasync`, and against a tmpfs data dir it is genuinely cheap (the +# Linux+NVMe regime the findings ran on) — so the per-shard LOCK / committer +# cadence becomes the bottleneck, not the drive barrier. The server and client +# run in separate containers with disjoint cpusets, so the throughput ceiling is +# not confounded by client/server CPU co-location the way the macOS box is. +# +# The server is built incrementally into a named Docker volume (fast rebuilds) +# and run from a slim glibc image; the data dir is an in-container tmpfs. +# +# Usage: +# scripts/contention-repro-linux.sh [--shards N] [--connections C] +# [--streams S] [--duration D] [--warmup W] [--payload P] [--batch B] +# [--srv-cpus 0-5] [--cli-cpus 6-9] [--label NAME] [--no-build] +# [--tmpfs SIZE] [--wal-stats 0|1] +set -euo pipefail + +SHARDS=1 +CONNECTIONS=256 +STREAMS=20000 +DURATION=20 +WARMUP=5 +PAYLOAD=256 +BATCH=1 +SRV_CPUS="0-5" +CLI_CPUS="6-9" +LABEL="" +DO_BUILD=1 +TMPFS_SIZE=2g # at high stream cardinality each non-empty file pins >=1 page: size ~ streams*4k + data +WAL_STATS=1 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CLIENT_CRATE="$(cd "$CRATE_DIR/../../../ds-bench/ds-bench" && pwd)" + +# Distinct names per crate dir so parallel worktrees don't collide on the build +# cache / containers. A short hash of the crate path keys the per-worktree state. +KEY="$(echo "$CRATE_DIR" | cksum | cut -d' ' -f1)" +TARGET_VOL="ds-ct-target-$KEY" +CARGO_VOL="ds-ct-cargo" # registry cache shared across worktrees (read-mostly) +NET="ds-ct-net-$KEY" +SRV="ds-ct-srv-$KEY" +CLIENT_IMG="ds-ct-client:latest" # built once; ds-bench changes rarely + +while [ $# -gt 0 ]; do + case "$1" in + --shards) SHARDS="$2"; shift 2;; + --connections) CONNECTIONS="$2"; shift 2;; + --streams) STREAMS="$2"; shift 2;; + --duration) DURATION="$2"; shift 2;; + --warmup) WARMUP="$2"; shift 2;; + --payload) PAYLOAD="$2"; shift 2;; + --batch) BATCH="$2"; shift 2;; + --srv-cpus) SRV_CPUS="$2"; shift 2;; + --cli-cpus) CLI_CPUS="$2"; shift 2;; + --label) LABEL="$2"; shift 2;; + --no-build) DO_BUILD=0; shift;; + --tmpfs) TMPFS_SIZE="$2"; shift 2;; + --wal-stats) WAL_STATS="$2"; shift 2;; + *) echo "unknown arg: $1" >&2; exit 2;; + esac +done +[ -n "$LABEL" ] || LABEL="linux-shards${SHARDS}-conn${CONNECTIONS}" + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/ds-ct-linux-${LABEL}.XXXXXX")" +CLIENT_JSON="$WORK/client.json"; SRV_LOG="$WORK/server.log"; CPU_LOG="$WORK/cpu.log" + +cleanup() { + [ -n "${CPU_PID:-}" ] && kill "$CPU_PID" 2>/dev/null || true + docker rm -f "$SRV" >/dev/null 2>&1 || true + rm -rf "$WORK" +} +trap cleanup EXIT INT TERM + +docker network inspect "$NET" >/dev/null 2>&1 || docker network create "$NET" >/dev/null + +if [ "$DO_BUILD" = "1" ]; then + echo "== building server (incremental, volume $TARGET_VOL) ==" >&2 + docker run --rm \ + -v "$CRATE_DIR":/app:ro \ + -v "$TARGET_VOL":/target \ + -v "$CARGO_VOL":/usr/local/cargo/registry \ + -w /app rust:1-bookworm \ + cargo build --release --locked --target-dir /target >&2 + echo "== building client image (cached) ==" >&2 + docker build -q -t "$CLIENT_IMG" -f "$CLIENT_CRATE/../dockerfiles/ds-bench.Dockerfile" "$CLIENT_CRATE" >&2 +fi + +echo "== run: $LABEL shards=$SHARDS conn=$CONNECTIONS streams=$STREAMS srv_cpus=$SRV_CPUS cli_cpus=$CLI_CPUS ==" >&2 +docker rm -f "$SRV" >/dev/null 2>&1 || true +# Run the freshly-built glibc binary from the target volume in a slim glibc image +# (bookworm → bookworm, ABI-compatible). Data dir is an in-container tmpfs. +docker run -d --name "$SRV" --network "$NET" \ + --cpuset-cpus="$SRV_CPUS" \ + -v "$TARGET_VOL":/target:ro \ + --tmpfs /data:rw,size="$TMPFS_SIZE" \ + debian:bookworm-slim \ + /target/release/durable-streams-server \ + --host 0.0.0.0 --port 4437 --data-dir /data \ + --durability wal --wal-shards "$SHARDS" --wal-stats "$WAL_STATS" >/dev/null + +for _ in $(seq 1 100); do + docker logs "$SRV" 2>&1 | grep -q "listening on" && break + docker ps -q --filter "name=$SRV" | grep -q . || { echo "server died:" >&2; docker logs "$SRV" >&2; exit 1; } + sleep 0.1 +done + +( while docker ps -q --filter "name=$SRV" | grep -q .; do + docker stats --no-stream --format '{{.CPUPerc}}' "$SRV" 2>/dev/null | tr -d '%' >> "$CPU_LOG" || true + done ) & +CPU_PID=$! + +docker run --rm --network "$NET" --cpuset-cpus="$CLI_CPUS" "$CLIENT_IMG" \ + multi-stream --target "http://$SRV:4437" --api-style durable \ + --streams "$STREAMS" --connections "$CONNECTIONS" --batch "$BATCH" \ + --payload-bytes "$PAYLOAD" --warmup-secs "$WARMUP" --duration-secs "$DURATION" \ + --setup-concurrency 256 >"$CLIENT_JSON" 2>>"$SRV_LOG" || { echo "client failed" >&2; tail -20 "$SRV_LOG" >&2; exit 1; } + +kill "$CPU_PID" 2>/dev/null || true +docker logs "$SRV" >>"$SRV_LOG" 2>&1 || true + +python3 - "$CLIENT_JSON" "$SRV_LOG" "$CPU_LOG" "$LABEL" "$WARMUP" <<'PY' +import json, sys, re +client_json, srv_log, cpu_log, label, warmup = sys.argv[1:6]; warmup=int(warmup) +c=json.load(open(client_json)) +ops=c.get("aggregate_ops_per_sec",0.0); lat=c.get("latency_ms",{}) or {} +p50=lat.get("p50_ms",0.0); p99=lat.get("p99_ms",0.0) +errs=sum(e.get("count",0) for e in (c.get("errors") or [])) +pat=re.compile(r"([\w/]+)=([-+0-9.eE]+)"); rows=[] +for line in open(srv_log,errors="ignore"): + if "WAL_CONT" in line: + d={k:float(v) for k,v in pat.findall(line)} + if d: rows.append(d) +steady=rows[warmup:] if len(rows)>warmup else rows +avg=lambda k:(sum(r[k] for r in steady if k in r)/len([r for r in steady if k in r])) if any(k in r for r in steady) else 0.0 +cpu=[float(x) for x in open(cpu_log).read().split() if x] if __import__("os").path.exists(cpu_log) else [] +cpu_steady=cpu[warmup:] if len(cpu)>warmup else cpu +cpu_avg=sum(cpu_steady)/len(cpu_steady) if cpu_steady else 0.0 +print(f"RESULT label={label}") +print(f" throughput_ops_s = {ops:,.0f}") +print(f" latency_ms p50/p99 = {p50:.1f} / {p99:.1f}") +print(f" errors = {errs}") +print(f" server_cpu_pct = {cpu_avg:.0f} (Docker %CPU; 600 = 6 full cores)") +print(f" fsync_per_s = {avg('fsync/s'):,.0f} batch_avg = {avg('batch_avg'):.1f}") +print(f" inner_wait_us = {avg('inner_wait_us'):.2f} inner_wait_load = {avg('inner_wait_load'):.2f} cores") +print(f" dirty_wait_us = {avg('dirty_wait_us'):.2f} dirty_wait_load = {avg('dirty_wait_load'):.2f} cores") +print(f" waiters_woken_avg = {avg('waiters_woken_avg'):.1f}") +ck=[{k:float(v) for k,v in pat.findall(l)} for l in open(srv_log,errors="ignore") if "WAL_CKPT" in l] +if ck: + n=len(ck) + m=lambda k:max(r.get(k,0.0) for r in ck) + a=lambda k:sum(r.get(k,0.0) for r in ck)/n + print(f" ckpt (n={n}) touched avg/max = {a('touched'):,.0f} / {m('touched'):,.0f} tails_entries max = {m('tails_entries'):,.0f} meta avg = {a('meta'):,.0f}") + print(f" ckpt us avg/max: capture={a('capture_us'):,.0f}/{m('capture_us'):,.0f} fsync={a('fsync_us'):,.0f}/{m('fsync_us'):,.0f} tails={a('tails_us'):,.0f}/{m('tails_us'):,.0f} rest={a('rest_us'):,.0f}/{m('rest_us'):,.0f} meta={a('meta_us'):,.0f}/{m('meta_us'):,.0f} total={a('total_us'):,.0f}/{m('total_us'):,.0f}") +PY diff --git a/packages/durable-streams-rust/scripts/contention-repro.sh b/packages/durable-streams-rust/scripts/contention-repro.sh new file mode 100755 index 0000000000..35bb8c5805 --- /dev/null +++ b/packages/durable-streams-rust/scripts/contention-repro.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# contention-repro.sh — drive the durable WAL server with the ds-bench pool +# client and report write throughput ALONGSIDE the per-shard lock-contention +# rates (the `WAL_CONT` stderr lines from `--wal-stats`). This is the local +# reproduction harness for the write-saturation contention study: it makes the +# per-shard `inner`/`dirty` lock-wait and the durability wakeup fan-out visible +# next to ops/s, so a candidate architecture change can be judged on whether it +# lifts throughput AND drops the contention it was supposed to. +# +# It runs the server and the load generator on the SAME box (fine for surfacing +# LOCK contention — the locks serialize regardless of where the load comes from; +# absolute ops/s is lower than a split client/server, but the contention signal +# and its before/after delta are what matter here). +# +# Usage: +# scripts/contention-repro.sh [--shards N] [--connections C] [--streams S] +# [--duration D] [--warmup W] [--payload P] [--batch B] [--port PORT] +# [--label NAME] [--server BIN] [--client BIN] [--keep-logs] +# +# Example — force single-shard contention, then the cores-many-shard baseline: +# scripts/contention-repro.sh --shards 1 --connections 256 --label shards1 +# scripts/contention-repro.sh --shards 10 --connections 256 --label shards10 +set -euo pipefail + +# ---- defaults ------------------------------------------------------------- +SHARDS=1 +CONNECTIONS=256 +STREAMS=20000 +DURATION=20 +WARMUP=5 +PAYLOAD=256 +BATCH=1 +PORT=4500 +LABEL="" +KEEP_LOGS=0 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SERVER_BIN="$CRATE_DIR/target/release/durable-streams-server" +# The ds-bench load generator. Default to the sibling ds-bench checkout. +CLIENT_BIN="$CRATE_DIR/../../../ds-bench/ds-bench/target/release/ds-bench" + +while [ $# -gt 0 ]; do + case "$1" in + --shards) SHARDS="$2"; shift 2;; + --connections) CONNECTIONS="$2"; shift 2;; + --streams) STREAMS="$2"; shift 2;; + --duration) DURATION="$2"; shift 2;; + --warmup) WARMUP="$2"; shift 2;; + --payload) PAYLOAD="$2"; shift 2;; + --batch) BATCH="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --label) LABEL="$2"; shift 2;; + --server) SERVER_BIN="$2"; shift 2;; + --client) CLIENT_BIN="$2"; shift 2;; + --keep-logs) KEEP_LOGS=1; shift;; + *) echo "unknown arg: $1" >&2; exit 2;; + esac +done + +[ -x "$SERVER_BIN" ] || { echo "server binary not found/executable: $SERVER_BIN (run: cargo build --release)" >&2; exit 2; } +[ -x "$CLIENT_BIN" ] || { echo "client binary not found/executable: $CLIENT_BIN (build ds-bench: cargo build --release)" >&2; exit 2; } + +[ -n "$LABEL" ] || LABEL="shards${SHARDS}-conn${CONNECTIONS}-streams${STREAMS}" + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/ds-contention-${LABEL}.XXXXXX")" +DATA_DIR="$WORK/data" +SRV_LOG="$WORK/server.log" +CLIENT_JSON="$WORK/client.json" +CPU_LOG="$WORK/cpu.log" +mkdir -p "$DATA_DIR" + +cleanup() { + [ -n "${CPU_PID:-}" ] && kill "$CPU_PID" 2>/dev/null || true + [ -n "${SRV_PID:-}" ] && kill "$SRV_PID" 2>/dev/null || true + wait 2>/dev/null || true + if [ "$KEEP_LOGS" = "1" ]; then + echo "logs kept in: $WORK" >&2 + else + rm -rf "$WORK" + fi +} +trap cleanup EXIT INT TERM + +echo "== contention-repro: $LABEL ==" >&2 +echo " shards=$SHARDS connections=$CONNECTIONS streams=$STREAMS batch=$BATCH payload=$PAYLOAD duration=${DURATION}s warmup=${WARMUP}s" >&2 + +# ---- launch the server (--wal-stats 1 → one WAL_CONT line/sec on stderr) --- +# DS_BENCH_FAST_FSYNC: on macOS, use plain fsync instead of F_FULLFSYNC so the +# RAM-disk data dir gives cheap fsync (the Linux+NVMe regime) and the per-shard +# LOCK becomes the bottleneck instead of the drive barrier. Bench-only; honoured +# from the caller's env, defaulting on for this harness. +DS_BENCH_FAST_FSYNC="${DS_BENCH_FAST_FSYNC:-1}" \ +"$SERVER_BIN" \ + --host 127.0.0.1 --port "$PORT" \ + --data-dir "$DATA_DIR" \ + --durability wal \ + --wal-shards "$SHARDS" \ + --wal-stats 1 \ + >/dev/null 2>"$SRV_LOG" & +SRV_PID=$! + +# wait for "listening" (≤10s) +for _ in $(seq 1 100); do + grep -q "listening on" "$SRV_LOG" 2>/dev/null && break + kill -0 "$SRV_PID" 2>/dev/null || { echo "server died at startup:" >&2; cat "$SRV_LOG" >&2; exit 1; } + sleep 0.1 +done + +# ---- sample server CPU (%) once/sec while the client runs ----------------- +( while kill -0 "$SRV_PID" 2>/dev/null; do + ps -o %cpu= -p "$SRV_PID" 2>/dev/null | tr -d ' ' >> "$CPU_LOG" || true + sleep 1 + done ) & +CPU_PID=$! + +# ---- drive the load ------------------------------------------------------ +"$CLIENT_BIN" multi-stream \ + --target "http://127.0.0.1:$PORT" \ + --api-style durable \ + --streams "$STREAMS" \ + --connections "$CONNECTIONS" \ + --batch "$BATCH" \ + --payload-bytes "$PAYLOAD" \ + --warmup-secs "$WARMUP" \ + --duration-secs "$DURATION" \ + --setup-concurrency 256 \ + >"$CLIENT_JSON" 2>>"$SRV_LOG" || { echo "client failed:" >&2; tail -20 "$SRV_LOG" >&2; exit 1; } + +kill "$CPU_PID" 2>/dev/null || true + +# ---- summarize (python3: parse client JSON + steady-state WAL_CONT + CPU) - +python3 - "$CLIENT_JSON" "$SRV_LOG" "$CPU_LOG" "$LABEL" "$WARMUP" <<'PY' +import json, sys, re +client_json, srv_log, cpu_log, label, warmup = sys.argv[1:6] +warmup = int(warmup) + +with open(client_json) as f: + c = json.load(f) +ops = c.get("aggregate_ops_per_sec", 0.0) +lat = c.get("latency_ms", {}) or {} +p50 = lat.get("p50_ms", lat.get("p50", 0.0)) +p99 = lat.get("p99_ms", lat.get("p99", 0.0)) +errs = sum(e.get("count", 0) for e in (c.get("errors") or [])) + +# WAL_CONT steady-state: drop the first `warmup` lines (warmup window), average. +pat = re.compile(r"([\w/]+)=([-+0-9.eE]+)") +rows = [] +with open(srv_log, errors="ignore") as f: + for line in f: + if "WAL_CONT" not in line: + continue + d = {k: float(v) for k, v in pat.findall(line)} + if d: + rows.append(d) +steady = rows[warmup:] if len(rows) > warmup else rows +def avg(key): + vals = [r[key] for r in steady if key in r] + return sum(vals) / len(vals) if vals else 0.0 + +# CPU: drop the first `warmup` samples, average; macOS %cpu is per-core summed +# (can exceed 100% on multi-core). +cpu_vals = [] +try: + with open(cpu_log) as f: + for line in f: + line = line.strip() + if line: + try: cpu_vals.append(float(line)) + except ValueError: pass +except FileNotFoundError: + pass +cpu_steady = cpu_vals[warmup:] if len(cpu_vals) > warmup else cpu_vals +cpu_avg = sum(cpu_steady) / len(cpu_steady) if cpu_steady else 0.0 + +print(f"RESULT label={label}") +print(f" throughput_ops_s = {ops:,.0f}") +print(f" latency_ms p50/p99 = {p50:.1f} / {p99:.1f}") +print(f" errors = {errs}") +print(f" server_cpu_pct = {cpu_avg:.0f} (summed across cores; 1000 = 10 full cores)") +print(f" staged_per_s = {avg('staged/s'):,.0f}") +print(f" fsync_per_s = {avg('fsync/s'):,.0f} batch_avg = {avg('batch_avg'):.1f}") +print(f" inner_wait_us = {avg('inner_wait_us'):.2f} inner_wait_load = {avg('inner_wait_load'):.2f} cores") +print(f" dirty_wait_us = {avg('dirty_wait_us'):.2f} dirty_wait_load = {avg('dirty_wait_load'):.2f} cores") +print(f" waiters_woken_avg = {avg('waiters_woken_avg'):.1f}") +PY diff --git a/packages/durable-streams-rust/src/engine_raw.rs b/packages/durable-streams-rust/src/engine_raw.rs index 1e7558ea05..4de4aad121 100644 --- a/packages/durable-streams-rust/src/engine_raw.rs +++ b/packages/durable-streams-rust/src/engine_raw.rs @@ -14,8 +14,6 @@ // zero-copy page-cache → socket transfer, but a fault parks a pool thread // instead. ReadOffload picks where each read runs; see set_read_offload. -#[cfg(target_os = "linux")] -use std::sync::atomic::AtomicBool; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Arc; @@ -60,25 +58,6 @@ impl ReadOffload { static READ_OFFLOAD: AtomicU8 = AtomicU8::new(ReadOffload::Tail as u8); -/// Process-global zero-copy-append toggle, set by `--durability memory` (Linux -/// only). When on, eligible binary appends are served via the socket→file splice -/// intercept and the tail cache is off. -#[cfg(target_os = "linux")] -static ZERO_COPY: AtomicBool = AtomicBool::new(false); - -/// Enable/disable the zero-copy append path (process-global). -#[cfg(target_os = "linux")] -pub fn set_zero_copy(on: bool) { - ZERO_COPY.store(on, Ordering::Relaxed); -} - -/// Whether the zero-copy append path is active. -#[cfg(target_os = "linux")] -#[inline] -pub fn zero_copy() -> bool { - ZERO_COPY.load(Ordering::Relaxed) -} - /// Set the read-offload strategy (process-global). Called once at startup. pub fn set_read_offload(mode: ReadOffload) { READ_OFFLOAD.store(mode as u8, Ordering::Relaxed); @@ -222,85 +201,6 @@ async fn conn_loop( let keep_alive = head.keep_alive; let is_head = head.is_head; - // ---- zero-copy binary-append intercept (Linux only, --durability memory) ---- - // - // Eligible: zero-copy on (set by --durability memory), POST, a known - // content-length (not chunked), and the route is a binary append. We DON'T - // `read_sized` here; instead the body is relayed socket→file via splice(2). - // The handler returns `Fallback` for any edge case (producer dup/gap, - // closed, close request, JSON, content-type mismatch, …), in which case we - // read the body buffered below and run the normal handler — byte-for-byte - // unchanged. - #[cfg(target_os = "linux")] - if zero_copy() - && matches!(head.method, crate::api::Method::Post) - && !head.chunked - && head.content_length.is_some_and(|n| n > 0) - { - let content_len = head.content_length.unwrap(); - // The bytes the head parser over-read are the body prefix. Take up to - // `content_len` of them (a tiny pipelined next-request tail must not be - // counted as body); the rest of the body is still on the socket. - let take = buf.len().min(content_len); - let prefix = buf.split_to(take).freeze(); - let remaining = content_len - prefix.len(); - // splice_rest moves the remaining socket bytes into (file_fd, off), - // awaiting socket read-readiness — the tokio `TcpStream` is - // non-blocking, so a splice that returns EAGAIN (body not fully - // arrived) must park on readiness rather than error. It borrows - // `stream` immutably (read-readiness only); the borrow lives across - // the handler's await while the appender lock is held — fine, the - // lock is an AsyncMutex. - let stream_ref = &stream; - let splice_rest = move |file_fd: std::os::fd::RawFd, off: i64| async move { - if remaining == 0 { - return Ok(()); - } - let mut o = off; - zerocopy::splice_socket_to_file(stream_ref, file_fd, &mut o, remaining).await - }; - let req = Req { - method: head.method, - path: head.path.clone(), - query: head.query.clone(), - headers: head.headers.clone(), - body: Bytes::new(), - }; - let outcome = handlers::handle_binary_append_zero_copy( - store.clone(), - &req, - &prefix, - content_len, - splice_rest, - ) - .await; - match outcome { - handlers::ZeroCopyOutcome::Done(resp) => { - write_response(&mut stream, resp, is_head, keep_alive).await?; - if !keep_alive { - return Ok(()); - } - continue; - } - handlers::ZeroCopyOutcome::DoneClose(resp) => { - // A splice leg failed mid-append: body bytes may already be - // consumed from the socket, so the connection is desynced. - // Send the 500 with `connection: close` and drop the - // connection — never reuse it as keep-alive. - write_response(&mut stream, resp, is_head, false).await?; - return Ok(()); - } - handlers::ZeroCopyOutcome::Fallback => { - // Put the prefix back at the front of the buffer so the - // buffered read below sees the full body again. - let mut rebuilt = BytesMut::with_capacity(prefix.len() + buf.len()); - rebuilt.extend_from_slice(&prefix); - rebuilt.extend_from_slice(&buf); - buf = rebuilt; - } - } - } - // ---- read request body ---- let body = if head.chunked { match http1::decode_chunked(&mut stream, &mut buf).await? { @@ -638,128 +538,6 @@ async fn write_segment(stream: &mut TcpStream, seg: &Segment) -> std::io::Result Ok(()) } -/// Zero-copy file-to-file / file-to-socket relay via `splice(2)`. -#[cfg(target_os = "linux")] -pub(crate) mod zerocopy { - use std::io; - use std::os::fd::{AsRawFd, RawFd}; - - /// Move exactly `len` bytes from a tokio (non-blocking) `TcpStream` into the - /// regular file `file_fd` at `*file_off`, in-kernel via an anonymous pipe, - /// awaiting socket read-readiness between attempts. - /// - /// The socket→pipe leg uses `stream.async_io(READABLE, …)`: a `splice(2)` - /// that would block (`EAGAIN`/`EWOULDBLOCK` — the body has not fully arrived - /// in the socket receive buffer) is reported to tokio as - /// `ErrorKind::WouldBlock`, so `async_io` parks on readiness and retries - /// instead of erroring. The pipe→file leg drains synchronously: the - /// destination is a regular file (positioned write, never `EAGAIN`), so no - /// readiness wait is needed and `*file_off` is advanced as bytes land. - /// - /// A socket `splice` returning 0 (peer closed before `len` bytes arrived) is - /// a hard error (`UnexpectedEof`) — and, crucially, NOT `WouldBlock`, so - /// `async_io` propagates it rather than spinning. Holding the appender - /// `AsyncMutex` across this await is intentional (per-stream serialization). - pub async fn splice_socket_to_file( - stream: &tokio::net::TcpStream, - file_fd: RawFd, - file_off: &mut i64, - mut len: usize, - ) -> io::Result<()> { - if len == 0 { - return Ok(()); - } - let socket_fd = stream.as_raw_fd(); - // One pipe for the whole transfer (re-created per call, not per retry): - // bytes spliced socket→pipe are drained pipe→file before the next - // readiness wait, so the pipe is empty across awaits. - let mut fds = [0i32; 2]; - // SAFETY: fds is a 2-element array; pipe2 fills both on success. - if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 { - return Err(io::Error::last_os_error()); - } - let (pr, pw) = (fds[0], fds[1]); - let res = async { - while len > 0 { - // socket → pipe, awaiting readiness. async_io retries ONLY on - // WouldBlock; any other error (incl. our EOF below) propagates. - let moved = stream - .async_io(tokio::io::Interest::READABLE, || { - // SAFETY: pw is the write end of a valid pipe; socket_fd - // is the stream's fd. splice aliases no Rust references. - let n = unsafe { - libc::splice( - socket_fd, - std::ptr::null_mut(), - pw, - std::ptr::null_mut(), - len, - libc::SPLICE_F_MOVE | libc::SPLICE_F_MORE, - ) - }; - if n < 0 { - let err = io::Error::last_os_error(); - // EAGAIN == EWOULDBLOCK on Linux: body not fully in - // the socket buffer yet. Map to WouldBlock so async_io - // parks on readiness and retries. - return match err.raw_os_error() { - Some(libc::EAGAIN) => Err(io::ErrorKind::WouldBlock.into()), - _ => Err(err), - }; - } - if n == 0 { - // Peer closed before all `len` bytes arrived. NOT - // WouldBlock, so this propagates out of async_io. - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "socket EOF before full append body", - )); - } - Ok(n as usize) - }) - .await?; - // pipe → file (drain exactly `moved` bytes; regular file, no EAGAIN). - let mut drained = 0usize; - while drained < moved { - // SAFETY: pr is the read end of the same pipe; file_fd is the - // splice file; *file_off is a valid &mut i64 cast to *mut i64. - let m = unsafe { - libc::splice( - pr, - std::ptr::null_mut(), - file_fd, - file_off as *mut i64, - moved - drained, - libc::SPLICE_F_MOVE | libc::SPLICE_F_MORE, - ) - }; - if m < 0 { - return Err(io::Error::last_os_error()); - } - if m == 0 { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "splice pipe drain returned 0", - )); - } - drained += m as usize; - } - len -= moved; - } - Ok(()) - } - .await; - // SAFETY: close both pipe ends regardless of outcome; valid (pipe2 - // succeeded) and unused after this point. - unsafe { - libc::close(pr); - libc::close(pw); - } - res - } - -} - /// Portable fallback: positioned reads through a reusable buffer. #[cfg(not(target_os = "linux"))] async fn write_segment(stream: &mut TcpStream, seg: &Segment) -> std::io::Result<()> { diff --git a/packages/durable-streams-rust/src/handlers.rs b/packages/durable-streams-rust/src/handlers.rs index 061e684f72..54a7317159 100644 --- a/packages/durable-streams-rust/src/handlers.rs +++ b/packages/durable-streams-rust/src/handlers.rs @@ -46,15 +46,19 @@ pub enum DurabilityMode { /// No WAL, no fsync: ack on the page-cache write. Durability comes from replication /// (future). Linux-only (binary appends use zero-copy socket→file). Memory, + /// No WAL, no fsync: ack once the op is decided by a quorum of replicas + /// (OmniPaxos) and applied locally. See REPLICATION.md. + Replicated, } static DURABILITY_MODE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); -/// Parse the `--durability` value. `wal` | `memory`; `None` → usage error. +/// Parse the `--durability` value. `wal` | `memory` | `replicated`; `None` → usage error. pub fn parse_durability(s: &str) -> Option { match s { "wal" => Some(DurabilityMode::Wal), "memory" => Some(DurabilityMode::Memory), + "replicated" => Some(DurabilityMode::Replicated), _ => None, } } @@ -66,6 +70,7 @@ pub fn set_durability(mode: DurabilityMode) { pub fn durability() -> DurabilityMode { match DURABILITY_MODE.load(std::sync::atomic::Ordering::Relaxed) { 1 => DurabilityMode::Memory, + 2 => DurabilityMode::Replicated, _ => DurabilityMode::Wal, } } @@ -266,19 +271,78 @@ async fn dispatch(store: Arc, req: Req) -> Resp { let path = req.path.clone(); if path == "/health" { text_response(200, "ok") + } else if path == "/_repl/status" { + if durability() == DurabilityMode::Replicated { + let mut r = Resp::new(200); + r.headers.push(("content-type", "application/json".to_string())); + r.body = full(crate::replication::handle().status_json()); + r + } else { + text_response(404, "not in replicated mode") + } + } else if path == "/_repl/add-learner" || path == "/_repl/change-membership" { + if durability() != DurabilityMode::Replicated { + text_response(404, "not in replicated mode") + } else if req.method != Method::Post { + text_response(405, "method not allowed") + } else { + handle_repl_admin(&path, &req.body).await + } } else { match req.method { Method::Put => handle_create(store, req, path).await, Method::Post => handle_append(store, req, path).await, Method::Get => handle_read(store, req, path).await, Method::Head => handle_head(store, path), - Method::Delete => handle_delete(store, path), + Method::Delete => handle_delete(store, path).await, Method::Options => ResponseBuilder::new(204).body(empty()), Method::Other => text_response(405, "method not allowed"), } } } +// ---------- replicated-mode admin (leader-only membership ops) ---------- + +/// `POST /_repl/add-learner {"id":4,"addr":"host:port"}` — start replicating +/// (snapshot manifest + byte fetch + log) to a new node without a vote. +/// `POST /_repl/change-membership {"members":[1,2,3,4]}` — joint-consensus +/// switch of the voter set (promote learners / retire nodes). +/// Both must be sent to the leader (see `/_repl/status`); elsewhere → 503. +async fn handle_repl_admin(path: &str, body: &Bytes) -> Resp { + let raft = &crate::replication::handle().raft; + if path == "/_repl/add-learner" { + #[derive(serde::Deserialize)] + struct AddLearner { + id: u64, + addr: String, + } + let req: AddLearner = match serde_json::from_slice(body) { + Ok(v) => v, + Err(e) => return text_response(400, &format!("bad body: {e}")), + }; + match raft + .add_learner(req.id, openraft::BasicNode::new(req.addr), true) + .await + { + Ok(_) => text_response(200, "learner added and caught up"), + Err(e) => text_response(503, &format!("add-learner failed: {e}")), + } + } else { + #[derive(serde::Deserialize)] + struct ChangeMembership { + members: std::collections::BTreeSet, + } + let req: ChangeMembership = match serde_json::from_slice(body) { + Ok(v) => v, + Err(e) => return text_response(400, &format!("bad body: {e}")), + }; + match raft.change_membership(req.members, false).await { + Ok(_) => text_response(200, "membership changed"), + Err(e) => text_response(503, &format!("change-membership failed: {e}")), + } + } +} + // ---------- PUT (create) ---------- fn parse_ttl(v: &str) -> Result { @@ -551,6 +615,55 @@ async fn handle_create(store: Arc, req: Req, path: String) -> Resp { } }; + // Replicated mode: the create (fork point pre-resolved above) goes through + // the consensus log; every node — this one included — applies it from the + // decided entry (log-first apply, REPLICATION.md). The response is built + // from the apply outcome so it matches the single-node path. + if durability() == DurabilityMode::Replicated { + use crate::replication::entry::{CreateApplyOutcome as C, LogOp, OpOutcome}; + let op = LogOp::Create { + path: path.clone(), + config, + base_offset, + wire: wire.as_ref().map(|w| w.to_vec()).unwrap_or_default(), + }; + return match crate::replication::handle().propose_and_wait(op).await { + Err(_) => text_response(503, "replication timeout — retry"), + Ok(OpOutcome::Create(c)) => match c { + C::Created { tail, closed } => { + let mut b = ResponseBuilder::new(201) + .h( + "location", + format!("http://{}{}", host.as_deref().unwrap_or("localhost"), path), + ) + .h("content-type", content_type) + .h(H_NEXT_OFFSET, format_offset(tail)); + if closed { + b = b.hs(H_CLOSED, "true"); + } + b.body(empty()) + } + C::Exists { + tail, + closed, + content_type, + } => { + let mut b = ResponseBuilder::new(200) + .h("content-type", content_type) + .h(H_NEXT_OFFSET, format_offset(tail)); + if closed { + b = b.hs(H_CLOSED, "true"); + } + b.body(empty()) + } + C::Conflict => text_response(409, "stream exists with different configuration"), + C::ForkSourceMissing => text_response(409, "fork source is deleted"), + C::WriteFailed => text_response(500, "write failed"), + }, + Ok(_) => text_response(500, "unexpected apply outcome"), + }; + } + // Run create on the blocking pool: it opens the data file and does a durable // (fsync) `.meta` write, which would otherwise block an async worker for the // whole fsync. Under concurrent stream creation that throttles creates to @@ -733,7 +846,7 @@ async fn wait_durable_lsn(store: &Arc, st: &Arc, lsn: u64) { /// ordering — PROTOCOL.md §4.1). Adds no fsync: the per-stream file stays /// async/WAL-recoverable; the only durability barrier is the WAL `fdatasync` /// awaited in `wait_durable_lsn`. -fn write_wire(st: &StreamState, ap: &mut Appender, wire: &Bytes) -> std::io::Result { +pub(crate) fn write_wire(st: &StreamState, ap: &mut Appender, wire: &Bytes) -> std::io::Result { use std::io::Write; (&*ap.file).write_all(wire)?; ap.written += wire.len() as u64; @@ -756,7 +869,7 @@ fn write_wire(st: &StreamState, ap: &mut Appender, wire: &Bytes) -> std::io::Res /// concurrent appenders (whose group-commit fsyncs may resolve out of order) /// safe: a later appender publishing the higher frontier first is fine (all /// lower bytes are durable too), and the earlier appender then no-ops. -fn publish_durable_tail(st: &StreamState, tail: u64, wire: &Bytes) { +pub(crate) fn publish_durable_tail(st: &StreamState, tail: u64, wire: &Bytes) { let closed; { let mut s = st.shared.write().unwrap(); @@ -864,6 +977,256 @@ fn gone() -> Resp { text_response(410, "stream is deleted") } +// ---------- replicated apply (log-first) ---------- +// +// The semantic twins of the single-node PUT/POST/DELETE paths, invoked by the +// replication core for every DECIDED log entry, in log order, on every node +// (leader included). They mirror the checks above exactly — closed → producer +// dedup/fencing → Stream-Seq — so replicas stay a deterministic function of +// the decided log. "Decided by a quorum" is this mode's durability, so bytes +// and closures are exposed to readers immediately after apply; the meta +// sidecar flush stays async (REPLICATION.md). + +/// Apply a decided `LogOp::Create`. The fork point (`base_offset`) was +/// resolved by the proposing node; only the parent lookup happens here. +pub(crate) async fn apply_replicated_create( + store: &Arc, + path: &str, + config: StreamConfig, + base_offset: u64, + wire: Vec, +) -> crate::replication::entry::CreateApplyOutcome { + use crate::replication::entry::CreateApplyOutcome as C; + let parent = match &config.forked_from { + Some(src) => match store.get(src) { + Some(s) if !s.shared.read().unwrap().soft_deleted => Some(s), + _ => return C::ForkSourceMissing, + }, + None => None, + }; + let result = { + let store2 = Arc::clone(store); + let path2 = path.to_string(); + // durable_meta: false — the decided log entry is the create's + // durability; an fsync here would serialize all creates through this + // single applier task (macOS F_FULLFSYNC makes that seconds per 1000). + match tokio::task::spawn_blocking(move || { + store2.create_with_meta_durability(&path2, config, parent, base_offset, false) + }) + .await + { + Ok(Ok(r)) => r, + _ => return C::WriteFailed, + } + }; + match result { + CreateResult::Conflict => C::Conflict, + CreateResult::Exists(st) => { + st.touch(); + let t = st.tail(); + C::Exists { + tail: t.bytes, + closed: t.closed, + content_type: st.config.content_type.clone(), + } + } + CreateResult::Created(st) => { + if !wire.is_empty() { + let wire = Bytes::from(wire); + let mut ap = st.appender.lock().await; + let new_tail = match write_wire(&st, &mut ap, &wire) { + Ok(t) => t, + Err(_) => return C::WriteFailed, + }; + drop(ap); + publish_durable_tail(&st, new_tail, &wire); + } + let t = st.tail(); + C::Created { + tail: t.bytes, + closed: t.closed, + } + } + } +} + +/// Apply a decided `LogOp::Append` — the authoritative twin of +/// `handle_append_inner`'s checked-write section. +pub(crate) async fn apply_replicated_append( + store: &Arc, + path: &str, + wire: Vec, + producer: Option, + seq_header: Option, + close_req: bool, +) -> crate::replication::entry::AppendApplyOutcome { + use crate::replication::entry::AppendApplyOutcome as A; + let wire = Bytes::from(wire); + let st = match store.get(path) { + Some(s) => s, + None => return A::NotFound, + }; + if st.shared.read().unwrap().soft_deleted { + return A::Gone; + } + let lock_t0 = crate::telemetry::Timer::start(); + let mut ap = st.appender.lock().await; + crate::telemetry::record_append_lock_wait(lock_t0.elapsed_secs()); + + // Closed checks (precedence: closed → producer → seq), as in the single-node path. + { + let s = st.shared.read().unwrap(); + if s.closed { + let tail = s.durable_tail; + if close_req { + if let Some(p) = &producer { + if let Some((cid, cep, cseq)) = &s.closed_by { + if *cid == p.id && *cep == p.epoch && *cseq == p.seq { + return A::ClosedDupClose { + tail, + epoch: p.epoch, + seq: p.seq, + }; + } + } + return A::Closed { tail }; + } + if wire.is_empty() { + return A::ClosedIdempotent { tail }; + } + } + return A::Closed { tail }; + } + } + if let Some(p) = &producer { + let ph = ProducerHeaders { + id: p.id.clone(), + epoch: p.epoch, + seq: p.seq, + }; + let outcome = { + let s = st.shared.read().unwrap(); + validate_producer(&s, &ph) + }; + match outcome { + ProducerOutcome::Accept => {} + ProducerOutcome::Duplicate { last_seq } => { + let (tail, closed) = { + let s = st.shared.read().unwrap(); + (s.durable_tail, s.closed_durable) + }; + return A::ProducerDuplicate { + tail, + closed, + epoch: p.epoch, + last_seq, + }; + } + ProducerOutcome::StaleEpoch { current } => { + let tail = st.shared.read().unwrap().durable_tail; + return A::ProducerStaleEpoch { tail, current }; + } + ProducerOutcome::Gap { expected } => { + return A::ProducerGap { + expected, + received: p.seq, + } + } + ProducerOutcome::BadEpochStart => return A::ProducerBadEpochStart, + } + } + if let Some(seq) = &seq_header { + let s = st.shared.read().unwrap(); + if let Some(last) = &s.last_seq_header { + if seq.as_str() <= last.as_str() { + return A::SeqConflict { + tail: s.durable_tail, + }; + } + } + } + + // NOTE: a local write failure here (disk error) after the entry decided + // means this replica diverges from the log — same failure class as a WAL + // fsync error in `wal` mode. The 500 surfaces it; the node should be + // replaced (v1 fail-stop assumption, REPLICATION.md). + let mut new_tail = None; + if !wire.is_empty() { + match write_wire(&st, &mut ap, &wire) { + Ok(t) => new_tail = Some(t), + Err(_) => return A::WriteFailed, + } + } + { + let mut s = st.shared.write().unwrap(); + if let Some(p) = &producer { + s.producers.insert( + p.id.clone(), + ProducerState { + epoch: p.epoch, + last_seq: p.seq, + }, + ); + } + if let Some(seq) = seq_header { + s.last_seq_header = Some(seq); + } + if close_req { + s.closed = true; + if let Some(p) = &producer { + s.closed_by = Some((p.id.clone(), p.epoch, p.seq)); + } + } + } + drop(ap); + + // Decided ⇒ durable in this mode: expose bytes (and the closure) now. + if let Some(t) = new_tail { + publish_durable_tail(&st, t, &wire); + } + let (tail, closed) = if close_req { + let tail = { + let mut s = st.shared.write().unwrap(); + s.closed_durable = true; + s.durable_tail + }; + st.tail_tx.send_replace(Tail { + bytes: tail, + closed: true, + }); + #[cfg(target_os = "linux")] + crate::sse_reactor::wake_stream(&st); + (tail, true) + } else { + let s = st.shared.read().unwrap(); + (s.durable_tail, s.closed_durable) + }; + // No per-append meta flush here: the shard applier sweeps dirty sidecars + // every few seconds (core::META_SWEEP) — per-append 100 ms-debounced + // rewrites generated enough rename churn to stall the fs journal. + A::Applied { tail, closed } +} + +/// Apply a decided `LogOp::Delete`. The decided log entry is the durability of +/// the delete, so the local removal uses the non-durable variant. +pub(crate) async fn apply_replicated_delete( + store: &Arc, + path: &str, +) -> crate::replication::entry::DeleteApplyOutcome { + use crate::replication::entry::DeleteApplyOutcome as D; + let st = match store.get(path) { + Some(s) => s, + None => return D::NotFound, + }; + if st.shared.read().unwrap().soft_deleted { + return D::Gone; + } + let store2 = Arc::clone(store); + let st2 = Arc::clone(&st); + let _ = tokio::task::spawn_blocking(move || store2.delete_or_soft_delete(&st2)).await; + D::Deleted +} + /// Append outcome, recorded as a bounded metric label on `ds.append.duration`. #[derive(Clone, Copy)] enum AppendOutcome { @@ -886,25 +1249,26 @@ impl AppendOutcome { async fn handle_append(store: Arc, req: Req, path: String) -> Resp { let t0 = crate::telemetry::Timer::start(); - // is_json is needed for the metric label even on the not-found path, where we - // don't have a stream; default to false there. - let is_json = store.get(&path).map(|s| s.is_json).unwrap_or(false); - let (resp, outcome) = handle_append_inner(store, req, path).await; + // is_json comes back from the inner handler (false on the not-found path) so + // the metric label doesn't cost a SECOND registry lookup per append — at high + // stream cardinality each lookup is a cold walk of a million-key map. + let (resp, outcome, is_json) = handle_append_inner(store, req, path).await; crate::telemetry::record_append(t0.elapsed_secs(), outcome.label(), is_json); resp } -async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp, AppendOutcome) { +async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp, AppendOutcome, bool) { use AppendOutcome::*; + let st = match store.get(&path) { + Some(s) => s, + None => return (text_response(404, "stream not found"), Conflict, false), + }; + let is_json = st.is_json; macro_rules! ret { ($resp:expr, $oc:expr) => { - return ($resp, $oc) + return ($resp, $oc, is_json) }; } - let st = match store.get(&path) { - Some(s) => s, - None => ret!(text_response(404, "stream not found"), Conflict), - }; if st.shared.read().unwrap().soft_deleted { ret!(gone(), Conflict); } @@ -946,6 +1310,110 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp } }; + // Replicated mode: propose the append into the consensus log and wait for + // the local apply of the decided entry (log-first apply — REPLICATION.md). + // The authoritative closed/producer/seq checks run in the applier + // (`apply_replicated_append`), in log order, deterministically on every + // node; the response is rebuilt here from the apply outcome, mirroring the + // single-node formats below. + if durability() == DurabilityMode::Replicated { + use crate::replication::entry::{AppendApplyOutcome as A, LogOp, OpOutcome, ReplProducer}; + let op = LogOp::Append { + path: path.clone(), + wire: wire.to_vec(), + producer: producer.as_ref().map(|p| ReplProducer { + id: p.id.clone(), + epoch: p.epoch, + seq: p.seq, + }), + seq: seq_header.clone(), + close: close_req, + }; + let outcome = match crate::replication::handle().propose_and_wait(op).await { + Ok(OpOutcome::Append(a)) => a, + Ok(_) => ret!(text_response(500, "unexpected apply outcome"), Conflict), + Err(_) => ret!(text_response(503, "replication timeout — retry"), Conflict), + }; + match outcome { + A::Applied { tail, closed } => { + let status = if producer.is_some() && !body.is_empty() { + 200 + } else { + 204 + }; + let mut b = ResponseBuilder::new(status).h(H_NEXT_OFFSET, format_offset(tail)); + if let Some(p) = &producer { + b = b + .h(H_PRODUCER_EPOCH, p.epoch.to_string()) + .h(H_PRODUCER_SEQ, p.seq.to_string()); + } + if closed { + b = b.hs(H_CLOSED, "true"); + } + ret!(b.body(empty()), Accept); + } + A::NotFound => ret!(text_response(404, "stream not found"), Conflict), + A::Gone => ret!(gone(), Conflict), + A::ClosedDupClose { tail, epoch, seq } => ret!( + ResponseBuilder::new(204) + .hs(H_CLOSED, "true") + .h(H_NEXT_OFFSET, format_offset(tail)) + .h(H_PRODUCER_EPOCH, epoch.to_string()) + .h(H_PRODUCER_SEQ, seq.to_string()) + .body(empty()), + Dup + ), + A::ClosedIdempotent { tail } => ret!( + ResponseBuilder::new(204) + .hs(H_CLOSED, "true") + .h(H_NEXT_OFFSET, format_offset(tail)) + .body(empty()), + Dup + ), + A::Closed { tail } => ret!(closed_conflict(tail), Closed), + A::ProducerDuplicate { + tail, + closed, + epoch, + last_seq, + } => { + let mut b = ResponseBuilder::new(204) + .h(H_NEXT_OFFSET, format_offset(tail)) + .h(H_PRODUCER_EPOCH, epoch.to_string()) + .h(H_PRODUCER_SEQ, last_seq.to_string()); + if closed { + b = b.hs(H_CLOSED, "true"); + } + ret!(b.body(empty()), Dup); + } + A::ProducerStaleEpoch { tail, current } => ret!( + ResponseBuilder::new(403) + .h(H_PRODUCER_EPOCH, current.to_string()) + .h(H_NEXT_OFFSET, format_offset(tail)) + .body(full("stale producer epoch")), + Conflict + ), + A::ProducerGap { expected, received } => ret!( + ResponseBuilder::new(409) + .h(H_PRODUCER_EXPECTED, expected.to_string()) + .h(H_PRODUCER_RECEIVED, received.to_string()) + .body(full("producer sequence gap")), + Conflict + ), + A::ProducerBadEpochStart => ret!( + text_response(400, "new producer epoch must start at seq 0"), + Conflict + ), + A::SeqConflict { tail } => ret!( + ResponseBuilder::new(409) + .h(H_NEXT_OFFSET, format_offset(tail)) + .body(full("Sequence conflict")), + Conflict + ), + A::WriteFailed => ret!(text_response(500, "write failed"), Conflict), + } + } + // Serialize per stream: producer validation + write + state update under one // lock. Time the wait separately — lock contention is a key bottleneck. let lock_t0 = crate::telemetry::Timer::start(); @@ -1156,7 +1624,18 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp st.tail_tx.send_replace(Tail { bytes: tail, closed: true }); #[cfg(target_os = "linux")] crate::sse_reactor::wake_stream(&st); + } else if staged_lsn.is_some() { + // WAL mode: the stream is in its shard's dirty set (register_dirty ran + // during staging), so the ~3 s checkpoint will write the sidecar for us — + // just mark it. This keeps the meta `File::create`+`rename` (and its + // parent-directory rwsem, measured at ~40% of server CPU under write + // saturation) plus a timer task OFF the per-append path. Producer/access + // updates are already documented as a non-durable, lagging flush; the lag + // bound moves from the 100 ms debounce to the checkpoint cadence. + st.meta_dirty.store(true, std::sync::atomic::Ordering::Release); } else { + // No WAL record staged (memory durability): no checkpoint will flush the + // sidecar — keep the debounced flush. st.schedule_meta_flush(); } if !wire.is_empty() { @@ -1178,7 +1657,7 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp if tail.closed { b = b.hs(H_CLOSED, "true"); } - (b.body(empty()), Accept) + (b.body(empty()), Accept, is_json) } fn closed_conflict(tail: u64) -> Resp { @@ -1188,214 +1667,6 @@ fn closed_conflict(tail: u64) -> Resp { .body(full("stream is closed")) } -// ---------- POST (append) — zero-copy socket→file splice path (Linux, --durability memory) ---------- - -/// Result of the zero-copy append entry. `Fallback` means the engine must fall -/// back to the buffered append path (read the whole body, run `handle`): the -/// request is anything but the simple happy-path append (a producer dup/gap/ -/// stale-epoch, a `Stream-Seq` regression, a closed stream, a close request, a -/// missing/mismatched content-type, …). Every such edge case is handled -/// byte-for-byte by the existing buffered handler — the splice path deliberately -/// covers only the accept-and-write case so its critical section is a tight -/// mirror of `write_wire` (memory mode has no WAL stage). -#[cfg(target_os = "linux")] -pub enum ZeroCopyOutcome { - /// Append handled; send `resp` and continue keep-alive as normal. - Done(Resp), - /// A splice leg failed mid-append: some body bytes may have already been - /// consumed from the socket, so the HTTP framing is desynced. Send `resp` - /// (a 500) but then CLOSE the connection — it must NOT be reused as - /// keep-alive (a stale partial body would corrupt the next request). - DoneClose(Resp), - Fallback, -} - -/// Zero-copy binary append (`--durability memory`): relay the request body -/// socket→file via `splice(2)` while holding the appender lock, then ack — the -/// offset/producer semantics are identical to the buffered append, only the byte -/// movement differs (no userspace `Bytes`). There is no WAL in memory mode: the -/// per-stream file write is the ack (durability comes from replication). -/// -/// The caller (engine) has already confirmed: `--durability memory` (the engine -/// zero-copy intercept), `POST`, a known -/// `content_length` (not chunked), and a binary (non-JSON) target stream. -/// `prefix` is the leading body bytes the HTTP parser over-read; `splice_rest` -/// moves the remaining `content_len - prefix.len()` bytes from the socket to the -/// given `(file_fd, offset)` (the engine owns the socket fd + pipe machinery). -/// -/// Returns `Fallback` for anything that is not a plain accept (see -/// `ZeroCopyOutcome`); the engine then reads the body buffered and runs the -/// normal handler. No stream state is mutated before the accept decision, so the -/// fallback re-validates from scratch and is fully idempotent. -/// -/// `splice_rest` moves the remaining `content_len - prefix.len()` body bytes from -/// the socket to `(file_fd, offset)` and is `async`: the socket is non-blocking, -/// so the socket→file splice awaits read-readiness (it may park if the body has -/// not fully arrived). It is awaited here WHILE the appender `AsyncMutex` is held -/// — intended per-stream serialization; holding it across `.await` is supported. -#[cfg(target_os = "linux")] -pub async fn handle_binary_append_zero_copy( - store: Arc, - req: &Req, - prefix: &[u8], - content_len: usize, - splice_rest: F, -) -> ZeroCopyOutcome -where - F: FnOnce(std::os::fd::RawFd, i64) -> Fut, - Fut: std::future::Future>, -{ - use std::os::fd::AsRawFd; - - let path = req.path.clone(); - // Route. A missing/soft-deleted/JSON stream, or one whose content-type does - // not match, is an edge case → buffered fallback. - let st = match store.get(&path) { - Some(s) => s, - None => return ZeroCopyOutcome::Fallback, - }; - if st.is_json || st.shared.read().unwrap().soft_deleted { - return ZeroCopyOutcome::Fallback; - } - // A close request, an empty body, or producer headers that don't parse are - // all handled by the buffered path. Likewise any explicit close. - if header_is_true(req, H_CLOSED) || content_len == 0 { - return ZeroCopyOutcome::Fallback; - } - let producer = match parse_producer_headers(req) { - Ok(p) => p, - Err(_) => return ZeroCopyOutcome::Fallback, - }; - let seq_header = header_str(req, H_SEQ).map(|s| s.to_string()); - // Content-type must be present and match (binary stream): a missing or - // mismatched type is a 4xx the buffered path renders. - match header_str(req, "content-type") { - None => return ZeroCopyOutcome::Fallback, - Some(ct) => { - if media_type(ct) != media_type(&st.config.content_type) { - return ZeroCopyOutcome::Fallback; - } - } - } - - // Serialize per stream — same lock the buffered path holds across the byte - // write. Held across the socket→file splice (the appender's per-stream - // serialization). - let lock_t0 = crate::telemetry::Timer::start(); - let mut ap = st.appender.lock().await; - crate::telemetry::record_append_lock_wait(lock_t0.elapsed_secs()); - - // Re-check closed / producer / seq under the lock. ANY non-accept outcome → - // fall back (the lock is released on drop, no state mutated). - { - let s = st.shared.read().unwrap(); - if s.closed { - return ZeroCopyOutcome::Fallback; - } - } - if let Some(p) = &producer { - let outcome = { - let s = st.shared.read().unwrap(); - validate_producer(&s, p) - }; - if !matches!(outcome, ProducerOutcome::Accept) { - return ZeroCopyOutcome::Fallback; - } - } - if let Some(seq) = &seq_header { - let s = st.shared.read().unwrap(); - if let Some(last) = &s.last_seq_header { - if seq.as_str() <= last.as_str() { - return ZeroCopyOutcome::Fallback; - } - } - } - - // ---- accepted: drive the socket→file splice ---- - // File offset O where this append lands in the stream's own data file. - let file_off = ap.written; - - // Open a fresh non-O_APPEND fd for positioned writes (splice rejects O_APPEND). - let splice_file = match st.open_splice_fd() { - Ok(f) => f, - Err(_) => return ZeroCopyOutcome::Fallback, - }; - let file_fd = splice_file.as_raw_fd(); - - // Write the already-buffered prefix at O (positioned). On failure the - // remaining body is still unconsumed on the socket, so the HTTP framing is - // desynced → force-close (see DoneClose). - if !prefix.is_empty() { - use std::os::unix::fs::FileExt; - if splice_file.write_all_at(prefix, file_off).is_err() { - return ZeroCopyOutcome::DoneClose(text_response(500, "write failed")); - } - } - // Relay the rest socket→file at O+prefix.len() (awaits socket read-readiness - // — the socket is non-blocking). On failure some body bytes may already be - // consumed from the socket → desynced → force-close. - let rest_off = (file_off + prefix.len() as u64) as i64; - if splice_rest(file_fd, rest_off).await.is_err() { - return ZeroCopyOutcome::DoneClose(text_response(500, "write failed")); - } - - // Publish the new logical tail. `ap.written` and `s.tail` advance under the - // lock exactly as in `write_wire`. The tail cache is OFF under - // --durability memory, so we do NOT call set_last_chunk. - ap.written += content_len as u64; - let (tail, closed) = { - let mut s = st.shared.write().unwrap(); - let tail = s.file_base + ap.written; - s.tail = tail; - // memory mode: the page-cache write IS the ack, so the bytes are - // reader-visible immediately (no WAL fsync to wait for). - s.durable_tail = tail; - s.last_access = SystemTime::now(); - (tail, s.closed_durable) - }; - st.tail_tx.send_replace(Tail { bytes: tail, closed }); - #[cfg(target_os = "linux")] - crate::sse_reactor::wake_stream(&st); - - // Shared response builder: captures `st`, `producer` by ref. - let make_ok = || { - let t = st.tail(); - let status = if producer.is_some() { 200 } else { 204 }; - let mut b = ResponseBuilder::new(status).h(H_NEXT_OFFSET, format_offset(t.bytes)); - if let Some(p) = &producer { - b = b - .h(H_PRODUCER_EPOCH, p.epoch.to_string()) - .h(H_PRODUCER_SEQ, p.seq.to_string()); - } - if t.closed { - b = b.hs(H_CLOSED, "true"); - } - ZeroCopyOutcome::Done(b.body(empty())) - }; - - // This path is reached only in `--durability memory` (the engine zero-copy - // intercept is enabled solely by that mode). The per-stream file write IS the - // durable-enough ack (no WAL, no fsync — durability comes from replication). - debug_assert_eq!(durability(), DurabilityMode::Memory); - // Commit producer/seq dedup state under the appender lock so a concurrent - // same-producer request cannot double-accept. - { - let mut s = st.shared.write().unwrap(); - if let Some(p) = &producer { - s.producers.insert( - p.id.clone(), - ProducerState { epoch: p.epoch, last_seq: p.seq }, - ); - } - if let Some(seq) = &seq_header { - s.last_seq_header = Some(seq.clone()); - } - } - drop(ap); // critical section ends - st.schedule_meta_flush(); - maybe_seal_bg(&store, &st); - make_ok() -} // ---------- reading bodies from the data file ---------- @@ -2107,7 +2378,8 @@ fn handle_sse(st: Arc, offset: ParsedOffset, client_cursor: Option< /// Read a logical byte range fully into memory (SSE batches are small). /// Returns `Err` if the range could not be fully materialized (a short local /// read or a cold-storage error/truncation) so callers never advance past a gap. -async fn read_range_bytes( +/// pub(crate): the replication tests use it to verify replica convergence. +pub(crate) async fn read_range_bytes( st: &Arc, start: u64, end: u64, @@ -2162,7 +2434,7 @@ fn handle_head(store: Arc, path: String) -> Resp { // ---------- DELETE ---------- -fn handle_delete(store: Arc, path: String) -> Resp { +async fn handle_delete(store: Arc, path: String) -> Resp { let st = match store.get(&path) { Some(s) => s, None => return text_response(404, "stream not found"), @@ -2170,8 +2442,29 @@ fn handle_delete(store: Arc, path: String) -> Resp { if st.shared.read().unwrap().soft_deleted { return gone(); } - store.delete_or_soft_delete(&st); - ResponseBuilder::new(204).body(empty()) + // Replicated mode: the delete goes through the consensus log; durability of + // the 204 comes from the decided entry, not local fsync (REPLICATION.md). + if durability() == DurabilityMode::Replicated { + use crate::replication::entry::{DeleteApplyOutcome as D, LogOp, OpOutcome}; + let op = LogOp::Delete { path }; + return match crate::replication::handle().propose_and_wait(op).await { + Err(_) => text_response(503, "replication timeout — retry"), + Ok(OpOutcome::Delete(D::Deleted)) => ResponseBuilder::new(204).body(empty()), + Ok(OpOutcome::Delete(D::NotFound)) => text_response(404, "stream not found"), + Ok(OpOutcome::Delete(D::Gone)) => gone(), + Ok(_) => text_response(500, "unexpected apply outcome"), + }; + } + // The 204 is a durability promise: once acked, a crash must never + // resurrect the stream. Await the on-disk removal (unlinks + parent-dir + // fsync, or the soft-delete meta flag) before responding — a detached + // removal task can be lost to a crash after the ack. + let store2 = Arc::clone(&store); + let st2 = Arc::clone(&st); + match tokio::task::spawn_blocking(move || store2.delete_or_soft_delete_durable(&st2)).await { + Ok(Ok(())) => ResponseBuilder::new(204).body(empty()), + _ => text_response(500, "delete not durable"), + } } #[cfg(test)] diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index aa4bc7b748..2bb70b852a 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -3,6 +3,7 @@ mod blobstore; mod engine_raw; mod handlers; mod http1; +mod replication; #[cfg(target_os = "linux")] mod sse_reactor; mod store; @@ -120,10 +121,31 @@ fn main() { // core count; on an existing one reuse the persisted N. A value ≠ the persisted // N is rejected with exit 2 (spec §5). let mut wal_shards: Option = None; + // `--worker-threads N` sizes the tokio runtime's worker-thread pool (and the + // default WAL shard count). `None` ⇒ `available_parallelism()`. This is + // load-bearing under a cgroup cpu limit: `available_parallelism()` reads + // `cpu.max`, so on a big node with a small limit it would under-size the pool; + // an explicit value (e.g. the ds-bench pool suites' `--worker-threads 32`) + // pins the pool to the intended core count regardless. + let mut worker_threads: Option = None; // `--wal-segment-bytes N` overrides the per-shard WAL segment size (the // `fallocate` size + segment-roll threshold). `None` ⇒ the 128 MiB default. // Useful for forcing rolls in tests and benches without writing a full 128 MiB segment. let mut wal_segment_bytes: Option = None; + // `--wal-stats N`: every N seconds print a `WAL_CONT` line of per-interval WAL + // contention rates (lock-wait, wakeup fan-out, coalescing) to stderr, and arm + // the hot-path timing that feeds it. OFF by default (no clock reads on the + // append path). Dependency-free — the measurement vehicle for the contention + // investigation, independent of the heavy `telemetry` OTLP feature. + let mut wal_stats_secs: Option = None; + // `--durability replicated` settings (REPLICATION.md "Configuration"). + let mut repl_id: Option = None; + let mut repl_peers: Option = None; + let mut repl_listen: Option = None; + let mut repl_ack_timeout_ms: u64 = 10_000; + let mut repl_snapshot_logs: u64 = 5000; + let mut repl_stats_secs: u64 = 0; + let mut repl_join = false; let mut args = std::env::args().skip(1); while let Some(a) = args.next() { match a.as_str() { @@ -191,6 +213,14 @@ fn main() { } wal_shards = Some(n); } + "--worker-threads" => { + let n: usize = parse_val(args.next(), "--worker-threads"); + if n == 0 { + eprintln!("--worker-threads must be ≥ 1"); + std::process::exit(2); + } + worker_threads = Some(n); + } "--wal-segment-bytes" => { let n: u64 = parse_val(args.next(), "--wal-segment-bytes"); if n == 0 { @@ -199,12 +229,31 @@ fn main() { } wal_segment_bytes = Some(n); } + "--wal-stats" => { + let n: u64 = parse_val(args.next(), "--wal-stats"); + if n == 0 { + eprintln!("--wal-stats must be ≥ 1 (seconds)"); + std::process::exit(2); + } + wal_stats_secs = Some(n); + } + "--repl-id" => repl_id = Some(parse_val(args.next(), "--repl-id")), + "--repl-peers" => repl_peers = Some(val(args.next(), "--repl-peers")), + "--repl-listen" => repl_listen = Some(val(args.next(), "--repl-listen")), + "--repl-ack-timeout-ms" => { + repl_ack_timeout_ms = parse_val(args.next(), "--repl-ack-timeout-ms") + } + "--repl-snapshot-logs" => { + repl_snapshot_logs = parse_val(args.next(), "--repl-snapshot-logs") + } + "--repl-stats" => repl_stats_secs = parse_val(args.next(), "--repl-stats"), + "--repl-join" => repl_join = true, "--durability" => { let v = val(args.next(), "--durability"); match handlers::parse_durability(&v) { Some(m) => handlers::set_durability(m), None => { - eprintln!("--durability must be wal|memory"); + eprintln!("--durability must be wal|memory|replicated"); std::process::exit(2); } } @@ -216,40 +265,73 @@ fn main() { } } - // Apply --durability memory AFTER the arg loop. On non-Linux, reject immediately. - // On Linux, force the tail cache off (binary appends use the zero-copy socket→file splice). - if handlers::durability() == handlers::DurabilityMode::Memory { - // Fail fast on a WAL left by a previous `--durability wal` run: memory mode - // never opens/replays it, so starting here would silently ignore those + // Apply --durability memory/replicated AFTER the arg loop. Both are the + // buffered append path with the WAL stage/wait skipped (no splice intercept, + // no forced tail-cache-off — those belonged to the removed zero-copy path); + // the only gate is refusing to silently ignore a WAL left by a previous wal run. + if handlers::durability() != handlers::DurabilityMode::Wal { + // Fail fast on a WAL left by a previous `--durability wal` run: these modes + // never open/replay it, so starting here would silently ignore those // records (and drop any not yet folded into the per-stream files). Refuse // rather than diverge quietly; the operator can replay with `--durability // wal` first, or remove the `wal/` directory to discard it deliberately. let wal_dir = data_dir.join("wal"); if wal_dir_has_segments(&wal_dir) { eprintln!( - "error: --durability memory refuses to start: {} holds a WAL from a previous \ - --durability wal run. Memory mode would ignore it and could drop un-checkpointed \ + "error: --durability {} refuses to start: {} holds a WAL from a previous \ + --durability wal run. This mode would ignore it and could drop un-checkpointed \ records. Replay it first with --durability wal, or remove {} to discard it.", + if handlers::durability() == handlers::DurabilityMode::Memory { + "memory" + } else { + "replicated" + }, wal_dir.display(), wal_dir.display() ); std::process::exit(2); } - #[cfg(not(target_os = "linux"))] - { - eprintln!("--durability memory is Linux-only (zero-copy socket→file)"); - std::process::exit(2); - } - #[cfg(target_os = "linux")] - { - if store::tail_cache_bytes() != 0 { - eprintln!("--durability memory disables the resident tail cache"); - } - store::set_tail_cache_bytes(0); - engine_raw::set_zero_copy(true); // binary appends take the splice intercept - } } + // Validate the replication flags before the runtime starts, so a bad config + // is a clean exit 2 (usage error), not a panic mid-boot. + let repl_config: Option = + if handlers::durability() == handlers::DurabilityMode::Replicated { + let (Some(id), Some(peers_raw)) = (repl_id, &repl_peers) else { + eprintln!("error: --durability replicated requires --repl-id and --repl-peers"); + std::process::exit(2); + }; + let peers = replication::ReplConfig::parse_peers(peers_raw).unwrap_or_else(|e| { + eprintln!("error: --repl-peers: {e}"); + std::process::exit(2); + }); + let mut cfg = replication::ReplConfig { + id, + peers, + listen: String::new(), + ack_timeout: std::time::Duration::from_millis(repl_ack_timeout_ms), + snapshot_logs: repl_snapshot_logs, + stats_secs: repl_stats_secs, + join: repl_join, + // Durable vote: election-safety across restarts (REPLICATION.md). + vote_path: Some(data_dir.join("repl.vote")), + }; + cfg.listen = match repl_listen { + Some(l) => l, + None => cfg.default_listen().unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(2); + }), + }; + Some(cfg) + } else { + if repl_id.is_some() || repl_peers.is_some() { + eprintln!("error: --repl-* flags require --durability replicated"); + std::process::exit(2); + } + None + }; + // S3 credentials come from env (never CLI flags), matching the OTEL_*/AWS // convention. Honour both the DS_* names and the standard AWS_* fallbacks. if tier.kind == tier::TierKind::S3 { @@ -261,9 +343,9 @@ fn main() { .ok(); } - let workers = std::thread::available_parallelism() + let workers = worker_threads.unwrap_or_else(|| std::thread::available_parallelism() .map(|n| n.get()) - .unwrap_or(4); + .unwrap_or(4)); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(workers) .enable_all() @@ -307,6 +389,10 @@ fn main() { // per-shard committers, and start the checkpoint ticker. No append can // run before this point (we have not begun serving yet), so no durable // record is lost and no new append collides with un-recovered WAL data. + // Held so the shutdown path can stop + join the dedicated committer + // threads (Tier-2a) after draining in-flight requests. `None` in + // `--durability memory` mode (no committers spawned). + let mut wal_for_shutdown: Option> = None; if handlers::durability() == handlers::DurabilityMode::Wal { let open_res = match wal_segment_bytes { Some(sz) => wal::walset::WalSet::open_with_segment_size( @@ -326,6 +412,17 @@ fn main() { .wal .set(Arc::clone(&walset)) .unwrap_or_else(|_| panic!("WAL already attached")); + // Arm the contention timing + spawn the dependency-free stderr + // emitter BEFORE committers/serving start, so every acquisition from + // the first append is timed. No-op (and no clock reads) when the flag + // is absent. + if let Some(secs) = wal_stats_secs { + wal::telemetry::set_stats_enabled(true); + wal::telemetry::spawn_stats_emitter( + Arc::clone(&walset), + std::time::Duration::from_secs(secs), + ); + } walset.spawn_committers(); // Per-shard checkpoint ticker (spec §7): periodically `fdatasync` each // shard's touched per-stream files and recycle its WAL below the @@ -336,6 +433,40 @@ fn main() { // distribution + durability gauges. No-op unless built with // `--features telemetry`; off the hot commit/append path. wal::telemetry::spawn_emitter(Arc::clone(&walset)); + wal_for_shutdown = Some(walset); + } + + // ---- replication wiring (Replicated mode only) ---- + // Start the peer mesh + consensus core BEFORE serving so no append can + // race the OnceLock install. Serving does not wait for an elected + // leader; readiness is `/_repl/status` reporting a leader. + if let Some(cfg) = &repl_config { + // The consensus log/snapshot is the source of truth in this mode: + // stale local stream files from a previous run must not + // double-apply under log replay — wipe them and let Raft resync + // this node (snapshot manifest + byte fetch + log replay). The + // durable vote file is intentionally preserved. + let n_stale = store.streams.len(); + if n_stale > 0 { + store.wipe_all().expect("failed to clear stale streams"); + println!( + "replication: cleared {n_stale} stale local stream(s); \ + state will resync from the cluster" + ); + } + let h = replication::start(Arc::clone(&store), cfg) + .await + .unwrap_or_else(|e| { + eprintln!("error: replication listener bind {} failed: {e}", cfg.listen); + std::process::exit(2); + }); + replication::install(h); + println!( + "replication: node {} of {:?}, peer listener on {}", + cfg.id, + cfg.peers.iter().map(|(id, _)| *id).collect::>(), + cfg.listen + ); } let addr: SocketAddr = (host, port).into(); @@ -355,6 +486,12 @@ fn main() { #[cfg(target_os = "linux")] sse_reactor::shutdown(); engine_raw::drain(std::time::Duration::from_secs(25)).await; + // Stop + join the dedicated committer threads (Tier-2a) AFTER the + // request drain, so any commit a just-drained request staged is + // covered by each committer's final drain before the thread exits. + if let Some(walset) = wal_for_shutdown.take() { + walset.stop_committers(); + } telemetry_guard.shutdown(); } } @@ -382,11 +519,22 @@ fn spawn_checkpoint_ticker(walset: Arc) { ticker.tick().await; loop { ticker.tick().await; + // All shards checkpoint CONCURRENTLY. Each checkpoint is one + // spawn_blocking task (capture + per-stream fdatasyncs + tails/ckpt + // persist + recycle), so a serial walk makes every per-stream fsync + // across the whole server queue behind a single shard's — at high + // stream cardinality that serialization is what stretches the + // checkpoint wave (and on real disks wastes the device's parallelism). + let mut wave = tokio::task::JoinSet::new(); for shard in walset.shards() { - if let Err(e) = shard.checkpoint().await { - eprintln!("WAL checkpoint failed for shard {:?}: {e}", shard.dir()); - } + let shard = Arc::clone(shard); + wave.spawn(async move { + if let Err(e) = shard.checkpoint().await { + eprintln!("WAL checkpoint failed for shard {:?}: {e}", shard.dir()); + } + }); } + while wave.join_next().await.is_some() {} } }); } diff --git a/packages/durable-streams-rust/src/replication/core.rs b/packages/durable-streams-rust/src/replication/core.rs new file mode 100644 index 0000000000..a5da192e7f --- /dev/null +++ b/packages/durable-streams-rust/src/replication/core.rs @@ -0,0 +1,197 @@ +//! The replication handle: proposes ops through openraft and returns the +//! state-machine apply outcome as the ack (log-first apply — REPLICATION.md). +//! +//! `Raft::client_write` resolves with the outcome the state machine computed +//! when it applied the entry, so there is no hand-rolled pending-ack map. On +//! a non-leader node the proposal is forwarded to the leader over the RPC +//! mesh; the origin then waits until its OWN applied index covers the entry, +//! preserving read-your-writes on the node that took the HTTP request. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use super::entry::{LogOp, OpOutcome}; +use super::log_store::LogStore; +use super::net::{Clients, RpcReply, RpcRequest}; +use super::sm::StateMachine; +use super::types::{typ, NodeId, Raft, TypeConfig}; + +/// Propose failed to produce an outcome within the ack timeout (no leader, +/// dropped forward, or a stalled quorum). Safe to retry — producer dedup +/// absorbs a duplicate that did land. +#[derive(Debug)] +pub struct AckTimeout; + +pub struct ReplHandle { + pub node_id: NodeId, + pub raft: Raft, + pub(super) sm: Arc, + pub(super) log_store: LogStore, + pub(super) clients: Arc, + ack_timeout: Duration, + /// Ops proposed by THIS node (cumulative). + pub proposed: AtomicU64, + /// Proposals forwarded to a remote leader (cumulative). + pub forwarded: AtomicU64, + /// Proposals that failed to resolve within the ack timeout (cumulative). + pub timeouts: AtomicU64, +} + +impl ReplHandle { + pub(super) fn new( + node_id: NodeId, + raft: Raft, + sm: Arc, + log_store: LogStore, + clients: Arc, + ack_timeout: Duration, + ) -> Arc { + Arc::new(ReplHandle { + node_id, + raft, + sm, + log_store, + clients, + ack_timeout, + proposed: AtomicU64::new(0), + forwarded: AtomicU64::new(0), + timeouts: AtomicU64::new(0), + }) + } + + pub async fn propose_and_wait(&self, op: LogOp) -> Result { + self.proposed.fetch_add(1, Ordering::Relaxed); + let write = self.raft.client_write(op.clone()); + match tokio::time::timeout(self.ack_timeout, write).await { + Ok(Ok(resp)) => Ok(resp.data), + Ok(Err(typ::RaftError::APIError(typ::ClientWriteError::ForwardToLeader(fw)))) => { + self.forward_to_leader(op, fw.leader_id).await + } + _ => { + self.timeouts.fetch_add(1, Ordering::Relaxed); + Err(AckTimeout) + } + } + } + + async fn forward_to_leader( + &self, + op: LogOp, + leader: Option, + ) -> Result { + let Some(leader) = leader else { + // No known leader (election in progress) — the client retries. + self.timeouts.fetch_add(1, Ordering::Relaxed); + return Err(AckTimeout); + }; + // Resolve the leader's mesh client — from the registry, or lazily + // from the current membership (handles freshly added nodes). + let client = match self.clients.get(leader) { + Some(c) => c, + None => { + let m = self.raft.metrics().borrow().clone(); + let addr = m + .membership_config + .membership() + .nodes() + .find(|(id, _)| **id == leader) + .map(|(_, n)| n.addr.clone()); + match addr { + Some(a) => self.clients.get_or_create(leader, &a), + None => { + self.timeouts.fetch_add(1, Ordering::Relaxed); + return Err(AckTimeout); + } + } + } + }; + self.forwarded.fetch_add(1, Ordering::Relaxed); + let call = client.call(RpcRequest::Forward(op)); + match tokio::time::timeout(self.ack_timeout, call).await { + Ok(Ok(RpcReply::Forward(Ok((outcome, log_index))))) => { + // Read-your-writes on THIS node: wait until the local state + // machine has applied at least the acked entry. + let _ = self + .raft + .wait(Some(self.ack_timeout)) + .metrics( + move |m| { + m.last_applied.map(|l| l.index >= log_index).unwrap_or(false) + }, + "local apply of forwarded write", + ) + .await; + Ok(outcome) + } + _ => { + self.timeouts.fetch_add(1, Ordering::Relaxed); + Err(AckTimeout) + } + } + } + + pub fn status_json(&self) -> String { + let m = self.raft.metrics().borrow().clone(); + let mut peers: Vec = self + .clients + .connected_ids() + .into_iter() + .map(|id| id.to_string()) + .collect(); + peers.sort(); + let applied = m.last_applied.map(|l| l.index).unwrap_or(0); + let last_log = m.last_log_index.unwrap_or(0); + let purged = m.purged.map(|l| l.index).unwrap_or(0); + format!( + "{{\"id\":{},\"leader\":{},\"decided_idx\":{},\"log_window\":{},\"pending\":{},\"timeouts\":{},\"connected_peers\":[{}]}}", + self.node_id, + m.current_leader.map(|l| l.to_string()).unwrap_or_else(|| "null".to_string()), + applied, + last_log.saturating_sub(purged), + last_log.saturating_sub(applied), + self.timeouts.load(Ordering::Relaxed), + peers.join(",") + ) + } +} + +/// `--repl-stats N`: every N seconds print a one-line `REPL_STATS` snapshot +/// to stderr — same shape as the omnipaxos incarnation so the monitoring +/// tooling and mental models carry over. +pub(super) fn spawn_stats_emitter(handle: Arc, every: Duration) { + tokio::spawn(async move { + let mut tick = tokio::time::interval(every); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + tick.tick().await; + let (mut p0, mut a0, mut f0) = (0u64, 0u64, 0u64); + loop { + tick.tick().await; + let m = handle.raft.metrics().borrow().clone(); + let p = handle.proposed.load(Ordering::Relaxed); + let f = handle.forwarded.load(Ordering::Relaxed); + let a = handle + .sm + .applied_ops + .load(std::sync::atomic::Ordering::Relaxed); + let (window, _) = handle.log_store.window().await; + let secs = every.as_secs_f64(); + eprintln!( + "REPL_STATS node={} leader={} proposed/s={:.0} forwarded/s={:.0} applied/s={:.0} window={} applied_idx={} timeouts={} apply_max_us={}", + handle.node_id, + m.current_leader.unwrap_or(0), + (p - p0) as f64 / secs, + (f - f0) as f64 / secs, + (a - a0) as f64 / secs, + window, + m.last_applied.map(|l| l.index).unwrap_or(0), + handle.timeouts.load(Ordering::Relaxed), + handle + .sm + .apply_max_us + .swap(0, std::sync::atomic::Ordering::Relaxed), + ); + (p0, a0, f0) = (p, a, f); + } + }); +} diff --git a/packages/durable-streams-rust/src/replication/entry.rs b/packages/durable-streams-rust/src/replication/entry.rs new file mode 100644 index 0000000000..157474be0f --- /dev/null +++ b/packages/durable-streams-rust/src/replication/entry.rs @@ -0,0 +1,109 @@ +//! The replicated-log operation types (what goes through consensus) and the +//! apply outcomes (what comes back). See REPLICATION.md. +//! +//! With openraft, `LogOp` is the Raft `D` (app data) type and `OpOutcome` the +//! `R` (response) type: `Raft::client_write(LogOp)` resolves with the outcome +//! the state machine computed when it applied the decided entry — the ack IS +//! the apply result. Outcomes are serde because they also travel back over +//! the forward-to-leader RPC. + +use serde::{Deserialize, Serialize}; + +use crate::store::StreamConfig; + +/// Producer identity for idempotent appends, replicated so dedup/fencing is +/// evaluated deterministically at apply time on every node. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReplProducer { + pub id: String, + pub epoch: u64, + pub seq: u64, +} + +/// A state-mutating operation. Everything that changes stream state goes +/// through the log; reads never do. Wire bytes are already encoded +/// (`encode_wire` ran on the proposing node), so apply is a byte append. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum LogOp { + /// `PUT /s` — create (including fork; the fork point is pre-resolved into + /// `base_offset` by the proposing node so apply does no read I/O). + Create { + path: String, + config: StreamConfig, + base_offset: u64, + /// Optional initial body (already wire-encoded). + wire: Vec, + }, + /// `POST /s` — append and/or close. + Append { + path: String, + wire: Vec, + producer: Option, + seq: Option, + close: bool, + }, + /// `DELETE /s`. + Delete { path: String }, +} + +impl LogOp { + pub fn path(&self) -> &str { + match self { + LogOp::Create { path, .. } | LogOp::Append { path, .. } | LogOp::Delete { path } => { + path + } + } + } +} + +// ---------- apply outcomes ---------- + +/// Outcome of applying a decided `LogOp`, returned to the proposing client. +/// Mirrors the single-node handlers' response cases. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum OpOutcome { + Append(AppendApplyOutcome), + Create(CreateApplyOutcome), + Delete(DeleteApplyOutcome), + /// Raft-internal entries (blank on leader change, membership) — never + /// surfaced to an HTTP client. + Noop, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppendApplyOutcome { + /// Bytes written (and/or closed) — the success ack. + Applied { tail: u64, closed: bool }, + /// Stream vanished between propose and apply. + NotFound, + Gone, + /// Producer retried the exact close that closed the stream → idempotent 204. + ClosedDupClose { tail: u64, epoch: u64, seq: u64 }, + /// Bare close of an already-closed stream → idempotent 204. + ClosedIdempotent { tail: u64 }, + /// Append to a closed stream → 409. + Closed { tail: u64 }, + ProducerDuplicate { tail: u64, closed: bool, epoch: u64, last_seq: u64 }, + ProducerStaleEpoch { tail: u64, current: u64 }, + ProducerGap { expected: u64, received: u64 }, + ProducerBadEpochStart, + SeqConflict { tail: u64 }, + WriteFailed, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum CreateApplyOutcome { + Created { tail: u64, closed: bool }, + Exists { tail: u64, closed: bool, content_type: String }, + Conflict, + /// Fork parent vanished between propose and apply. + ForkSourceMissing, + WriteFailed, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DeleteApplyOutcome { + Deleted, + NotFound, + Gone, +} diff --git a/packages/durable-streams-rust/src/replication/log_store.rs b/packages/durable-streams-rust/src/replication/log_store.rs new file mode 100644 index 0000000000..53f93167fc --- /dev/null +++ b/packages/durable-streams-rust/src/replication/log_store.rs @@ -0,0 +1,203 @@ +//! In-memory openraft log store (log entries + vote). Deliberate: durability +//! comes from quorum replication, not disk (REPLICATION.md "Storage"). +//! Vendored from openraft's `examples/memstore` (MIT/Apache-2.0) — the +//! reference in-memory `RaftLogStorage` — to avoid an examples-only dep. +//! +//! The VOTE, however, is durable when a `vote_path` is configured (it is in +//! production boots): it changes only at election time — never on the append +//! hot path — and persisting it is what makes restart-rejoin safe (a node +//! that forgot its vote could grant conflicting elections). + +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::ops::RangeBounds; +use std::sync::Arc; + +use openraft::storage::{LogFlushed, RaftLogStorage}; +use openraft::{LogId, LogState, RaftLogId, RaftLogReader, RaftTypeConfig, StorageError, Vote}; +use tokio::sync::Mutex; + +#[derive(Clone, Debug, Default)] +pub struct LogStore { + inner: Arc>>, + /// When set, the vote is fsynced here on every change and reloaded at boot. + vote_path: Option, +} + +#[derive(Debug)] +pub struct LogStoreInner { + last_purged_log_id: Option>, + log: BTreeMap, + committed: Option>, + vote: Option>, +} + +impl Default for LogStoreInner { + fn default() -> Self { + Self { + last_purged_log_id: None, + log: BTreeMap::new(), + committed: None, + vote: None, + } + } +} + +impl LogStore +where + Vote: serde::Serialize + serde::de::DeserializeOwned, +{ + /// A log store with a durable vote file (loaded now if present). + pub fn with_vote_path(path: std::path::PathBuf) -> std::io::Result { + let mut inner = LogStoreInner::::default(); + match std::fs::read(&path) { + Ok(bytes) => { + inner.vote = Some(bincode::deserialize(&bytes).map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("corrupt vote file {}: {e}", path.display()), + ) + })?); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + Ok(LogStore { + inner: Arc::new(Mutex::new(inner)), + vote_path: Some(path), + }) + } +} + +impl LogStore { + /// (window_entries, last_purged_index) for REPL_STATS. + pub async fn window(&self) -> (usize, u64) { + let inner = self.inner.lock().await; + ( + inner.log.len(), + inner.last_purged_log_id.as_ref().map(|l| l.index).unwrap_or(0), + ) + } +} + +impl RaftLogReader for LogStore +where + C::Entry: Clone, +{ + async fn try_get_log_entries + Clone + Debug>( + &mut self, + range: RB, + ) -> Result, StorageError> { + let inner = self.inner.lock().await; + Ok(inner.log.range(range).map(|(_, val)| val.clone()).collect()) + } +} + +impl RaftLogStorage for LogStore +where + C::Entry: Clone, +{ + type LogReader = Self; + + async fn get_log_state(&mut self) -> Result, StorageError> { + let inner = self.inner.lock().await; + let last = inner.log.iter().next_back().map(|(_, ent)| ent.get_log_id().clone()); + let last_purged = inner.last_purged_log_id.clone(); + Ok(LogState { + last_purged_log_id: last_purged.clone(), + last_log_id: last.or(last_purged), + }) + } + + async fn save_committed( + &mut self, + committed: Option>, + ) -> Result<(), StorageError> { + self.inner.lock().await.committed = committed; + Ok(()) + } + + async fn read_committed( + &mut self, + ) -> Result>, StorageError> { + Ok(self.inner.lock().await.committed.clone()) + } + + async fn save_vote(&mut self, vote: &Vote) -> Result<(), StorageError> { + // Durable BEFORE visible: a vote acknowledged and then forgotten can + // grant two conflicting elections. Election-time only — never on the + // append hot path. + if let Some(path) = &self.vote_path { + let write = || -> std::io::Result<()> { + let bytes = bincode::serialize(vote) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let tmp = path.with_extension("tmp"); + { + use std::io::Write; + let mut f = std::fs::File::create(&tmp)?; + f.write_all(&bytes)?; + f.sync_all()?; + } + std::fs::rename(&tmp, path)?; + Ok(()) + }; + let path2 = path.clone(); + write().map_err(|e| { + StorageError::from(openraft::StorageIOError::write_vote( + &std::io::Error::new( + e.kind(), + format!("vote file {}: {e}", path2.display()), + ), + )) + })?; + } + self.inner.lock().await.vote = Some(vote.clone()); + Ok(()) + } + + async fn read_vote(&mut self) -> Result>, StorageError> { + Ok(self.inner.lock().await.vote.clone()) + } + + async fn append( + &mut self, + entries: I, + callback: LogFlushed, + ) -> Result<(), StorageError> + where + I: IntoIterator, + { + let mut inner = self.inner.lock().await; + for entry in entries { + inner.log.insert(entry.get_log_id().index, entry); + } + // In-memory: the "flush" is the insert itself. + callback.log_io_completed(Ok(())); + Ok(()) + } + + async fn truncate(&mut self, log_id: LogId) -> Result<(), StorageError> { + let mut inner = self.inner.lock().await; + let keys: Vec = inner.log.range(log_id.index..).map(|(k, _)| *k).collect(); + for key in keys { + inner.log.remove(&key); + } + Ok(()) + } + + async fn purge(&mut self, log_id: LogId) -> Result<(), StorageError> { + let mut inner = self.inner.lock().await; + assert!(inner.last_purged_log_id.as_ref() <= Some(&log_id)); + let purge_idx = log_id.index; + inner.last_purged_log_id = Some(log_id); + let keys: Vec = inner.log.range(..=purge_idx).map(|(k, _)| *k).collect(); + for key in keys { + inner.log.remove(&key); + } + Ok(()) + } + + async fn get_log_reader(&mut self) -> Self::LogReader { + self.clone() + } +} diff --git a/packages/durable-streams-rust/src/replication/mod.rs b/packages/durable-streams-rust/src/replication/mod.rs new file mode 100644 index 0000000000..b704818201 --- /dev/null +++ b/packages/durable-streams-rust/src/replication/mod.rs @@ -0,0 +1,189 @@ +//! Raft-replicated durability (`--durability replicated`) — see REPLICATION.md. +//! +//! Layering: +//! - `entry` — what goes through the log (`LogOp`) and apply outcomes +//! - `types` — openraft type wiring (D = LogOp, R = OpOutcome) +//! - `log_store` — in-memory Raft log + vote (vendored reference impl) +//! - `sm` — the state machine: sharded log-first apply into the Store +//! - `net` — RPC mesh (openraft RPCs + forward-to-leader proposals) +//! - `core` — ReplHandle: propose/ack, status, stats +//! - the semantic apply functions (`apply_replicated_*`) live in `handlers.rs`, +//! next to their single-node twins +//! +//! `main.rs` calls `start()` then `install()`; the HTTP handlers reach the +//! running instance through `handle()`. + +pub mod core; +pub mod entry; +mod log_store; +mod net; +mod sm; +pub mod types; + +#[cfg(test)] +mod tests; + +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +pub use core::ReplHandle; +use openraft::BasicNode; + +use crate::store::Store; + +/// Replication settings, parsed from `--repl-*` flags (see REPLICATION.md +/// "Configuration"). +#[derive(Clone, Debug)] +pub struct ReplConfig { + /// This node's id (must appear in `peers`). + pub id: u64, + /// Full membership, `(id, host:port)` of every node's replication listener + /// — including this node (that entry provides our default listen port). + pub peers: Vec<(u64, String)>, + /// Peer-listener bind address. + pub listen: String, + /// How long `propose_and_wait` waits for the acked outcome before 503. + pub ack_timeout: Duration, + /// Snapshot (log-purge) cadence in log entries: a manifest snapshot is + /// taken every N entries and the log behind it is purged, bounding the + /// in-memory log to roughly N + keep-margin entries per node. + pub snapshot_logs: u64, + /// `REPL_STATS` stderr emit cadence in seconds (0 = off). + pub stats_secs: u64, + /// Joining an EXISTING cluster (`--repl-join`): skip the initial-membership + /// bootstrap and wait to be added via `/_repl/add-learner` + + /// `/_repl/change-membership` — the leader ships a snapshot + log. + pub join: bool, + /// When set, the Raft vote is fsynced here (and reloaded at boot), making + /// restart-rejoin election-safe. Production boots set it; tests may not. + pub vote_path: Option, +} + +impl ReplConfig { + /// Parse `1@host:port,2@host:port,…` (the `--repl-peers` value). + pub fn parse_peers(s: &str) -> Result, String> { + let mut peers = Vec::new(); + for part in s.split(',') { + let (id, addr) = part + .split_once('@') + .ok_or_else(|| format!("bad peer {part:?}: want id@host:port"))?; + let id: u64 = id + .parse() + .map_err(|_| format!("bad peer id in {part:?}"))?; + if id == 0 { + return Err("peer ids are 1-based (0 is reserved)".into()); + } + if addr.rsplit_once(':').is_none() { + return Err(format!("bad peer address {addr:?}: want host:port")); + } + peers.push((id, addr.to_string())); + } + if peers.len() < 2 { + return Err("--repl-peers needs at least 2 nodes".into()); + } + Ok(peers) + } + + /// The default listen address: `0.0.0.0:`. + pub fn default_listen(&self) -> Result { + let (_, addr) = self + .peers + .iter() + .find(|(id, _)| *id == self.id) + .ok_or_else(|| format!("--repl-id {} not present in --repl-peers", self.id))?; + let (_, port) = addr.rsplit_once(':').expect("validated in parse_peers"); + Ok(format!("0.0.0.0:{port}")) + } +} + +/// Bind the peer listener and start the Raft node. Separated from `install()` +/// so tests can run several nodes in one process. +pub async fn start(store: Arc, cfg: &ReplConfig) -> std::io::Result> { + let listener = tokio::net::TcpListener::bind(&cfg.listen).await?; + Ok(start_with_listener(store, cfg, listener).await) +} + +pub async fn start_with_listener( + store: Arc, + cfg: &ReplConfig, + listener: tokio::net::TcpListener, +) -> Arc { + // Election/heartbeat sized to match the omnipaxos incarnation (~500 ms + // fail-over) so the comparison and the documented behavior carry over. + let config = openraft::Config { + heartbeat_interval: 100, + election_timeout_min: 500, + election_timeout_max: 1000, + snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(cfg.snapshot_logs.max(1)), + max_in_snapshot_log_to_keep: 256, + ..Default::default() + }; + let config = Arc::new(config.validate().expect("invalid raft config")); + + let log_store = match &cfg.vote_path { + Some(p) => log_store::LogStore::::with_vote_path(p.clone()) + .expect("failed to load vote file"), + None => log_store::LogStore::::default(), + }; + let sm = sm::StateMachine::new(Arc::clone(&store)); + sm.spawn_meta_sweeper(); + + let clients = Arc::new(net::Clients::default()); + for (id, addr) in cfg.peers.iter().filter(|(id, _)| *id != cfg.id) { + clients.get_or_create(*id, addr); + } + + let raft = types::Raft::new( + cfg.id, + config, + net::MeshFactory { + clients: Arc::clone(&clients), + }, + log_store.clone(), + Arc::clone(&sm), + ) + .await + .expect("failed to start raft"); + + // Wire the snapshot-install fetch path (leader lookup + mesh clients). + let _ = sm.install_ctx.set(sm::InstallCtx { + self_id: cfg.id, + clients: Arc::clone(&clients), + metrics: raft.metrics(), + }); + + net::spawn_server(listener, raft.clone(), Arc::clone(&store)); + + if !cfg.join { + // Bootstrap: every founding node proposes the same initial membership; + // on an already-initialized node this errs (NotAllowed) and is ignored. + let members: BTreeMap = cfg + .peers + .iter() + .map(|(id, addr)| (*id, BasicNode::new(addr.clone()))) + .collect(); + let _ = raft.initialize(members).await; + } + + let handle = ReplHandle::new(cfg.id, raft, sm, log_store, clients, cfg.ack_timeout); + if cfg.stats_secs > 0 { + core::spawn_stats_emitter( + Arc::clone(&handle), + std::time::Duration::from_secs(cfg.stats_secs), + ); + } + handle +} + +// The process-wide instance the HTTP handlers use. Set once by main; tests +// drive their multi-node clusters through the returned handles instead. +static HANDLE: OnceLock> = OnceLock::new(); + +pub fn install(h: Arc) { + HANDLE.set(h).ok().expect("replication already installed"); +} + +pub fn handle() -> &'static Arc { + HANDLE.get().expect("replication not started") +} diff --git a/packages/durable-streams-rust/src/replication/net.rs b/packages/durable-streams-rust/src/replication/net.rs new file mode 100644 index 0000000000..f074341351 --- /dev/null +++ b/packages/durable-streams-rust/src/replication/net.rs @@ -0,0 +1,420 @@ +//! RPC mesh between replicas: one persistent outbound connection per peer +//! with correlation ids and automatic reconnect; frames are +//! `u32-BE length + bincode((req_id, payload))`. Carries openraft's three +//! RPCs plus our forward-to-leader proposal RPC. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use openraft::error::{InstallSnapshotError, NetworkError, Unreachable}; +use openraft::network::{RPCOption, RaftNetwork, RaftNetworkFactory}; +use openraft::raft::{ + AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, + VoteRequest, VoteResponse, +}; +use openraft::BasicNode; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{tcp::OwnedWriteHalf, TcpListener, TcpStream}; +use tokio::sync::{mpsc, oneshot, Mutex}; + +use super::entry::{LogOp, OpOutcome}; +use super::types::{typ, NodeId, Raft, TypeConfig}; +use crate::store::Store; + +const MAX_FRAME: u32 = 256 * 1024 * 1024; +const RECONNECT_BACKOFF: Duration = Duration::from_millis(500); +/// Cap on any single RPC round trip (openraft also applies its own ttls). +const RPC_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Serialize, Deserialize)] +pub(super) enum RpcRequest { + Append(AppendEntriesRequest), + Vote(VoteRequest), + Snapshot(InstallSnapshotRequest), + /// Proposal forwarded from a non-leader node: the leader runs + /// `client_write` and returns the apply outcome + log index. + Forward(LogOp), + /// Snapshot-install byte pull: the immutable prefix `[from, to)` of a + /// stream (verified by `id` against delete+recreate races). + FetchStream { + path: String, + id: u64, + from: u64, + to: u64, + }, +} + +#[derive(Serialize, Deserialize)] +pub(super) enum RpcReply { + Append(Result, String>), + Vote(Result, String>), + Snapshot(Result, String>), + Forward(Result<(OpOutcome, u64), String>), + FetchStream(Result, String>), +} + +// ---------- client ---------- + +struct Conn { + tx: mpsc::Sender>, + pending: Arc>>, +} + +/// A lazily-connected, auto-reconnecting RPC client for one peer. +pub struct RpcClient { + pub addr: String, + conn: Mutex>, + next_id: AtomicU64, + pub connected: AtomicBool, +} + +impl RpcClient { + pub fn new(addr: String) -> Arc { + Arc::new(RpcClient { + addr, + conn: Mutex::new(None), + next_id: AtomicU64::new(1), + connected: AtomicBool::new(false), + }) + } + + async fn ensure_conn(&self) -> std::io::Result<(mpsc::Sender>, Arc>>)> + { + let mut guard = self.conn.lock().await; + if let Some(c) = guard.as_ref() { + if !c.tx.is_closed() { + return Ok((c.tx.clone(), Arc::clone(&c.pending))); + } + } + let sock = TcpStream::connect(&self.addr).await?; + let _ = sock.set_nodelay(true); + let (mut rd, wr) = sock.into_split(); + let (tx, rx) = mpsc::channel::>(1024); + let pending: Arc>> = + Arc::new(dashmap::DashMap::new()); + tokio::spawn(write_loop(wr, rx)); + let pending2 = Arc::clone(&pending); + tokio::spawn(async move { + // Reader: match replies to pending calls; on EOF/err fail them all + // (their oneshot senders drop → callers see a network error). + let mut len_buf = [0u8; 4]; + loop { + if rd.read_exact(&mut len_buf).await.is_err() { + break; + } + let len = u32::from_be_bytes(len_buf); + if len == 0 || len > MAX_FRAME { + break; + } + let mut buf = vec![0u8; len as usize]; + if rd.read_exact(&mut buf).await.is_err() { + break; + } + let Ok((req_id, reply)) = bincode::deserialize::<(u64, RpcReply)>(&buf) else { + break; + }; + if let Some((_, tx)) = pending2.remove(&req_id) { + let _ = tx.send(reply); + } + } + pending2.clear(); + }); + *guard = Some(Conn { + tx: tx.clone(), + pending: Arc::clone(&pending), + }); + self.connected.store(true, Ordering::Relaxed); + Ok((tx, pending)) + } + + pub async fn call(&self, req: RpcRequest) -> Result { + let (tx, pending) = match self.ensure_conn().await { + Ok(v) => v, + Err(e) => { + self.connected.store(false, Ordering::Relaxed); + return Err(e); + } + }; + let req_id = self.next_id.fetch_add(1, Ordering::Relaxed); + let (otx, orx) = oneshot::channel(); + pending.insert(req_id, otx); + let mut frame = Vec::new(); + let body = bincode::serialize(&(req_id, req)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + frame.extend_from_slice(&(body.len() as u32).to_be_bytes()); + frame.extend_from_slice(&body); + if tx.send(frame).await.is_err() { + pending.remove(&req_id); + self.connected.store(false, Ordering::Relaxed); + *self.conn.lock().await = None; + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "peer connection lost", + )); + } + match tokio::time::timeout(RPC_TIMEOUT, orx).await { + Ok(Ok(reply)) => Ok(reply), + _ => { + pending.remove(&req_id); + self.connected.store(false, Ordering::Relaxed); + *self.conn.lock().await = None; + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "rpc timeout / connection lost", + )) + } + } + } +} + +async fn write_loop(mut wr: OwnedWriteHalf, mut rx: mpsc::Receiver>) { + while let Some(frame) = rx.recv().await { + if wr.write_all(&frame).await.is_err() { + return; // dropping rx closes the channel; callers reconnect + } + } +} + +/// Fetch a stream's byte range for snapshot install. +impl RpcClient { + pub async fn fetch_stream( + &self, + path: &str, + id: u64, + from: u64, + to: u64, + ) -> Result, String> { + match self + .call(RpcRequest::FetchStream { + path: path.to_string(), + id, + from, + to, + }) + .await + { + Ok(RpcReply::FetchStream(r)) => r, + Ok(_) => Err("mismatched rpc reply".into()), + Err(e) => Err(e.to_string()), + } + } +} + +// ---------- dynamic client registry ---------- + +/// Per-peer RPC clients, keyed by node id and created lazily from membership +/// addresses — so membership changes (added learners/voters) get mesh links +/// without a restart. +#[derive(Default)] +pub struct Clients { + map: dashmap::DashMap>, +} + +impl Clients { + pub fn get_or_create(&self, id: NodeId, addr: &str) -> Arc { + // Replace the client if the node re-registered under a new address. + if let Some(c) = self.map.get(&id) { + if c.addr == addr { + return Arc::clone(&c); + } + } + let c = RpcClient::new(addr.to_string()); + self.map.insert(id, Arc::clone(&c)); + c + } + + pub fn get(&self, id: NodeId) -> Option> { + self.map.get(&id).map(|c| Arc::clone(&c)) + } + + pub fn connected_ids(&self) -> Vec { + self.map + .iter() + .filter(|e| e.value().connected.load(Ordering::Relaxed)) + .map(|e| *e.key()) + .collect() + } +} + +// ---------- openraft network adapter ---------- + +pub(super) struct MeshFactory { + pub clients: Arc, +} + +pub(super) struct NodeClient { + client: Arc, +} + +impl RaftNetworkFactory for MeshFactory { + type Network = NodeClient; + + async fn new_client(&mut self, target: NodeId, node: &BasicNode) -> Self::Network { + NodeClient { + client: self.clients.get_or_create(target, &node.addr), + } + } +} + +fn net_err(e: std::io::Error) -> openraft::error::RPCError +where + E: std::error::Error, +{ + if e.kind() == std::io::ErrorKind::ConnectionRefused { + openraft::error::RPCError::Unreachable(Unreachable::new(&e)) + } else { + openraft::error::RPCError::Network(NetworkError::new(&e)) + } +} + +fn remote_err(msg: String) -> openraft::error::RPCError +where + E: std::error::Error, +{ + let e = std::io::Error::other(msg); + openraft::error::RPCError::Network(NetworkError::new(&e)) +} + +impl RaftNetwork for NodeClient { + async fn append_entries( + &mut self, + req: AppendEntriesRequest, + _option: RPCOption, + ) -> Result, typ::RPCError> { + match self.client.call(RpcRequest::Append(req)).await { + Ok(RpcReply::Append(Ok(resp))) => Ok(resp), + Ok(RpcReply::Append(Err(m))) => Err(remote_err(m)), + Ok(_) => Err(remote_err("mismatched rpc reply".into())), + Err(e) => Err(net_err(e)), + } + } + + async fn install_snapshot( + &mut self, + req: InstallSnapshotRequest, + _option: RPCOption, + ) -> Result, typ::RPCError> { + match self.client.call(RpcRequest::Snapshot(req)).await { + Ok(RpcReply::Snapshot(Ok(resp))) => Ok(resp), + Ok(RpcReply::Snapshot(Err(m))) => Err(remote_err(m)), + Ok(_) => Err(remote_err("mismatched rpc reply".into())), + Err(e) => Err(net_err(e)), + } + } + + async fn vote( + &mut self, + req: VoteRequest, + _option: RPCOption, + ) -> Result, typ::RPCError> { + match self.client.call(RpcRequest::Vote(req)).await { + Ok(RpcReply::Vote(Ok(resp))) => Ok(resp), + Ok(RpcReply::Vote(Err(m))) => Err(remote_err(m)), + Ok(_) => Err(remote_err("mismatched rpc reply".into())), + Err(e) => Err(net_err(e)), + } + } +} + +// ---------- server ---------- + +/// Accept loop: serve openraft RPCs + forwarded proposals against the local +/// Raft. Each request runs as its own task so a slow apply (Forward) doesn't +/// head-of-line-block heartbeats on the same connection. +pub(super) fn spawn_server(listener: TcpListener, raft: Raft, store: Arc) { + tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((sock, _)) => { + let _ = sock.set_nodelay(true); + let raft = raft.clone(); + let store = Arc::clone(&store); + tokio::spawn(serve_conn(sock, raft, store)); + } + Err(_) => tokio::time::sleep(RECONNECT_BACKOFF).await, + } + } + }); +} + +async fn serve_conn(sock: TcpStream, raft: Raft, store: Arc) { + let (mut rd, wr) = sock.into_split(); + let (tx, rx) = mpsc::channel::>(1024); + tokio::spawn(write_loop(wr, rx)); + let mut len_buf = [0u8; 4]; + loop { + if rd.read_exact(&mut len_buf).await.is_err() { + return; + } + let len = u32::from_be_bytes(len_buf); + if len == 0 || len > MAX_FRAME { + return; + } + let mut buf = vec![0u8; len as usize]; + if rd.read_exact(&mut buf).await.is_err() { + return; + } + let Ok((req_id, req)) = bincode::deserialize::<(u64, RpcRequest)>(&buf) else { + return; + }; + let raft = raft.clone(); + let store = Arc::clone(&store); + let tx = tx.clone(); + tokio::spawn(async move { + let reply = handle_rpc(&raft, &store, req).await; + let Ok(body) = bincode::serialize(&(req_id, reply)) else { + return; + }; + let mut frame = Vec::with_capacity(4 + body.len()); + frame.extend_from_slice(&(body.len() as u32).to_be_bytes()); + frame.extend_from_slice(&body); + let _ = tx.send(frame).await; + }); + } +} + +async fn handle_rpc(raft: &Raft, store: &Arc, req: RpcRequest) -> RpcReply { + match req { + RpcRequest::Append(r) => { + RpcReply::Append(raft.append_entries(r).await.map_err(|e| e.to_string())) + } + RpcRequest::Vote(r) => RpcReply::Vote(raft.vote(r).await.map_err(|e| e.to_string())), + RpcRequest::Snapshot(r) => { + RpcReply::Snapshot(raft.install_snapshot(r).await.map_err(|e| e.to_string())) + } + RpcRequest::Forward(op) => RpcReply::Forward( + raft.client_write(op) + .await + .map(|resp| (resp.data, resp.log_id.index)) + .map_err(|e| e.to_string()), + ), + RpcRequest::FetchStream { path, id, from, to } => { + RpcReply::FetchStream(fetch_stream(store, &path, id, from, to).await) + } + } +} + +/// Serve a snapshot-install byte pull. `[from, to)` of a live stream is an +/// immutable prefix once `to ≤ durable_tail`, so this is safe against any +/// concurrent appends; the `id` check catches delete+recreate races. +async fn fetch_stream( + store: &Arc, + path: &str, + id: u64, + from: u64, + to: u64, +) -> Result, String> { + let st = store.get(path).ok_or_else(|| format!("{path}: not found"))?; + if st.id != id { + return Err(format!("{path}: stream id changed (deleted+recreated)")); + } + if st.tail().bytes < to { + return Err(format!("{path}: tail below requested range")); + } + crate::handlers::read_range_bytes(&st, from, to) + .await + .map(|b| b.to_vec()) + .map_err(|e| e.to_string()) +} diff --git a/packages/durable-streams-rust/src/replication/sm.rs b/packages/durable-streams-rust/src/replication/sm.rs new file mode 100644 index 0000000000..c200573fbf --- /dev/null +++ b/packages/durable-streams-rust/src/replication/sm.rs @@ -0,0 +1,487 @@ +//! The Raft state machine: folds decided `LogOp`s into the local `Store` +//! through the same apply twins the single-node handlers mirror +//! (`handlers::apply_replicated_*`) — log-first apply, REPLICATION.md. +//! +//! Batch applies are SHARDED by stream path (per-stream log order preserved): +//! one slow store write (an fs-journal stall) must not serialize every other +//! stream's ack — the lesson measured under omnipaxos (100–450 ms apply +//! stalls) carries over unchanged. Fork-creates read the parent stream, which +//! may hash to another shard, so they act as a barrier and apply inline. +//! +//! Snapshots are METADATA-ONLY MARKERS: the stream files are the real state, +//! and v1 has no state transfer. Building one is cheap (it exists so openraft +//! purges the log — that is how memory stays bounded); INSTALLING one is +//! refused loudly, so a follower that fell behind the purge horizon stays +//! behind (fail-stop, REPLICATION.md "Guarantees") instead of silently +//! diverging. + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; + +use openraft::storage::{RaftStateMachine, Snapshot}; +use openraft::{ + BasicNode, EntryPayload, LogId, RaftSnapshotBuilder, SnapshotMeta, StorageError, + StorageIOError, StoredMembership, +}; +use tokio::sync::{Mutex, RwLock}; + +use serde::{Deserialize, Serialize}; + +use super::entry::{AppendApplyOutcome, CreateApplyOutcome, LogOp, OpOutcome}; +use super::types::{Entry, NodeId, TypeConfig}; +use crate::store::{Store, StreamConfig, StreamState}; + +/// One stream in a snapshot MANIFEST. Snapshots carry metadata only — the +/// byte content is pulled over the mesh at install time (`FetchStream` RPC), +/// which the append-only model makes safe: a stream's `[0, tail)` prefix is +/// immutable, so fetching it later from a peer that has advanced still yields +/// exactly the bytes at snapshot time. `id` guards against the path being +/// deleted+recreated in between (the serving peer verifies it). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StreamSnap { + pub path: String, + pub id: u64, + pub config: StreamConfig, + pub tail: u64, + pub closed: bool, + pub closed_by: Option<(String, u64, u64)>, + pub producers: Vec<(String, u64, u64)>, + pub last_seq_header: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Manifest { + pub streams: Vec, +} + +/// Wiring the state machine needs for snapshot INSTALL (fetching bytes from +/// the current leader) — populated after `Raft::new` by `mod.rs`. +pub struct InstallCtx { + pub self_id: NodeId, + pub clients: Arc, + pub metrics: tokio::sync::watch::Receiver>, +} + +const APPLY_SHARDS: usize = 4; +/// Dirty meta sidecars are swept on this cadence (vs per-append rewrites, +/// whose rename churn caused the fs stalls in the first place). +pub(super) const META_SWEEP: std::time::Duration = std::time::Duration::from_secs(3); + +#[derive(Debug, Clone, Default)] +pub struct SmMeta { + pub last_applied: Option>, + pub last_membership: StoredMembership, +} + +pub struct StateMachine { + pub store: Arc, + pub meta: RwLock, + /// Streams touched since the last sweep — flushed by the sweeper task. + pub dirty: Mutex>>, + /// Worst single-op apply time since last stats emit, µs. + pub apply_max_us: std::sync::atomic::AtomicU64, + /// Ops applied (cumulative). + pub applied_ops: std::sync::atomic::AtomicU64, + /// The last built manifest snapshot (offered to lagging followers). + pub current_snapshot: RwLock, Manifest)>>, + /// Fetch wiring for snapshot install — set once by `mod.rs` after boot. + pub install_ctx: std::sync::OnceLock, +} + +impl StateMachine { + pub fn new(store: Arc) -> Arc { + Arc::new(StateMachine { + store, + meta: RwLock::new(SmMeta::default()), + dirty: Mutex::new(HashMap::new()), + apply_max_us: std::sync::atomic::AtomicU64::new(0), + applied_ops: std::sync::atomic::AtomicU64::new(0), + current_snapshot: RwLock::new(None), + install_ctx: std::sync::OnceLock::new(), + }) + } + + /// Spawn the periodic dirty-meta sweeper. Flushes are detached onto the + /// blocking pool — a slow sweep must not stall applies. + pub fn spawn_meta_sweeper(self: &Arc) { + let sm = Arc::clone(self); + tokio::spawn(async move { + let mut tick = tokio::time::interval(META_SWEEP); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tick.tick().await; + let drained: Vec> = + sm.dirty.lock().await.drain().map(|(_, st)| st).collect(); + for st in drained { + tokio::task::spawn_blocking(move || { + let _ = crate::store::write_meta_sync(&st, false); + }); + } + } + }); + } + + fn shard_for(path: &str) -> usize { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + path.hash(&mut h); + (h.finish() as usize) % APPLY_SHARDS + } + + async fn apply_op(store: &Arc, op: LogOp) -> (OpOutcome, bool) { + match op { + LogOp::Create { + path, + config, + base_offset, + wire, + } => { + let out = crate::handlers::apply_replicated_create( + store, + &path, + config, + base_offset, + wire, + ) + .await; + let dirty = matches!(out, CreateApplyOutcome::Created { .. }); + (OpOutcome::Create(out), dirty) + } + LogOp::Append { + path, + wire, + producer, + seq, + close, + } => { + let out = crate::handlers::apply_replicated_append( + store, &path, wire, producer, seq, close, + ) + .await; + let dirty = matches!(out, AppendApplyOutcome::Applied { .. }); + (OpOutcome::Append(out), dirty) + } + LogOp::Delete { path } => ( + OpOutcome::Delete(crate::handlers::apply_replicated_delete(store, &path).await), + false, + ), + } + } + + /// Apply a run of (batch position, op) pairs — sequential within the run + /// (same shard ⇒ per-stream order), one run per shard concurrently. + async fn apply_run( + store: Arc, + run: Vec<(usize, LogOp, String)>, + sm: Arc, + ) -> Vec<(usize, OpOutcome)> { + let mut out = Vec::with_capacity(run.len()); + for (pos, op, path) in run { + let t0 = std::time::Instant::now(); + let (outcome, dirty) = Self::apply_op(&store, op).await; + let us = t0.elapsed().as_micros() as u64; + sm.apply_max_us + .fetch_max(us, std::sync::atomic::Ordering::Relaxed); + sm.applied_ops + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if dirty { + if let Some(st) = store.get(&path) { + sm.dirty.lock().await.entry(st.id).or_insert(st); + } + } + out.push((pos, outcome)); + } + out + } + + /// Apply a group of ops sharded by path; returns outcomes indexed by + /// batch position. + async fn apply_group(self: &Arc, group: Vec<(usize, LogOp)>, out: &mut Vec<(usize, OpOutcome)>) { + let mut runs: Vec> = (0..APPLY_SHARDS).map(|_| Vec::new()).collect(); + for (pos, op) in group { + let path = op.path().to_string(); + runs[Self::shard_for(&path)].push((pos, op, path)); + } + let mut joins = Vec::new(); + for run in runs { + if run.is_empty() { + continue; + } + let store = Arc::clone(&self.store); + let sm = Arc::clone(self); + joins.push(tokio::spawn(Self::apply_run(store, run, sm))); + } + for j in joins { + if let Ok(res) = j.await { + out.extend(res); + } + } + } +} + +impl RaftSnapshotBuilder for Arc { + async fn build_snapshot(&mut self) -> Result, StorageError> { + // MANIFEST snapshot: O(streams) metadata, no byte content (module + // docs). Built under the meta read lock so `last_applied` and the + // captured stream tails are mutually consistent (applies advance + // last_applied only after mutating the store, and tails captured here + // are ≥ the snapshot index's — over-capture is safe: prefixes are + // immutable and log replay past the snapshot re-applies + // deterministically via producer dedup / publish monotonicity). + let meta = self.meta.read().await; + let mut streams = Vec::new(); + for entry in self.store.streams.iter() { + let st = entry.value(); + let s = st.shared.read().unwrap(); + if s.soft_deleted { + continue; + } + streams.push(StreamSnap { + path: st.path.clone(), + id: st.id, + config: st.config.clone(), + tail: s.durable_tail, + closed: s.closed_durable, + closed_by: s.closed_by.clone(), + producers: s + .producers + .iter() + .map(|(k, v)| (k.clone(), v.epoch, v.last_seq)) + .collect(), + last_seq_header: s.last_seq_header.clone(), + }); + } + let manifest = Manifest { streams }; + let snapshot_id = meta + .last_applied + .map(|l| format!("manifest-{}-{}", l.leader_id, l.index)) + .unwrap_or_else(|| "manifest-empty".to_string()); + let snap_meta = SnapshotMeta { + last_log_id: meta.last_applied, + last_membership: meta.last_membership.clone(), + snapshot_id, + }; + let data = bincode::serialize(&manifest) + .map_err(|e| StorageIOError::write_snapshot(Some(snap_meta.signature()), &e))?; + drop(meta); + *self.current_snapshot.write().await = Some((snap_meta.clone(), manifest)); + Ok(Snapshot { + meta: snap_meta, + snapshot: Box::new(Cursor::new(data)), + }) + } +} + +impl StateMachine { + /// Rebuild the whole store from a manifest: wipe, recreate every stream, + /// pull its immutable byte prefix from the current leader over the mesh, + /// and restore producer/closed state. Fork children are materialized flat + /// (their logical content, standalone) — reads are byte-identical. + async fn install_manifest( + &self, + meta: &SnapshotMeta, + manifest: Manifest, + ) -> Result<(), StorageError> { + fn io_err_from( + sig: openraft::storage::SnapshotSignature, + e: impl std::error::Error + 'static, + ) -> StorageError { + StorageError::from(StorageIOError::read_snapshot(Some(sig), &e)) + } + let io_err = |e: std::io::Error| io_err_from(meta.signature(), e); + let ctx = self.install_ctx.get().ok_or_else(|| { + io_err(std::io::Error::other("snapshot install ctx not wired")) + })?; + // Pull from the current leader (who else could have sent a snapshot); + // any peer at/above the snapshot index would do. The leader's mesh + // address comes from the snapshot's membership. + let m = ctx.metrics.borrow().clone(); + let leader = m + .current_leader + .filter(|l| *l != ctx.self_id) + .ok_or_else(|| io_err(std::io::Error::other("no leader to fetch from")))?; + let leader_addr = meta + .last_membership + .membership() + .nodes() + .find(|(id, _)| **id == leader) + .map(|(_, n)| n.addr.clone()) + .or_else(|| { + m.membership_config + .membership() + .nodes() + .find(|(id, _)| **id == leader) + .map(|(_, n)| n.addr.clone()) + }) + .ok_or_else(|| io_err(std::io::Error::other("leader not in membership")))?; + let client = ctx.clients.get_or_create(leader, &leader_addr); + + self.store.wipe_all().map_err(|e| io_err(std::io::Error::other(e.to_string())))?; + for snap in manifest.streams { + let store = Arc::clone(&self.store); + let path = snap.path.clone(); + let config = snap.config.clone(); + let created = tokio::task::spawn_blocking(move || { + store.create_with_meta_durability(&path, config, None, 0, false) + }) + .await + .map_err(|e| io_err(std::io::Error::other(e.to_string())))? + .map_err(|e| io_err(std::io::Error::other(e.to_string())))?; + let crate::store::CreateResult::Created(st) = created else { + return Err(io_err(std::io::Error::other(format!( + "stream {} already exists during install", + snap.path + )))); + }; + // Fetch [0, tail) in chunks from the leader and append. + const CHUNK: u64 = 4 * 1024 * 1024; + let mut off = 0u64; + while off < snap.tail { + let to = (off + CHUNK).min(snap.tail); + let bytes = client + .fetch_stream(&snap.path, snap.id, off, to) + .await + .map_err(|e| io_err(std::io::Error::other(e)))?; + if bytes.len() as u64 != to - off { + return Err(io_err(std::io::Error::other(format!( + "short fetch for {} [{off},{to})", + snap.path + )))); + } + let wire = bytes::Bytes::from(bytes); + let mut ap = st.appender.lock().await; + let new_tail = + crate::handlers::write_wire(&st, &mut ap, &wire).map_err(|e| io_err(std::io::Error::other(e.to_string())))?; + drop(ap); + crate::handlers::publish_durable_tail(&st, new_tail, &wire); + off = to; + } + // Restore replicated per-stream state. + { + let mut sh = st.shared.write().unwrap(); + for (id, epoch, last_seq) in snap.producers { + sh.producers + .insert(id, crate::store::ProducerState { epoch, last_seq }); + } + sh.last_seq_header = snap.last_seq_header; + if snap.closed { + sh.closed = true; + sh.closed_durable = true; + sh.closed_by = snap.closed_by; + } + } + if snap.closed { + let t = st.tail().bytes; + st.tail_tx.send_replace(crate::store::Tail { + bytes: t, + closed: true, + }); + } + self.dirty.lock().await.insert(st.id, Arc::clone(&st)); + } + Ok(()) + } +} + +impl RaftStateMachine for Arc { + type SnapshotBuilder = Self; + + async fn applied_state( + &mut self, + ) -> Result<(Option>, StoredMembership), StorageError> + { + let meta = self.meta.read().await; + Ok((meta.last_applied, meta.last_membership.clone())) + } + + async fn apply(&mut self, entries: I) -> Result, StorageError> + where + I: IntoIterator + Send, + { + // Partition the batch into parallel groups; a fork-create is a + // barrier (it reads its parent stream, possibly on another shard). + let mut responses: Vec<(usize, OpOutcome)> = Vec::new(); + let mut group: Vec<(usize, LogOp)> = Vec::new(); + let mut last_applied = None; + let mut last_membership = None; + for (pos, entry) in entries.into_iter().enumerate() { + last_applied = Some(entry.log_id); + match entry.payload { + EntryPayload::Blank => responses.push((pos, OpOutcome::Noop)), + EntryPayload::Membership(ref mem) => { + last_membership = Some(StoredMembership::new(Some(entry.log_id), mem.clone())); + responses.push((pos, OpOutcome::Noop)); + } + EntryPayload::Normal(op) => { + let is_fork = matches!( + &op, + LogOp::Create { config, .. } if config.forked_from.is_some() + ); + if is_fork { + self.apply_group(std::mem::take(&mut group), &mut responses).await; + let (outcome, _) = StateMachine::apply_op(&self.store, op).await; + responses.push((pos, outcome)); + } else { + group.push((pos, op)); + } + } + } + } + self.apply_group(group, &mut responses).await; + + { + let mut meta = self.meta.write().await; + meta.last_applied = last_applied.or(meta.last_applied); + if let Some(m) = last_membership { + meta.last_membership = m; + } + } + responses.sort_by_key(|(p, _)| *p); + Ok(responses.into_iter().map(|(_, o)| o).collect()) + } + + async fn begin_receiving_snapshot( + &mut self, + ) -> Result>>, StorageError> { + Ok(Box::new(Cursor::new(Vec::new()))) + } + + async fn install_snapshot( + &mut self, + meta: &SnapshotMeta, + snapshot: Box>>, + ) -> Result<(), StorageError> { + let manifest: Manifest = bincode::deserialize(snapshot.get_ref()) + .map_err(|e| StorageIOError::read_snapshot(Some(meta.signature()), &e))?; + self.install_manifest(meta, manifest.clone()).await?; + { + let mut m = self.meta.write().await; + m.last_applied = meta.last_log_id; + m.last_membership = meta.last_membership.clone(); + } + *self.current_snapshot.write().await = Some((meta.clone(), manifest)); + Ok(()) + } + + async fn get_current_snapshot( + &mut self, + ) -> Result>, StorageError> { + let cur = self.current_snapshot.read().await; + match &*cur { + None => Ok(None), + Some((meta, manifest)) => { + let data = bincode::serialize(manifest) + .map_err(|e| StorageIOError::read_snapshot(Some(meta.signature()), &e))?; + Ok(Some(Snapshot { + meta: meta.clone(), + snapshot: Box::new(Cursor::new(data)), + })) + } + } + } + + async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder { + self.clone() + } +} diff --git a/packages/durable-streams-rust/src/replication/tests.rs b/packages/durable-streams-rust/src/replication/tests.rs new file mode 100644 index 0000000000..a67af3dc53 --- /dev/null +++ b/packages/durable-streams-rust/src/replication/tests.rs @@ -0,0 +1,441 @@ +//! Multi-node replication tests: three full consensus cores (real TCP mesh on +//! loopback ephemeral ports, real stores on disk) inside one tokio runtime. +//! These exercise propose → forward → decide → apply → ack end to end; the +//! HTTP layer on top is covered by the deploy smoke script. + +use std::sync::Arc; +use std::time::Duration; + +use super::entry::{ + AppendApplyOutcome, CreateApplyOutcome, DeleteApplyOutcome, LogOp, OpOutcome, ReplProducer, +}; +use super::{start_with_listener, ReplConfig, ReplHandle}; +use crate::store::{Store, StreamConfig}; +use crate::tier::TierConfig; + +/// A unique temp data dir for one node of one test. +fn tmp(tag: &str, node: u64) -> std::path::PathBuf { + use std::time::{SystemTime, UNIX_EPOCH}; + let p = std::env::temp_dir().join(format!( + "ds-repl-{tag}-{node}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = std::fs::remove_dir_all(&p); + p +} + +struct Cluster { + nodes: Vec<(Arc, Arc)>, + peers: Vec<(u64, String)>, +} + +/// Boot an n-node cluster on ephemeral loopback ports and wait for a leader. +async fn cluster(tag: &str, n: u64) -> Cluster { + let mut listeners = Vec::new(); + let mut peers = Vec::new(); + for id in 1..=n { + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + peers.push((id, l.local_addr().unwrap().to_string())); + listeners.push(l); + } + let mut nodes = Vec::new(); + for (i, listener) in listeners.into_iter().enumerate() { + let id = i as u64 + 1; + let store = Arc::new( + Store::new_with_tier(tmp(tag, id), TierConfig::default()).expect("store init"), + ); + let cfg = ReplConfig { + id, + peers: peers.clone(), + listen: String::new(), // pre-bound listener supplied below + ack_timeout: Duration::from_secs(10), + snapshot_logs: 64, // snapshot/purge frequently so tests cover it + stats_secs: 0, + join: false, + vote_path: None, + }; + let handle = start_with_listener(Arc::clone(&store), &cfg, listener).await; + nodes.push((store, handle)); + } + // Wait for an agreed leader everywhere (election timeout is 500-1000 ms). + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let leaders: Vec = nodes + .iter() + .map(|(_, h)| h.raft.metrics().borrow().current_leader.unwrap_or(0)) + .collect(); + if leaders.iter().all(|l| *l != 0) && leaders.windows(2).all(|w| w[0] == w[1]) { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "no leader elected: {leaders:?}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + Cluster { nodes, peers } +} + +fn plain_config() -> StreamConfig { + StreamConfig { + content_type: "application/octet-stream".to_string(), + ttl_seconds: None, + expires_at: None, + expires_at_raw: None, + create_closed: false, + forked_from: None, + fork_offset_raw: None, + fork_sub_offset: None, + } +} + +fn create_op(path: &str) -> LogOp { + LogOp::Create { + path: path.to_string(), + config: plain_config(), + base_offset: 0, + wire: vec![], + } +} + +fn append_op(path: &str, bytes: &[u8]) -> LogOp { + LogOp::Append { + path: path.to_string(), + wire: bytes.to_vec(), + producer: None, + seq: None, + close: false, + } +} + +/// Wait until every node's store shows `path` at `tail` (decided entries apply +/// asynchronously on non-origin nodes). +async fn wait_converged(cl: &Cluster, path: &str, tail: u64) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let tails: Vec> = cl + .nodes + .iter() + .map(|(s, _)| s.get(path).map(|st| st.tail().bytes)) + .collect(); + if tails.iter().all(|t| *t == Some(tail)) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "no convergence on {path} at {tail}: {tails:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn append_replicates_to_all_nodes_and_any_node_proposes() { + let cl = cluster("basic", 3).await; + let path = "/repl-basic"; + + // Create via node 1. + let out = cl.nodes[0].1.propose_and_wait(create_op(path)).await.unwrap(); + assert!( + matches!( + out, + OpOutcome::Create(CreateApplyOutcome::Created { tail: 0, .. }) + ), + "unexpected create outcome: {out:?}" + ); + + // Append via node 2 and node 3 (follower proposals forward to the leader). + let out = cl.nodes[1].1.propose_and_wait(append_op(path, b"hello ")).await.unwrap(); + assert!( + matches!(out, OpOutcome::Append(AppendApplyOutcome::Applied { tail: 6, .. })), + "unexpected append outcome: {out:?}" + ); + let out = cl.nodes[2].1.propose_and_wait(append_op(path, b"world")).await.unwrap(); + assert!( + matches!(out, OpOutcome::Append(AppendApplyOutcome::Applied { tail: 11, .. })), + "unexpected append outcome: {out:?}" + ); + + // Every replica converges to identical bytes. + wait_converged(&cl, path, 11).await; + for (store, _) in &cl.nodes { + let st = store.get(path).unwrap(); + let bytes = crate::handlers::read_range_bytes(&st, 0, 11).await.unwrap(); + assert_eq!(&bytes[..], b"hello world"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn producer_dedup_is_replicated() { + let cl = cluster("dedup", 3).await; + let path = "/repl-dedup"; + cl.nodes[0].1.propose_and_wait(create_op(path)).await.unwrap(); + + let with_producer = |seq: u64| LogOp::Append { + path: path.to_string(), + wire: b"x".to_vec(), + producer: Some(ReplProducer { + id: "p1".to_string(), + epoch: 0, + seq, + }), + seq: None, + close: false, + }; + + let out = cl.nodes[0].1.propose_and_wait(with_producer(0)).await.unwrap(); + assert!(matches!( + out, + OpOutcome::Append(AppendApplyOutcome::Applied { tail: 1, .. }) + )); + // The SAME (producer, epoch, seq) again — even via a different node — is a + // duplicate: acknowledged without a second append. + let out = cl.nodes[1].1.propose_and_wait(with_producer(0)).await.unwrap(); + assert!( + matches!( + out, + OpOutcome::Append(AppendApplyOutcome::ProducerDuplicate { last_seq: 0, .. }) + ), + "expected duplicate, got {out:?}" + ); + wait_converged(&cl, path, 1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn close_and_delete_replicate() { + let cl = cluster("close-del", 3).await; + let path = "/repl-close"; + cl.nodes[0].1.propose_and_wait(create_op(path)).await.unwrap(); + let out = cl.nodes[0] + .1 + .propose_and_wait(LogOp::Append { + path: path.to_string(), + wire: b"bye".to_vec(), + producer: None, + seq: None, + close: true, + }) + .await + .unwrap(); + assert!(matches!( + out, + OpOutcome::Append(AppendApplyOutcome::Applied { tail: 3, closed: true }) + )); + wait_converged(&cl, path, 3).await; + // Every replica observes the closure. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + if cl.nodes.iter().all(|(s, _)| s.get(path).unwrap().tail().closed) { + break; + } + assert!(tokio::time::Instant::now() < deadline, "closure did not replicate"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + // Appending to the closed stream via another node conflicts deterministically. + let out = cl.nodes[2].1.propose_and_wait(append_op(path, b"z")).await.unwrap(); + assert!(matches!( + out, + OpOutcome::Append(AppendApplyOutcome::Closed { tail: 3 }) + )); + + // Delete replicates. + let out = cl.nodes[1] + .1 + .propose_and_wait(LogOp::Delete { + path: path.to_string(), + }) + .await + .unwrap(); + assert!(matches!(out, OpOutcome::Delete(DeleteApplyOutcome::Deleted))); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + let all_gone = cl.nodes.iter().all(|(s, _)| match s.get(path) { + None => true, + Some(st) => st.shared.read().unwrap().soft_deleted, + }); + if all_gone { + break; + } + assert!(tokio::time::Instant::now() < deadline, "delete did not replicate"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn initial_body_and_fork_offsets_replicate() { + let cl = cluster("fork", 3).await; + // Create with an initial body. + let out = cl.nodes[0] + .1 + .propose_and_wait(LogOp::Create { + path: "/src".to_string(), + config: plain_config(), + base_offset: 0, + wire: b"0123456789".to_vec(), + }) + .await + .unwrap(); + assert!(matches!( + out, + OpOutcome::Create(CreateApplyOutcome::Created { tail: 10, .. }) + )); + wait_converged(&cl, "/src", 10).await; + + // Fork at offset 4 (pre-resolved by the proposing node, as the handler does). + let mut fork_cfg = plain_config(); + fork_cfg.forked_from = Some("/src".to_string()); + fork_cfg.fork_offset_raw = Some("4".to_string()); + let out = cl.nodes[1] + .1 + .propose_and_wait(LogOp::Create { + path: "/fork".to_string(), + config: fork_cfg, + base_offset: 4, + wire: vec![], + }) + .await + .unwrap(); + assert!(matches!( + out, + OpOutcome::Create(CreateApplyOutcome::Created { tail: 4, .. }) + )); + // The fork reads through to the parent on every node. + wait_converged(&cl, "/fork", 4).await; + for (store, _) in &cl.nodes { + let st = store.get("/fork").unwrap(); + let bytes = crate::handlers::read_range_bytes(&st, 0, 4).await.unwrap(); + assert_eq!(&bytes[..], b"0123"); + } +} + + +/// The "lift fail-stop" milestone end to end: run a 3-node cluster far past +/// the purge horizon, then bring up a FRESH node 4 (empty store, --repl-join +/// semantics) and add it via add_learner + change_membership. It can only +/// catch up via the manifest snapshot + FetchStream byte pull — the log +/// behind the snapshot is gone. Then verify it serves byte-identical data +/// and participates as a voter. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn learner_joins_after_purge_via_manifest_snapshot() { + let cl = cluster("join", 3).await; + let path = "/repl-join"; + cl.nodes[0].1.propose_and_wait(create_op(path)).await.unwrap(); + + // Push well past snapshot_logs=64 PLUS the 256-entry purge margin + // (max_in_snapshot_log_to_keep) so the log genuinely purges. + let mut expected = String::new(); + for i in 0..600 { + let payload = format!("r{i:04};"); + expected.push_str(&payload); + cl.nodes[i % 3] + .1 + .propose_and_wait(append_op(path, payload.as_bytes())) + .await + .unwrap(); + } + let tail = expected.len() as u64; + wait_converged(&cl, path, tail).await; + // The purge must actually have happened for this test to mean anything. + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let purged = cl.nodes[0] + .1 + .raft + .metrics() + .borrow() + .purged + .map(|l| l.index) + .unwrap_or(0); + if purged > 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "log never purged; metrics: {:?}", + cl.nodes[0].1.raft.metrics().borrow().clone() + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // Fresh node 4: empty store, join mode (no initialize). + let l4 = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr4 = l4.local_addr().unwrap().to_string(); + let store4 = Arc::new( + Store::new_with_tier(tmp("join", 4), TierConfig::default()).expect("store init"), + ); + let mut peers4 = cl.peers.clone(); + peers4.push((4, addr4.clone())); + let cfg4 = ReplConfig { + id: 4, + peers: peers4, + listen: String::new(), + ack_timeout: Duration::from_secs(10), + snapshot_logs: 64, + stats_secs: 0, + join: true, + vote_path: None, + }; + let h4 = start_with_listener(Arc::clone(&store4), &cfg4, l4).await; + + // Add as learner (blocking: returns once caught up), then promote. + let leader = cl.nodes[0].1.raft.metrics().borrow().current_leader.unwrap(); + let lh = &cl.nodes[(leader - 1) as usize].1; + lh.raft + .add_learner(4, openraft::BasicNode::new(addr4), true) + .await + .expect("add_learner"); + lh.raft + .change_membership([1u64, 2, 3, 4].into_iter().collect::>(), false) + .await + .expect("change_membership"); + + // Node 4 must have byte-identical content, fetched via the snapshot path. + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if store4.get(path).map(|st| st.tail().bytes) == Some(tail) { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "node 4 never caught up (tail {:?} want {tail})", + store4.get(path).map(|st| st.tail().bytes) + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let st = store4.get(path).unwrap(); + let bytes = crate::handlers::read_range_bytes(&st, 0, tail).await.unwrap(); + assert_eq!(std::str::from_utf8(&bytes).unwrap(), expected); + + // And it participates: writes proposed via node 4 ack and replicate. + let out = h4.propose_and_wait(append_op(path, b"after-join;")).await.unwrap(); + assert!(matches!( + out, + OpOutcome::Append(AppendApplyOutcome::Applied { .. }) + )); + wait_converged(&cl, path, tail + 11).await; +} + +/// The durable vote: saved through the log store, fsynced, and reloaded on a +/// fresh instance with the same path — the restart-election-safety invariant. +#[tokio::test] +async fn vote_survives_restart_via_vote_file() { + use openraft::storage::RaftLogStorage; + let dir = tmp("vote", 1); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("repl.vote"); + let vote = openraft::Vote::::new(7, 3); + { + let mut ls = + super::log_store::LogStore::::with_vote_path(path.clone()) + .unwrap(); + ls.save_vote(&vote).await.unwrap(); + } + let mut ls2 = + super::log_store::LogStore::::with_vote_path(path).unwrap(); + let restored = ls2.read_vote().await.unwrap(); + assert_eq!(restored, Some(vote)); +} diff --git a/packages/durable-streams-rust/src/replication/types.rs b/packages/durable-streams-rust/src/replication/types.rs new file mode 100644 index 0000000000..103dcb2584 --- /dev/null +++ b/packages/durable-streams-rust/src/replication/types.rs @@ -0,0 +1,34 @@ +//! openraft type wiring: `LogOp` in, `OpOutcome` out. + +use std::io::Cursor; + +use super::entry::{LogOp, OpOutcome}; + +// Cursor is referenced by the declare_raft_types! expansion (SnapshotData). +#[allow(unused_imports)] +use Cursor as _Cursor; + +pub type NodeId = u64; + +openraft::declare_raft_types!( + /// D = the replicated op, R = the apply outcome (resolved by client_write). + /// Defaults: NodeId = u64, Node = BasicNode, Entry = Entry, + /// SnapshotData = Cursor>, AsyncRuntime = TokioRuntime. + pub TypeConfig: + D = LogOp, + R = OpOutcome, +); + +pub type Raft = openraft::Raft; +pub type Entry = openraft::Entry; + +pub mod typ { + use openraft::BasicNode; + + use super::NodeId; + + pub type RaftError = openraft::error::RaftError; + pub type RPCError = + openraft::error::RPCError>; + pub type ClientWriteError = openraft::error::ClientWriteError; +} diff --git a/packages/durable-streams-rust/src/sse_reactor.rs b/packages/durable-streams-rust/src/sse_reactor.rs index 549fe27b6c..50636a53d4 100644 --- a/packages/durable-streams-rust/src/sse_reactor.rs +++ b/packages/durable-streams-rust/src/sse_reactor.rs @@ -143,6 +143,12 @@ pub fn wake_stream(st: &StreamState) { // server) with no SSE subscribers never spawns a reactor thread. let guard = st.sse_subs.lock().unwrap(); let Some(list) = guard.as_ref() else { return }; + // Coalesce: if a wake for this stream is already queued and not yet + // flushed, this publish is covered by the flush's tail read — skip the + // queue pushes and the eventfd syscall entirely. + if list.wake_pending.swap(true, Ordering::AcqRel) { + return; + } // Subscribers exist only because `register` ran, so the pool is live. let pool = pool(); for h in &list.subs { @@ -278,6 +284,14 @@ impl Reactor { .get(key as usize) .is_some_and(|s| s.gen == gen && s.sub.is_some()) { + // Clear the stream's coalescing latch BEFORE producing: + // produce() reads the tail after the clear, so a publish + // racing this flush is either included or re-queues. + if let Some(sub) = self.slab[key as usize].sub.as_ref() { + if let Some(list) = sub.st.sse_subs.lock().unwrap().as_ref() { + list.wake_pending.store(false, Ordering::Release); + } + } self.produce(key); self.flush(key); } @@ -319,13 +333,21 @@ impl Reactor { // Link onto the stream so the append path can find and wake this sub. { let mut g = st.sse_subs.lock().unwrap(); - g.get_or_insert_with(|| Box::new(StreamSubs { subs: Vec::new() })) - .subs - .push(SubHandle { - shard: self.shard_idx, - key, - gen, - }); + let list = g.get_or_insert_with(|| { + Box::new(StreamSubs { + subs: Vec::new(), + wake_pending: std::sync::atomic::AtomicBool::new(false), + }) + }); + list.subs.push(SubHandle { + shard: self.shard_idx, + key, + gen, + }); + // A gen-stale wake (subscriber closed between queue and drain) is + // dropped without clearing the latch; reset it on every register so + // a fresh subscriber can never inherit a stale-set latch. + list.wake_pending.store(false, Ordering::Release); } // Watch for peer close/errors; EPOLLOUT is armed lazily by flush(). let mut ev = libc::epoll_event { diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index c3ba17f5b8..ead837f483 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -32,7 +32,9 @@ pub struct ProducerState { pub last_seq: u64, } -#[derive(Clone, Debug)] +// Serialize/Deserialize: the config travels inside replicated `LogOp::Create` +// entries (src/replication/entry.rs). +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct StreamConfig { pub content_type: String, pub ttl_seconds: Option, @@ -106,10 +108,35 @@ pub struct Appender { /// Returns the fsync result: a failure (e.g. EIO writeback error) MUST be /// surfaced to the caller so an append is never acked as durable when the data /// did not reach stable storage. +/// BENCH-ONLY: whether `DS_BENCH_FAST_FSYNC` requests plain `fsync` over +/// `F_FULLFSYNC` on macOS (see [`barrier_fsync`]). Read once and cached — the env +/// is fixed for the process lifetime. +#[cfg(target_os = "macos")] +fn fast_fsync_enabled() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("DS_BENCH_FAST_FSYNC").is_some()) +} + pub(crate) fn barrier_fsync(file: &File) -> std::io::Result<()> { let fd = file.as_raw_fd(); #[cfg(target_os = "macos")] unsafe { + // BENCH-ONLY escape hatch (NOT for production durability): when + // `DS_BENCH_FAST_FSYNC` is set, use a plain `fsync` instead of + // `F_FULLFSYNC`. On macOS `F_FULLFSYNC` forces a true drive-cache barrier + // (~tens of ms even on a RAM disk), which dominates the commit path and + // masks the per-shard LOCK contention this build is meant to study. Plain + // `fsync` on a RAM disk is ~free, reproducing the cheap-fsync (Linux + + // NVMe) regime where the lock is the bottleneck. Never set this where data + // must survive power loss. + if fast_fsync_enabled() { + return if libc::fsync(fd) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + }; + } // Force a true flush to platter; fall back to a plain fsync. Only error // if the final fallback also fails. if libc::fcntl(fd, libc::F_FULLFSYNC) == 0 { @@ -152,8 +179,22 @@ pub struct StreamState { pub appender: AsyncMutex, pub shared: RwLock, pub tail_tx: watch::Sender, + /// The sidecar's persisted `durable_tail` as read at BOOT (None for sidecars + /// written by older servers). Consumed once by WAL recovery as this stream's + /// truncation-proof seed; never updated afterwards (the live value lives in + /// `Shared.durable_tail` and is re-captured on every meta write). + pub boot_meta_durable_tail: Option, /// True while a debounced meta flush is pending. pub meta_dirty: AtomicBool, + /// **Lock-free WAL dirty-set marker** (Tier-1a). The shard's checkpoint epoch + /// at which this stream was last registered into its shard's dirty set. The + /// hot append path compares this against the shard's current epoch: equal ⇒ + /// already registered this interval (no lock, no push); not-equal ⇒ CAS it to + /// the current epoch and, on the winning transition only, push this stream's + /// `Arc` into the shard's dirty collection. Initialised to `0` + /// (the shard's epoch starts at `1`), so the first append after creation always + /// registers. See `Shard::register_dirty`. + pub dirty_epoch: AtomicU64, /// Serializes sidecar writes for this stream. Concurrent writers (append /// flush, close, tiering offload flip, delete) otherwise race on the shared /// `.meta.tmp` file and can reorder their renames, letting a stale non-durable @@ -192,6 +233,15 @@ pub struct StreamState { #[cfg(target_os = "linux")] pub struct StreamSubs { pub subs: Vec, + /// Wake-coalescing latch: set by `wake_stream` when it queues this stream's + /// subscribers, cleared by the reactor BEFORE it reads the tail to flush + /// (clear-then-read: a publish racing the flush either lands before the + /// clear — its bytes are covered by the post-clear tail read — or after, + /// and re-queues). Converts per-append wakes into one wake per stream per + /// reactor cycle under load — the fan-out batching that wal mode gets for + /// free from group commit, without which memory mode drowns the reactor in + /// per-append eventfd signals (measured: delivery collapse past ~16k w/s). + pub wake_pending: std::sync::atomic::AtomicBool, } /// Locates one reactor subscriber: which shard owns it, its slab key, and the @@ -243,25 +293,6 @@ pub fn tail_cache_bytes() -> usize { } impl StreamState { - /// Open a fresh `O_WRONLY` fd on the data file for positioned splice writes. - /// - /// The shared `Appender.file` is opened `O_APPEND`, which `splice(2)` rejects - /// as a target (it ignores the supplied offset). The zero-copy append path - /// therefore opens its own non-`O_APPEND` write fd and positions every write - /// explicitly (`pwrite` for the buffered prefix, `splice` with an offset for - /// the socket relay). Called under the appender lock, so no other writer can - /// move the logical tail underneath it. - #[cfg(target_os = "linux")] - pub fn open_splice_fd(&self) -> std::io::Result { - // O_RDWR (not O_WRONLY): this same fd is the positioned-WRITE target for the - // socket→file splice AND the READ source for the file→WAL relay splice, so it - // must be readable. (Not O_APPEND — splice rejects O_APPEND targets.) - std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&self.file_path) - } - /// Record the just-appended wire chunk as the resident tail. `start` is the /// logical offset where `bytes` begins. Chunks larger than the tail-cache cap /// (or any append when the cache is disabled) are not cached (the entry is @@ -570,6 +601,7 @@ impl Store { file_path: data_path.clone(), base_offset: meta.base_offset, parent, + boot_meta_durable_tail: meta.durable_tail, appender: AsyncMutex::new(Appender { file: file.clone(), written }), shared: RwLock::new(Shared { tail, @@ -588,6 +620,9 @@ impl Store { }), tail_tx, meta_dirty: AtomicBool::new(false), + // Epoch 0 < the shard's initial epoch (1), so the first append + // registers this stream into the dirty set. + dirty_epoch: AtomicU64::new(0), meta_lock: StdMutex::new(()), last_chunk: RwLock::new(None), tier: crate::tier::TierState::from_meta( @@ -638,7 +673,43 @@ impl Store { } /// Hard-delete when nothing references the stream; soft-delete otherwise. + /// + /// NON-durable, detached variant for the expiry sweep on the read path: the + /// on-disk removals / soft-meta write run on a fire-and-forget blocking + /// task, so a crash can undo them (an expired stream re-expires on the next + /// access — harmless). The DELETE handler must NOT use this: an acked + /// DELETE undone by a crash resurrects the stream with all its data — use + /// [`Store::delete_or_soft_delete_durable`] there. pub fn delete_or_soft_delete(&self, st: &Arc) { + let _ = self.delete_impl(st, false); + } + + /// [`Store::delete_or_soft_delete`] with the DELETE-ack durability contract: + /// the file + sidecar unlinks (and their parent-directory entry) — or the + /// soft-delete meta flag — are durable on disk before this returns, so a + /// post-ack crash can never resurrect the stream. Synchronous file I/O + + /// fsync: call from a blocking context. + pub fn delete_or_soft_delete_durable(&self, st: &Arc) -> std::io::Result<()> { + self.delete_impl(st, true) + } + + /// Drop EVERY stream — in-memory state and on-disk files — regardless of + /// fork refcounts. Used by replicated-mode snapshot install (the incoming + /// manifest replaces the whole store) and by replicated-mode boot (the + /// consensus log/snapshot is the source of truth; stale local files must + /// not double-apply). Readers racing a wipe see 404s, matching a follower + /// that is legitimately behind. + pub fn wipe_all(&self) -> std::io::Result<()> { + self.streams.clear(); + let dir = self.data_dir.join("streams"); + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let _ = std::fs::remove_file(entry.path()); + } + Ok(()) + } + + fn delete_impl(&self, st: &Arc, durable: bool) -> std::io::Result<()> { let soft = { let mut s = st.shared.write().unwrap(); if s.ref_count > 0 { @@ -649,10 +720,14 @@ impl Store { } }; if soft { - let st2 = st.clone(); - tokio::task::spawn_blocking(move || { - let _ = write_meta_sync(&st2, true); - }); + if durable { + write_meta_sync(st, true)?; + } else { + let st2 = st.clone(); + tokio::task::spawn_blocking(move || { + let _ = write_meta_sync(&st2, true); + }); + } } else { self.streams .remove_if(&st.path, |_, v| Arc::ptr_eq(v, st)); @@ -661,12 +736,21 @@ impl Store { // with no remaining fork references. self.gc_remote_segments(st); let fp = st.file_path.clone(); - tokio::task::spawn_blocking(move || { + if durable { + // Both unlinks live in the same directory; one dir fsync makes + // them crash-durable together. let _ = std::fs::remove_file(meta_path(&fp)); - let _ = std::fs::remove_file(fp); - }); + let _ = std::fs::remove_file(&fp); + fsync_parent_dir(&fp)?; + } else { + tokio::task::spawn_blocking(move || { + let _ = std::fs::remove_file(meta_path(&fp)); + let _ = std::fs::remove_file(fp); + }); + } self.release_parent(st); } + Ok(()) } /// Decrement the parent's fork refcount; cascade-collect soft-deleted parents @@ -705,6 +789,22 @@ impl Store { config: StreamConfig, parent: Option>, base_offset: u64, + ) -> std::io::Result { + self.create_with_meta_durability(path, config, parent, base_offset, true) + } + + /// `create` with the meta-sidecar fsync optional. The replicated applier + /// passes `durable_meta: false`: the decided log entry is the create's + /// durability there, and the fsync would serialize every stream creation + /// through the single applier task (REPLICATION.md). The sidecar is still + /// written (non-durably) so local tooling sees it. + pub fn create_with_meta_durability( + &self, + path: &str, + config: StreamConfig, + parent: Option>, + base_offset: u64, + durable_meta: bool, ) -> std::io::Result { use dashmap::mapref::entry::Entry; // Fast path: existing stream → config comparison. @@ -741,6 +841,9 @@ impl Store { file_path, base_offset, parent: parent.clone(), + // Live-created stream: the durable frontier IS the initial tail (the + // create meta below persists it). Only consulted by boot recovery. + boot_meta_durable_tail: Some(base_offset), appender: AsyncMutex::new(Appender { file: file.clone(), written: 0 }), shared: RwLock::new(Shared { tail: base_offset, @@ -758,6 +861,9 @@ impl Store { }), tail_tx, meta_dirty: AtomicBool::new(false), + // Epoch 0 < the shard's initial epoch (1), so the first append + // registers this stream into the dirty set. + dirty_epoch: AtomicU64::new(0), meta_lock: StdMutex::new(()), last_chunk: RwLock::new(None), tier: crate::tier::TierState::default(), @@ -788,9 +894,9 @@ impl Store { // rejected/raced creates never leak a refcount on the source. if let Some(p) = &parent { p.shared.write().unwrap().ref_count += 1; - write_meta_sync(p, true)?; + write_meta_sync(p, durable_meta)?; } - write_meta_sync(&state, true)?; + write_meta_sync(&state, durable_meta)?; Ok(CreateResult::Created(state)) } } @@ -951,6 +1057,16 @@ pub struct Meta { /// once the rewrite + swap completes durably. #[serde(default)] pub pending_compaction: Option, + /// The stream's durable frontier (logical bytes) when this sidecar was + /// written — WAL recovery's per-stream truncation proof for streams with NO + /// retained WAL record and NO checkpoint `tails` entry (e.g. a stream + /// created after the last checkpoint whose only in-flight append was torn + /// by power loss). Only ever holds values that were durable at capture + /// time; a lagging (lazily-flushed) value is safe because recovery takes + /// the max of every proof. `None` in sidecars written by older servers → + /// recovery falls back to trusting the file size (the pre-field behavior). + #[serde(default)] + pub durable_tail: Option, } /// Serialized form of a sealed-segment manifest entry. @@ -1019,6 +1135,7 @@ impl Meta { sealed_offset: st.tier.manifest.lock().unwrap().sealed_offset, file_base: Some(s.file_base), pending_compaction: *st.compaction.lock().unwrap(), + durable_tail: Some(s.durable_tail), } } } diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index 7f8099122c..e2e3819f3e 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -18,13 +18,13 @@ use std::io; use std::sync::Arc; use bytes::Bytes; -use tokio::task::JoinHandle; use crate::api::{Method, Req}; use crate::handlers; use crate::handlers::test_support::DurabilityGuard; use crate::store::Store; use crate::tier::TierConfig; +use crate::wal::shard::CommitterHandle; use crate::wal::walset::WalSet; /// A unique temp data dir for one test. @@ -40,12 +40,15 @@ fn tmp(tag: &str) -> std::path::PathBuf { } /// A booted WAL-mode server harness: the store with its WAL attached and the -/// per-shard committers running. Committer `JoinHandle`s are held so a test can -/// `.abort()` them to simulate a crash (a graceful shutdown would drain them). +/// per-shard committers running on their dedicated OS threads. Committer +/// [`CommitterHandle`]s are held so a test can stop + join them to simulate a +/// crash. (Every record a crash test relies on is already acked — hence durable — +/// before the stop, so the final drain a graceful stop performs is a no-op for +/// those records; the on-disk state matches an abrupt crash.) struct Harness { store: Arc, walset: Arc, - committers: Vec>, + committers: Vec, } impl Harness { @@ -60,7 +63,18 @@ impl Harness { /// the durable WAL into the per-stream files + fsync) → `reset_after_recovery` /// (wipe the old WAL) → attach + `spawn_committers`. fn boot(dir: &std::path::Path, shards: Option, default_n: usize) -> io::Result { - let walset = WalSet::open(dir, shards, default_n)?; + Harness::boot_with_segment_size(dir, shards, default_n, crate::wal::segment::SEGMENT_BYTES) + } + + /// [`Harness::boot`] with an explicit WAL segment size, so a test can force + /// segment rolls/recycles cheaply (multi-segment recovery coverage). + fn boot_with_segment_size( + dir: &std::path::Path, + shards: Option, + default_n: usize, + segment_size: u64, + ) -> io::Result { + let walset = WalSet::open_with_segment_size(dir, shards, default_n, segment_size)?; let store = Arc::new(Store::new_with_tier(dir.to_path_buf(), TierConfig::default())?); crate::wal::recovery::recover(&store, &walset)?; walset.reset_after_recovery()?; @@ -69,23 +83,29 @@ impl Harness { .set(Arc::clone(&walset)) .unwrap_or_else(|_| panic!("WAL already attached")); // Spawn committers ourselves (not `walset.spawn_committers()`) so we keep - // the JoinHandles and can abort them to simulate a crash. + // the handles and can stop them to simulate a crash. let mut committers = Vec::new(); for shard in walset.shards() { - let shard = Arc::clone(shard); - committers.push(tokio::spawn(shard.run_committer())); + committers.push(shard.spawn_committer()); } Ok(Harness { store, walset, committers }) } - /// Simulate a crash: abort the committers (so no further `durable_lsn` + /// Stop + join every committer thread (so no further `durable_lsn` advance can + /// race the test's subsequent file surgery). All records the caller cares + /// about are already acked/durable, so each committer's final drain is a + /// no-op for them. + fn stop_committers(&mut self) { + for h in self.committers.drain(..) { + h.stop(); + } + } + + /// Simulate a crash: stop the committers (so no further `durable_lsn` /// advance) and drop the store + WalSet WITHOUT a graceful drain/shutdown. /// The data dir on disk is left exactly as the live process left it. - fn crash(self) { - for h in &self.committers { - h.abort(); - } - drop(self.committers); + fn crash(mut self) { + self.stop_committers(); drop(self.store); drop(self.walset); } @@ -253,7 +273,7 @@ async fn e2e_no_loss_unacked_tail_is_absent_after_crash() { let dir = tmp("noloss-unacked"); - let h = Harness::boot(&dir, Some(1), 1).unwrap(); + let mut h = Harness::boot(&dir, Some(1), 1).unwrap(); create_stream(&h.store, "s", OCTET).await; // Two genuinely-acked appends through the real path (durable in the WAL). @@ -267,9 +287,7 @@ async fn e2e_no_loss_unacked_tail_is_absent_after_crash() { // (a) append the un-acked bytes to the per-stream file (page-cache write). // (b) plant a TORN partial WAL record right after the two whole durable // records: only a few header bytes, never a full framed record. - for c in &h.committers { - c.abort(); - } + h.stop_committers(); let st = h.store.get("s").unwrap(); let unacked: &[u8] = b"UNACKED-NEVER-DURABLE|"; { @@ -480,7 +498,7 @@ async fn e2e_sharding_below_file_base_record_skipped() { let _guard = DurabilityGuard::wal(); let dir = tmp("below-base"); - let h = Harness::boot(&dir, Some(1), 1).unwrap(); + let mut h = Harness::boot(&dir, Some(1), 1).unwrap(); // Parent with content so a fork can diverge at offset > 0. create_stream(&h.store, "parent", OCTET).await; append_acked(&h.store, "parent", OCTET, b"0123456789").await; // tail = 10 @@ -507,10 +525,8 @@ async fn e2e_sharding_below_file_base_record_skipped() { // Stage a WAL record for the fork BELOW its file_base (stream_offset 2 < 5): // the frontier-skip case. It must be skipped on replay (re-applying would be // an out-of-range / double-apply; those bytes live in the parent/sealed - // prefix). Abort the committer first so we don't perturb the acked frontier. - for c in &h.committers { - c.abort(); - } + // prefix). Stop the committers first so we don't perturb the acked frontier. + h.stop_committers(); let shard = h.walset.shard_for(child.id); shard .reserve_and_stage( @@ -522,11 +538,10 @@ async fn e2e_sharding_below_file_base_record_skipped() { .unwrap(); // Re-run a committer briefly so this staged record DOES become durable in the // WAL (proving the skip is in recovery's replay, not just an un-acked drop). - let shard_arc = Arc::clone(shard); - let c = tokio::spawn(async move { shard_arc.run_committer().await }); + let c = shard.spawn_committer(); // Wait until durable so the record is genuinely in the WAL's durable range. shard.wait_durable(shard.tail_lsn()).await; - c.abort(); + c.stop(); drop(child); let store = h.store; @@ -784,6 +799,15 @@ async fn strict_created_dir_reopens_wal_only_without_data_loss() { .await .unwrap() .unwrap(); + // A strict-era server predates the `durable_tail` sidecar proof — strip + // the field the CURRENT writer emitted so the sidecar is byte-faithful + // to what an old deployment left behind (recovery must fall back to + // trusting the file size for such sidecars). + let meta_path = crate::store::meta_path(&st.file_path); + let mut v: serde_json::Value = + serde_json::from_slice(&std::fs::read(&meta_path).unwrap()).unwrap(); + v.as_object_mut().unwrap().remove("durable_tail"); + std::fs::write(&meta_path, serde_json::to_vec(&v).unwrap()).unwrap(); } // Remove any wal/ subtree — a strict-era dir has none. std::fs::remove_dir_all(dir.join("wal")).ok(); @@ -817,6 +841,261 @@ async fn strict_created_dir_reopens_wal_only_without_data_loss() { let _ = std::fs::remove_dir_all(&dir); } +// =========================================================================== +// (7b) MULTI-SEGMENT recovery: boot must not clobber sealed/recycled segments +// =========================================================================== + +/// Acked records that live in WAL segments AFTER the first survive a crash. +/// +/// With a small segment size, enough acked appends roll the WAL: `1.wal` is +/// SEALED (truncated to its exactly-packed length + fsync'd) and later records +/// land in `.wal`. A crash + reboot must replay ALL retained segments. +/// +/// Regression (sim seed 89837): `Shard::open` re-preallocated `1.wal` to full +/// segment size, so the sealed segment grew a zero tail; replay read that tail +/// as `Incomplete` (= end of the durable log) and silently dropped every +/// record in later segments — then `reconcile_tail` TRUNCATED the per-stream +/// files back to the stale frontier. Acked-data loss on the recovery path. +#[tokio::test] +async fn e2e_multi_segment_acked_records_after_first_seal_survive_crash() { + let _guard = DurabilityGuard::wal(); + const SEG: u64 = 4096; + let dir = tmp("multi-seg"); + + let h = Harness::boot_with_segment_size(&dir, Some(1), 1, SEG).unwrap(); + create_stream(&h.store, "s", OCTET).await; + + // Append acked records until the shard has rolled at least once (≥2 + // on-disk segments), then a few more so the post-roll segment holds data. + let mut expected = Vec::new(); + let mut i = 0usize; + while h.walset.shards()[0].wal_segments() < 2 || i < 40 { + let rec = format!("multi-seg-record-{i:04}|").into_bytes(); + append_acked(&h.store, "s", OCTET, &rec).await; + expected.extend_from_slice(&rec); + i += 1; + assert!(i < 10_000, "never rolled a segment; check SEG/record sizing"); + } + assert!( + h.walset.shards()[0].wal_segments() >= 2, + "test needs ≥2 retained segments" + ); + + h.crash(); + + let h2 = Harness::boot_with_segment_size(&dir, None, 1, SEG).unwrap(); + let got = stream_file_bytes(&h2.store, "s"); + assert_eq!( + got.len(), + expected.len(), + "every acked record recovers across ALL retained segments (lost {} bytes)", + expected.len().saturating_sub(got.len()) + ); + assert_eq!(got, expected, "recovered bytes byte-identical across segment seams"); + h2.crash(); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Acked records recover when `1.wal` was RECYCLED (checkpoint deleted it and +/// the oldest retained segment starts at lsn > 1). +/// +/// Regression (same root cause, worse case): `Shard::open` unconditionally +/// created a fresh, all-zero `1.wal`. Replay walked segments in start-lsn +/// order, began with the spurious zero-filled `1.wal`, decoded `Incomplete` at +/// offset 0, and treated that as the end of the durable log — replaying +/// NOTHING. Every acked record after the last checkpoint was truncated away. +#[tokio::test] +async fn e2e_recycled_first_segment_acked_records_survive_crash() { + let _guard = DurabilityGuard::wal(); + const SEG: u64 = 4096; + let dir = tmp("recycled-first"); + + let h = Harness::boot_with_segment_size(&dir, Some(1), 1, SEG).unwrap(); + create_stream(&h.store, "s", OCTET).await; + + // Phase 1: roll past 1.wal, then checkpoint → sealed segments fully below + // the floor (including 1.wal) are recycled (deleted). + let mut expected = Vec::new(); + let mut i = 0usize; + while h.walset.shards()[0].wal_segments() < 3 { + let rec = format!("pre-ckpt-{i:04}|").into_bytes(); + append_acked(&h.store, "s", OCTET, &rec).await; + expected.extend_from_slice(&rec); + i += 1; + assert!(i < 10_000, "never rolled; check SEG/record sizing"); + } + h.walset.shards()[0].checkpoint().await.unwrap(); + assert!( + !dir.join("wal").join("0").join("1.wal").exists(), + "checkpoint recycled 1.wal (else this test is vacuous)" + ); + + // Phase 2: more ACKED records after the checkpoint (they live only in the + // WAL + page cache; the checkpoint that would fsync them never runs). + for j in 0..25usize { + let rec = format!("post-ckpt-{j:04}|").into_bytes(); + append_acked(&h.store, "s", OCTET, &rec).await; + expected.extend_from_slice(&rec); + } + + h.crash(); + + let h2 = Harness::boot_with_segment_size(&dir, None, 1, SEG).unwrap(); + let got = stream_file_bytes(&h2.store, "s"); + assert_eq!( + got.len(), + expected.len(), + "post-checkpoint acked records recover from the retained (recycle-survivor) segments" + ); + assert_eq!(got, expected, "recovered bytes byte-identical"); + h2.crash(); + let _ = std::fs::remove_dir_all(&dir); +} + +// =========================================================================== +// (7c) WAL-QUIET stream: torn unacked tail truncated via the sidecar proof +// =========================================================================== + +/// A stream with NO durable WAL record and NO checkpoint `tails` entry (created +/// after the last checkpoint; its only append was in-flight at the crash) must +/// still have its torn, never-acked page-cache tail truncated on recovery. +/// +/// Regression (sim seed 20230): with the WAL bytes for the in-flight append +/// torn by power loss and its data-file bytes partially persisted, recovery had +/// NO truncation proof for the stream — the sidecar pass trusted +/// `tail = file size` and exposed the torn fragment to readers (the exact C1 +/// shape the WAL exists to prevent). The sidecar now persists a `durable_tail` +/// proof (fsynced at create/close, refreshed at checkpoint + recovery), and +/// recovery seeds every stream's frontier from it. +#[tokio::test] +async fn e2e_wal_quiet_stream_torn_unacked_tail_truncated() { + let _guard = DurabilityGuard::wal(); + let dir = tmp("quiet-torn"); + + let mut h = Harness::boot(&dir, Some(1), 1).unwrap(); + + // An earlier checkpointed stream so the shard's tails file is non-empty + // (proves the fix is not just "empty tails == reconcile everything"). + create_stream(&h.store, "older", OCTET).await; + append_acked(&h.store, "older", OCTET, b"older-rec|").await; + h.walset.shards()[0].checkpoint().await.unwrap(); + + // The WAL-quiet stream: created AFTER the checkpoint, never acked an append. + create_stream(&h.store, "fresh", OCTET).await; + + // Its only append is in-flight at the crash: bytes reached the data file's + // page cache and the WAL staging buffer, but the committer never fsync'd + // (stop it first), so no ack was ever released. + h.stop_committers(); + let st = h.store.get("fresh").unwrap(); + let torn: &[u8] = b"TORN-IN-FLIGHT-NEVER-ACKED"; + { + use std::io::Write; + let mut f = std::fs::OpenOptions::new().append(true).open(&st.file_path).unwrap(); + f.write_all(torn).unwrap(); + f.sync_all().unwrap(); // even fully-persisted: still un-acked, must go + } + let shard = h.walset.shard_for(st.id).clone(); + shard + .reserve_and_stage(crate::wal::codec::RecordKind::Append, st.id, 0, torn) + .unwrap(); + // Power loss tears the staged (never-fdatasync'd) WAL record: zero it out. + // Everything at/above this record was never covered by an ack. + { + use std::io::{Seek, SeekFrom, Write}; + let seg = crate::wal::segment::seg_path(&dir.join("wal").join("0"), 1); + let len = std::fs::metadata(&seg).unwrap().len(); + let mut f = std::fs::OpenOptions::new().write(true).open(&seg).unwrap(); + // The quiet stream's record is the LAST staged record; zeroing the whole + // segment suffix past the durable prefix models its loss. Find the + // offset by decoding up to the first record for `st.id`. + let bytes = std::fs::read(&seg).unwrap(); + let mut off = 0usize; + while let crate::wal::codec::Decoded::Record { stream_id, total, .. } = + crate::wal::codec::decode_at(&bytes, off) + { + if stream_id == st.id { + break; + } + off += total; + } + f.seek(SeekFrom::Start(off as u64)).unwrap(); + f.write_all(&vec![0u8; (len as usize) - off]).unwrap(); + f.sync_all().unwrap(); + } + + drop(st); + let store = h.store; + let walset = h.walset; + drop(store); + drop(walset); + + // Reopen: recovery must truncate the torn tail even though the stream has + // zero surviving WAL records and no tails entry — the sidecar's durable_tail + // proof (0, persisted at create) is the seed. + let h2 = Harness::boot(&dir, None, 1).unwrap(); + let got = stream_file_bytes(&h2.store, "fresh"); + assert_eq!( + got, + b"", + "torn un-acked tail truncated on a WAL-quiet stream (sidecar durable_tail proof)" + ); + let st2 = h2.store.get("fresh").unwrap(); + assert_eq!(st2.tail().bytes, 0, "tail reconciled to the durable frontier (0)"); + // The checkpointed stream is untouched. + assert_eq!(stream_file_bytes(&h2.store, "older"), b"older-rec|"); + h2.crash(); + let _ = std::fs::remove_dir_all(&dir); +} + +// =========================================================================== +// (7d) DELETE ack durability: an acked DELETE survives a crash +// =========================================================================== + +/// The 204 for DELETE is a durability promise. Regression (sim seed 20387): +/// `handle_delete` acked while the file + sidecar unlinks ran on a DETACHED +/// blocking task — a crash right after the ack (before the task ran) left both +/// files on disk and the stream RESURRECTED with all its data on reboot. The +/// unlinks (+ parent-dir fsync) are now awaited before the 204. +#[tokio::test] +async fn e2e_acked_delete_is_durable_no_resurrection_after_crash() { + let _guard = DurabilityGuard::wal(); + let dir = tmp("delete-durable"); + + let h = Harness::boot(&dir, Some(1), 1).unwrap(); + create_stream(&h.store, "victim", OCTET).await; + append_acked(&h.store, "victim", OCTET, b"doomed-data|").await; + let file_path = h.store.get("victim").unwrap().file_path.clone(); + let meta = crate::store::meta_path(&file_path); + + let resp = handlers::handle( + Arc::clone(&h.store), + Req { + method: Method::Delete, + path: "victim".into(), + query: None, + headers: vec![], + body: Bytes::new(), + }, + ) + .await; + assert_eq!(resp.status, 204, "delete acked"); + // The ack IS the durability point: both on-disk artifacts are already gone + // when the response returns (not on some detached task's schedule). + assert!(!file_path.exists(), "data file removed before the DELETE ack"); + assert!(!meta.exists(), "meta sidecar removed before the DELETE ack"); + + // Crash + reboot: the stream must not resurrect. + h.crash(); + let h2 = Harness::boot(&dir, None, 1).unwrap(); + assert!( + h2.store.get("victim").is_none(), + "acked-deleted stream must not resurrect after a crash" + ); + h2.crash(); + let _ = std::fs::remove_dir_all(&dir); +} + // =========================================================================== // (8) MEMORY-MODE sidecar recovery (no WAL) // =========================================================================== diff --git a/packages/durable-streams-rust/src/wal/mod.rs b/packages/durable-streams-rust/src/wal/mod.rs index e8f0ef67f6..e0ff787b9b 100644 --- a/packages/durable-streams-rust/src/wal/mod.rs +++ b/packages/durable-streams-rust/src/wal/mod.rs @@ -17,3 +17,6 @@ pub mod walset; #[cfg(test)] mod e2e_tests; + +#[cfg(test)] +mod sim_tests; diff --git a/packages/durable-streams-rust/src/wal/recovery.rs b/packages/durable-streams-rust/src/wal/recovery.rs index 7f1942e6e8..ecf4a9d3d4 100644 --- a/packages/durable-streams-rust/src/wal/recovery.rs +++ b/packages/durable-streams-rust/src/wal/recovery.rs @@ -70,9 +70,34 @@ pub fn recover(store: &Arc, wal: &Arc) -> io::Result<()> { // Build the id → StreamState index once from the sidecar-recovered streams. // Streams are keyed by NAME in the store; recovery routes by `stream_id`, so // we re-key on the stable id. Shared across shards read-only. + // + // Alongside it, build each shard's per-stream durable-frontier SEED: the + // sidecar's persisted `durable_tail` proof (`None` in old sidecars → trust + // the file size, i.e. the boot tail — the pre-field behavior). Seeding EVERY + // stream (not just those the WAL touched) is what lets recovery truncate a + // torn, never-durable page-cache tail on a stream with NO retained WAL + // record and NO checkpoint `tails` entry — e.g. a stream created after the + // last checkpoint whose only in-flight append was torn by power loss. The + // reconcile loop below skips streams whose file already sits exactly at + // their frontier, so the seeding adds no boot I/O for untouched streams. let mut index: HashMap> = HashMap::new(); + let mut seeds: Vec> = vec![HashMap::new(); wal.shards().len()]; + let shard_pos: HashMap<*const crate::wal::shard::Shard, usize> = wal + .shards() + .iter() + .enumerate() + .map(|(i, s)| (Arc::as_ptr(s), i)) + .collect(); for entry in store.streams.iter() { let st = entry.value().clone(); + let proof = { + let s = st.shared.read().unwrap(); + // `s.tail` at boot is file_base + on-disk file size (the sidecar + // pass's trust-the-file seed). + st.boot_meta_durable_tail.unwrap_or(s.tail) + }; + let pos = shard_pos[&Arc::as_ptr(wal.shard_for(st.id))]; + seeds[pos].insert(st.id, proof); index.insert(st.id, st); } let index = Arc::new(index); @@ -80,10 +105,10 @@ pub fn recover(store: &Arc, wal: &Arc) -> io::Result<()> { // Per-shard passes are independent (disjoint stream sets). Spawn one blocking // task per shard and join — the replay is synchronous file I/O. let mut handles = Vec::with_capacity(wal.shards().len()); - for shard in wal.shards() { + for (shard, seed) in wal.shards().iter().zip(seeds) { let shard = Arc::clone(shard); let index = Arc::clone(&index); - handles.push(std::thread::spawn(move || recover_shard(&shard, &index))); + handles.push(std::thread::spawn(move || recover_shard(&shard, &index, seed))); } for h in handles { // A panicked recovery thread is a bug (poisoned state); surface it. @@ -93,9 +118,13 @@ pub fn recover(store: &Arc, wal: &Arc) -> io::Result<()> { } /// Replay one shard and reconcile the tails of the streams it touched. +/// `seed` carries every one of this shard's streams' sidecar durable-tail proof +/// (see `recover`); the durable frontier per stream is the max of that seed, +/// the checkpoint-persisted tails, and the replayed record ends. fn recover_shard( shard: &crate::wal::shard::Shard, index: &HashMap>, + seed: HashMap, ) -> io::Result<()> { // `checkpoint_lsn` is a WRITE-SKIP optimization ONLY — NOT the boundary for // which streams get reconciled. We replay from the OLDEST RETAINED record so @@ -136,13 +165,23 @@ fn recover_shard( // below. Only streams with EITHER a persisted tail OR an in-range Append are // inserted, so we reconcile exactly the streams the WAL touched. let mut frontier: HashMap = shard.read_durable_tails(); - // (debug) Snapshot the persisted-tail seed and track the lowest replayed - // offset per stream, to assert that a stream reconstructed purely from - // replayed Appends covers `[file_base, frontier)` with no interior gap. - #[cfg(debug_assertions)] - let persisted_tails = frontier.clone(); + // Fold in the per-stream sidecar proof (every stream of this shard gets an + // entry; max keeps the strongest proof). + for (id, proof) in seed { + let slot = frontier.entry(id).or_insert(0); + *slot = (*slot).max(proof); + } + // Streams whose per-stream FILE was actually written by the replay below — + // their reconcile must run unconditionally (fsync the repair). + let mut replay_wrote: std::collections::HashSet = std::collections::HashSet::new(); + // (debug) Track, per replayed stream, the lowest replayed offset AND the + // file's logical end BEFORE the first replay write touched it, to assert the + // replayed records tile onto the existing durable prefix with no interior + // hole (see the debug_assert below). #[cfg(debug_assertions)] let mut min_applied: HashMap = HashMap::new(); + #[cfg(debug_assertions)] + let mut pre_replay_end: HashMap = HashMap::new(); // Captured replay error from inside the closure (the closure cannot return // `io::Result`). The first error aborts further application for this shard. let mut replay_err: Option = None; @@ -167,10 +206,18 @@ fn recover_shard( return; } let file_pos = stream_offset - file_base; + #[cfg(debug_assertions)] + pre_replay_end.entry(stream_id).or_insert_with(|| { + // Logical end of the per-stream file before replay writes to it — + // the durable prefix the replayed records must tile onto. + let len = std::fs::metadata(&st.file_path).map(|m| m.len()).unwrap_or(0); + file_base + len + }); if let Err(e) = write_at(st, file_pos, payload) { replay_err = Some(e); return; } + replay_wrote.insert(stream_id); let end = stream_offset + payload.len() as u64; let slot = frontier.entry(stream_id).or_insert(0); *slot = (*slot).max(end); @@ -185,9 +232,9 @@ fn recover_shard( return Err(e); } - // Reconcile each touched stream's file tail to its durable frontier (the max - // of the persisted durable tail and the replayed WAL frontier, already folded - // into `frontier`). + // Reconcile each stream's file tail to its durable frontier (the max of the + // sidecar durable-tail seed, the checkpoint-persisted tails, and the + // replayed WAL frontier, already folded into `frontier`). for (stream_id, &logical_tail) in &frontier { let st = match index.get(stream_id) { Some(st) => st, @@ -199,19 +246,49 @@ fn recover_shard( // underflow `logical_tail - file_base`; skip — the live file already // starts at `file_base` and its tail is governed by the sealed watermark, // not this stale recorded tail. - if logical_tail < st.shared.read().unwrap().file_base { + let (file_base, boot_tail) = { + let s = st.shared.read().unwrap(); + (s.file_base, s.tail) + }; + if logical_tail < file_base { + continue; + } + // Fast path (keeps 1M-stream boots cheap): the replay never wrote this + // stream's file and the file already sits exactly at its frontier + // (`boot_tail` is file_base + on-disk size, seeded by the sidecar pass) + // — nothing to truncate, extend, or fsync. Persist the strengthened + // proof only when the sidecar's recorded one lags the frontier (e.g. a + // checkpoint `tails` entry outran the meta): the `tails` file is wiped + // by `reset_after_recovery`, so without the bump the NEXT boot's seed + // would regress below this durable frontier and truncate acked bytes. + // Old sidecars (no recorded proof) are left as-is — they keep + // file-size trust rather than paying a boot-time meta rewrite per + // stream (the proof materializes on their next natural meta write). + if !replay_wrote.contains(stream_id) && boot_tail == logical_tail { + if st.boot_meta_durable_tail.is_some_and(|d| d < logical_tail) { + write_meta_durable(st)?; + } continue; } - // (debug) A stream rebuilt PURELY from replay (no persisted durable tail) - // must start at `file_base`; otherwise `[file_base, lowest_record)` is a - // torn interior prefix kept silently. Fail loudly in tests if a future - // compaction/`file_base` interaction breaks the contiguous-tiling premise. + // (debug) No interior HOLE: a stream's first replayed record must start + // at or below the file's pre-replay logical end. The prefix below it is + // durable via one of THREE sources — a checkpoint-persisted tail, the + // previous boot's recovery reconcile (which fdatasync'd the repaired + // file before `reset_after_recovery` wiped the old WAL — so a stream + // recovered last boot legitimately has post-boot WAL records starting + // at its recovered tail with NO persisted-tail entry yet), or the + // records replayed earlier in this pass. A first record strictly ABOVE + // the pre-replay end means `[pre_end, lowest_record)` was never written + // by anything durable — a gap kept silently. Fail loudly in tests. #[cfg(debug_assertions)] debug_assert!( - persisted_tails.contains_key(stream_id) - || min_applied.get(stream_id).copied() - == Some(st.shared.read().unwrap().file_base), - "WAL replay gap: stream {stream_id} rebuilt from replay does not start at file_base" + min_applied.get(stream_id).map_or(true, |&lo| { + pre_replay_end.get(stream_id).is_some_and(|&pre| lo <= pre) + }), + "WAL replay hole: stream {stream_id}: first replayed record at {:?} is past \ + the pre-replay file end {:?}", + min_applied.get(stream_id), + pre_replay_end.get(stream_id) ); reconcile_tail(st, logical_tail)?; } @@ -303,9 +380,21 @@ fn reconcile_tail(st: &StreamState, logical_tail: u64) -> io::Result<()> { let mut ap = st.appender.blocking_lock(); ap.written = file_len; } + // Persist the reconciled durable frontier into the sidecar BEFORE + // `reset_after_recovery` wipes the WAL and the checkpoint `tails` file. If + // the proof stayed stale (below the frontier just made durable above), the + // NEXT boot's seed would truncate this stream back below acked, durable + // bytes it can no longer re-replay. + write_meta_durable(st)?; Ok(()) } +/// Durably rewrite a stream's sidecar (recovery-time only — captures the +/// just-reconciled `Shared.durable_tail` as the next boot's truncation proof). +fn write_meta_durable(st: &StreamState) -> io::Result<()> { + crate::store::write_meta_sync(st, true) +} + /// Open a fresh read-write (non-append) handle to a stream's per-stream data file /// for positioned writes / truncation during recovery. fn open_rw(st: &StreamState) -> io::Result { @@ -797,9 +886,10 @@ mod tests { /// Drive a real shard so the committer makes records durable. Used by the 11b /// tests to checkpoint (recording per-stream durable tails) and roll/recycle. - fn spawn_committer(shard: &std::sync::Arc) -> tokio::task::JoinHandle<()> { - let s = std::sync::Arc::clone(shard); - tokio::spawn(async move { s.run_committer().await }) + fn spawn_committer( + shard: &std::sync::Arc, + ) -> crate::wal::shard::CommitterHandle { + shard.spawn_committer() } /// CRITICAL (11b): a stream X whose durable WAL records are ALL recycled @@ -886,7 +976,7 @@ mod tests { // segment below it (incl. X's original `1.wal`) is recycled. let ckpt2 = shard.checkpoint().await.unwrap(); assert_eq!(ckpt2, last); - h.abort(); + h.stop(); // X must now have NO retained WAL record: replay sees nothing for X. let mut x_records_in_wal = 0usize; @@ -1020,7 +1110,7 @@ mod tests { } let l3 = shard.reserve_and_stage(RecordKind::Append, x_id, after_r2, r3).unwrap(); shard.wait_durable(l3).await; - h.abort(); + h.stop(); // Torn page-cache tail past r3 (un-acked). { diff --git a/packages/durable-streams-rust/src/wal/segment.rs b/packages/durable-streams-rust/src/wal/segment.rs index 5c46790ab2..bd74249960 100644 --- a/packages/durable-streams-rust/src/wal/segment.rs +++ b/packages/durable-streams-rust/src/wal/segment.rs @@ -84,6 +84,22 @@ impl FileSegment { Ok(FileSegment { file }) } + /// Open an EXISTING segment without changing its size or contents. + /// + /// Boot-time (`Shard::open`) must be non-destructive: a sealed segment is + /// exactly packed (its length IS the durable-log seam recovery walks across), + /// so re-preallocating it to full size would graft a zero tail onto it and + /// recovery would mis-read that tail as the end of the durable log — dropping + /// every later segment's acked records. This constructor only opens the fd. + pub fn open_existing(path: PathBuf) -> io::Result { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .truncate(false) + .open(&path)?; + Ok(FileSegment { file }) + } + /// **Seal** this segment at a roll: truncate it to exactly `len` bytes (drop /// the unused `fallocate`'d zero tail) and `fdatasync` so its new size + data /// are durable. After this the segment is **exactly packed** — its on-disk @@ -117,8 +133,33 @@ impl FileSegment { /// preserves the ORIGINAL F_FULLFSYNC errno in context — the fallback's errno /// alone would mislead durability diagnostics. (Shared by `seal_to` and /// `FileSegment::fdatasync`; mirrors `store::barrier_fsync`.) +/// BENCH-ONLY: whether `DS_BENCH_FAST_FSYNC` requests plain `fsync` over +/// `F_FULLFSYNC` on macOS. Read once and cached. Mirrors the gate in +/// `store::barrier_fsync`. +#[cfg(target_os = "macos")] +fn fast_fsync_enabled() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("DS_BENCH_FAST_FSYNC").is_some()) +} + #[cfg(target_os = "macos")] fn macos_full_fsync(fd: libc::c_int) -> io::Result<()> { + // BENCH-ONLY (`DS_BENCH_FAST_FSYNC`): plain `fsync` instead of the + // `F_FULLFSYNC` drive barrier so the committer's hot fsync is cheap on a RAM + // disk and the per-shard LOCK becomes the bottleneck (the Linux+NVMe regime + // this build studies). NOT power-loss durable; never set in production. See + // `store::barrier_fsync` for the rationale. + if fast_fsync_enabled() { + // SAFETY: `fd` is a valid open fd for the lifetime of the call. + return unsafe { + if libc::fsync(fd) == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + }; + } // SAFETY: callers pass a valid open fd for the lifetime of the call. unsafe { if libc::fcntl(fd, libc::F_FULLFSYNC) == 0 { diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index fbad8b60c6..765d61dbf1 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -29,12 +29,14 @@ //! the invariant the committer relies on. `lsn`s start at 1; `written_high == 0` //! means "nothing contiguous yet". -use std::collections::{BTreeSet, HashMap}; +use std::cmp::Ordering as CmpOrdering; +use std::collections::{BTreeSet, BinaryHeap, HashMap}; use std::io; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; -use tokio::sync::{watch, Notify}; +use tokio::sync::oneshot; use super::codec::{encode_into, Record, RecordKind}; use super::segment::{seg_path, FileSegment, SegmentWriter, SEGMENT_BYTES}; @@ -104,13 +106,205 @@ impl ShardInner { } } +/// A single parked `wait_durable(lsn)` caller: its target `lsn`, a monotonic +/// `seq` (tie-break so equal-lsn waiters have a total order — `BinaryHeap` +/// requires `Ord`), and the `oneshot` whose fire un-parks it. +/// +/// Ordered as a **min-heap by `lsn`** (then `seq`): we reverse the natural +/// comparison so [`BinaryHeap`] (a max-heap) yields the *smallest* lsn at the +/// top. That lets [`Shard::publish_durable`] pop exactly the prefix of waiters +/// with `lsn <= watermark` and stop — the coalesced wakeup — instead of the old +/// `watch` broadcast that woke every parked subscriber. +struct DurableWaiter { + lsn: u64, + seq: u64, + tx: oneshot::Sender<()>, +} + +impl PartialEq for DurableWaiter { + fn eq(&self, other: &Self) -> bool { + self.lsn == other.lsn && self.seq == other.seq + } +} +impl Eq for DurableWaiter {} +impl PartialOrd for DurableWaiter { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for DurableWaiter { + fn cmp(&self, other: &Self) -> CmpOrdering { + // Reverse `(lsn, seq)` so the max-heap pops the LOWEST lsn first. + other + .lsn + .cmp(&self.lsn) + .then_with(|| other.seq.cmp(&self.seq)) + } +} + +/// Per-shard min-ordered registry of parked durability waiters (the coalesced +/// replacement for the `watch` broadcast). Guarded by a short `Mutex` held only +/// to push one waiter (`wait_durable`) or drain the satisfied prefix +/// (`publish_durable`). +struct WaiterReg { + heap: BinaryHeap, + /// Monotonic id handed to each waiter for the heap tie-break. + next_seq: u64, +} + +/// Stage→committer signalling for the **dedicated-OS-thread committer** (Tier-2a). +/// +/// The committer no longer lives on the shared async runtime, so the wakeup can +/// no longer be an async `tokio::sync::Notify`. This is a plain +/// `Mutex + Condvar` the sync appender (`reserve_and_stage`) signals and +/// the committer thread blocks on. +/// +/// **Lost-wakeup safety (the load-bearing invariant):** `work_pending` is set +/// **under the same mutex** the committer checks before parking, and the +/// committer always *re-snapshots the watermark off-lock after waking* — so a +/// stage that races the committer's park is never lost. Two cases: if the stage +/// takes the mutex *after* the committer parks, the condvar wakes the committer +/// (standard predicate-under-lock pattern), which then re-snapshots and commits; +/// if the stage takes the mutex *before* the committer parks, the committer sees +/// `work_pending == true` and does not park, looping to re-snapshot. +/// +/// Because the watermark is always read fresh after the park (never carried +/// across), cross-mutex ordering between `inner` (watermark) and this mutex +/// cannot drop a staged record. This is the same guarantee the old +/// `Notify`-registration-before-snapshot gave. +struct CommitSignal { + state: Mutex, + cv: Condvar, +} + +#[derive(Default)] +struct CommitState { + /// A stage happened since the committer last cleared this — there may be new + /// contiguous-written work to fsync. Set under `state`, cleared by the + /// committer under `state` before it re-snapshots the watermark. + work_pending: bool, + /// Shutdown requested: the committer must do a final drain and exit. + stop: bool, +} + +impl CommitSignal { + fn new() -> Self { + CommitSignal { + state: Mutex::new(CommitState::default()), + cv: Condvar::new(), + } + } + + /// Appender → committer (hot path): mark work pending and wake the committer. + /// The flag is set under `state` so it can never be lost against the + /// committer's park (see type docs). + fn signal_work(&self) { + { + let mut g = self.state.lock().unwrap(); + g.work_pending = true; + } + // Notify after releasing the lock so a woken committer doesn't immediately + // re-block on the mutex we still hold. + self.cv.notify_one(); + } + + /// Shutdown → committer: request a clean stop (committer drains then exits). + fn signal_stop(&self) { + { + let mut g = self.state.lock().unwrap(); + g.stop = true; + } + self.cv.notify_one(); + } + + /// Block until a stage signals work or stop is requested; clear the + /// `work_pending` flag and return whether `stop` was requested. Called by the + /// committer when it has caught up (nothing left to commit). + fn wait_for_work(&self) -> bool { + let mut g = self.state.lock().unwrap(); + while !g.work_pending && !g.stop { + g = self.cv.wait(g).unwrap(); + } + g.work_pending = false; + g.stop + } + + /// Error-backoff wait: park up to `timeout`, returning early if stop is + /// requested. Returns whether `stop` was requested. Does **not** clear + /// `work_pending` — we want to retry the same un-acked watermark, and a + /// pending stage should still drive the next wake. Using the condvar (rather + /// than `thread::sleep`) lets shutdown interrupt a long backoff. + fn backoff_wait(&self, timeout: std::time::Duration) -> bool { + let g = self.state.lock().unwrap(); + if g.stop { + return true; + } + let (g, _timed_out) = self.cv.wait_timeout(g, timeout).unwrap(); + g.stop + } +} + +/// Handle to a shard's dedicated committer OS thread. Signalling stop + joining +/// happens on `stop()` (or on `Drop`, so a handle dropped without an explicit +/// stop still shuts the thread down cleanly rather than detaching it). +pub struct CommitterHandle { + shard: Arc, + join: Option>, +} + +impl CommitterHandle { + /// Signal the committer to stop (non-blocking; does not join). Idempotent. + pub fn signal_stop(&self) { + self.shard.commit_signal.signal_stop(); + } + + /// Join the committer thread (call after `signal_stop`). Consumes the handle. + pub fn join(mut self) { + if let Some(j) = self.join.take() { + let _ = j.join(); + } + } + + /// Signal stop and join — the common single-handle path (the WalSet shutdown + /// path instead `signal_stop`s all shards then `join`s all, for a parallel + /// drain). Used by the test harnesses; allowed dead in the production binary. + #[cfg_attr(not(test), allow(dead_code))] + pub fn stop(self) { + self.signal_stop(); + self.join(); + } +} + +impl Drop for CommitterHandle { + fn drop(&mut self) { + // Ensure the thread is asked to stop and reaped even if the handle is + // dropped without an explicit `stop()` (e.g. a test guard going out of + // scope, or the WalSet being torn down). + self.shard.commit_signal.signal_stop(); + if let Some(j) = self.join.take() { + let _ = j.join(); + } + } +} + /// One WAL shard: a segmented append-only log with its own committer. pub struct Shard { inner: Mutex, - /// Publishes `durable_lsn` to `wait_durable` waiters. - durable_tx: watch::Sender, - /// Wakes the committer whenever a record is staged. - notify: Notify, + /// The highest lsn made durable + acked. Read cheaply (one relaxed-ish + /// atomic load) by `wait_durable`'s fast path, `checkpoint`, the committer, + /// and stats; advanced only by `publish_durable`. Replaces the `watch` + /// channel's value role — the WAKE role lives in `waiters` below. Storing an + /// atomic + firing oneshots both work from the dedicated committer thread + /// (Tier-2a), with no tokio runtime handle, so async `wait_durable` futures + /// are still woken across the thread boundary. + durable_lsn: AtomicU64, + /// Min-ordered (by lsn) registry of parked `wait_durable` waiters. A commit + /// drains only the waiters whose lsn the new watermark satisfies, instead of + /// broadcasting to every parked subscriber (the Tier-1c thundering-herd fix). + waiters: Mutex, + /// Wakes the dedicated committer thread whenever a record is staged (and + /// carries the shutdown signal). See [`CommitSignal`]. + commit_signal: CommitSignal, /// `/wal//` — where this shard's segments live. dir: PathBuf, /// The size each segment is `fallocate`'d to and the roll threshold. Defaults @@ -119,17 +313,45 @@ pub struct Shard { /// Immutable for the shard's lifetime, so a plain field (no lock) is enough. segment_size: u64, /// **Dirty set** (spec §7): the per-stream `StreamState`s this shard has - /// *touched* since its last checkpoint, deduped by `stream_id`. The append - /// path (`maybe_sync_on_ack`) registers the touched stream's `Arc` - /// here. `checkpoint()` drains this set, reads each stream's current logical - /// `Shared.tail` and live `Shared.file`, `fdatasync`s exactly these files — no - /// double-fsync of untouched streams — *before* recycling the WAL, and records - /// each stream's durable tail (the tail at the moment its file is fsync'd) into - /// the persisted per-shard tail map (spec §7, task 11b). Holding the - /// `StreamState` (rather than a bare `Arc`) lets checkpoint read the - /// logical tail it must record alongside the file it fsyncs. This stays - /// decoupled from `reserve_and_stage`, which never needs the `StreamState`. - dirty: Mutex>>, + /// *touched* since its last checkpoint. The append path (`maybe_sync_on_ack`) + /// registers the touched stream's `Arc` here via + /// `register_dirty`. `checkpoint()` drains this collection, reads each stream's + /// current logical `Shared.tail` and live `Shared.file`, `fdatasync`s exactly + /// these files — no double-fsync of untouched streams — *before* recycling the + /// WAL, and records each stream's durable tail (the tail at the moment its file + /// is fsync'd) into the persisted per-shard tail map (spec §7, task 11b). + /// Holding the `StreamState` (rather than a bare `Arc`) lets checkpoint + /// read the logical tail it must record alongside the file it fsyncs. This + /// stays decoupled from `reserve_and_stage`, which never needs the + /// `StreamState`. + /// + /// **Lock-free hot path (Tier-1a):** dedup is no longer done by a per-append + /// `HashMap` insert under this lock. Instead each stream carries a + /// `StreamState.dirty_epoch`; `register_dirty` only takes this lock to `push` + /// on the winning 0→current-epoch CAS transition (at most once per stream per + /// checkpoint interval, ~3 s). The already-dirty hot path never touches this + /// lock — it is a `Vec` (not a map) because the epoch CAS already guarantees + /// each stream is pushed at most once per interval, so no in-collection dedup + /// is needed (a stale-epoch race can at worst push a stream twice, costing one + /// redundant — harmless — fdatasync, never a lost stream). + dirty: Mutex>>, + /// **Current checkpoint epoch** (Tier-1a), starts at `1` (StreamStates start at + /// `dirty_epoch == 0`, so the first append always registers). `checkpoint()` + /// `fetch_add(1)`s this and drains `dirty` in the same step. A stream's + /// `register_dirty` compares the stream's `dirty_epoch` to this: equal ⇒ + /// already registered this interval (pure relaxed-load hot path, no lock); + /// otherwise CAS the stream to this epoch and, on the winning transition, push + /// it into `dirty`. Bumping BEFORE/with the drain is load-bearing: an append + /// racing the drain sees a stale stream epoch and re-registers into the next + /// interval's collection — no touched stream is ever dropped. + dirty_epoch: AtomicU64, + /// Resident copy of the CUMULATIVE per-stream durable-tail map persisted at + /// `/tails` (task 11b). `None` until the first checkpoint needs it + /// (then seeded from disk once); afterwards `persist_durable_tails` merges and + /// serializes from memory instead of re-reading + re-parsing the whole file + /// every ~3 s (O(total streams per shard) — ~20 ms/tick at 400k streams). + /// Only the (serialized, per-shard) checkpoint path locks it. + tails_cache: Mutex>>, /// Per-shard batch-size + durability counters (spec §11). Updated once per /// successful committer `fdatasync` (`record_batch`) — cheap relaxed atomics, /// no lock/alloc/syscall on the commit path. Read off-path by the 1 Hz @@ -202,9 +424,47 @@ impl Shard { /// [`WalSet::open_with_segment_size`]: super::walset::WalSet::open_with_segment_size pub fn open_with_segment_size(dir: PathBuf, segment_size: u64) -> io::Result> { std::fs::create_dir_all(&dir)?; - let seg_start_lsn = 1; - let active = Arc::new(FileSegment::create(seg_path(&dir, seg_start_lsn), segment_size)?); - let (durable_tx, _durable_rx) = watch::channel(0u64); + // Boot must be NON-DESTRUCTIVE (spec §9: recover-before-clobber): the + // on-disk segments are the durable log recovery is about to replay. + // + // - A SEALED segment was truncated to its exactly-packed length at the + // roll; that exact length is how replay walks across the segment seam. + // Re-`fallocate`ing it to full size grafts a zero tail onto it, which + // replay reads as `Incomplete` = end of the durable log — silently + // dropping every later segment's acked records (and recovery then + // truncates the per-stream files back to that stale frontier). + // - A RECYCLED `1.wal` no longer exists; creating a fresh zero-filled + // one makes replay (which walks in start-lsn order) decode zero + // records and stop before the real oldest retained segment. + // + // So: if any segment exists, open the highest-start one as a read-only + // placeholder handle and leave the disk untouched. The in-memory cursor + // (`write_pos`/`next_lsn`) is a placeholder too — `reset_after_recovery` + // MUST rebuild both (fresh `1.wal`, cursor to 0/1) before any append, + // which is the documented boot order (`open` → `recover` → `reset`). + // Only a genuinely fresh shard dir creates + preallocates `1.wal` here. + let mut existing: Vec<(u64, PathBuf)> = Vec::new(); + for entry in std::fs::read_dir(&dir)? { + let path = entry?.path(); + let Some(stem) = path + .file_name() + .and_then(|s| s.to_str()) + .and_then(|n| n.strip_suffix(".wal")) + else { + continue; + }; + if let Ok(start) = stem.parse::() { + existing.push((start, path)); + } + } + existing.sort_by_key(|(s, _)| *s); + let (seg_start_lsn, active) = match existing.pop() { + Some((start, path)) => (start, Arc::new(FileSegment::open_existing(path)?)), + None => ( + 1, + Arc::new(FileSegment::create(seg_path(&dir, 1), segment_size)?), + ), + }; Ok(std::sync::Arc::new(Shard { inner: Mutex::new(ShardInner { active, @@ -215,11 +475,19 @@ impl Shard { written_high: 0, written_ahead: BTreeSet::new(), }), - durable_tx, - notify: Notify::new(), + durable_lsn: AtomicU64::new(0), + waiters: Mutex::new(WaiterReg { + heap: BinaryHeap::new(), + next_seq: 0, + }), + commit_signal: CommitSignal::new(), dir, segment_size, - dirty: Mutex::new(HashMap::new()), + dirty: Mutex::new(Vec::new()), + // Epoch starts at 1; StreamStates start at dirty_epoch 0, so the first + // append on every stream registers it (0 != 1). + dirty_epoch: AtomicU64::new(1), + tails_cache: Mutex::new(None), stats: ShardStats::default(), #[cfg(test)] on_stage: Mutex::new(None), @@ -346,7 +614,13 @@ impl Shard { // --- Phase 1: (maybe roll, then) reserve under the short lock. --- let (lsn, off, seg) = { - let mut g = self.inner.lock().unwrap(); + // Time the contended `inner` acquisition (--wal-stats only). This is + // the headline per-shard write-serialization signal: every stream on + // this shard funnels through this one lock. + let mut g = super::telemetry::timed_lock( + |ns| self.stats.record_inner_lock_wait(ns), + || self.inner.lock().unwrap(), + ); // Segment roll (spec §4): if this record would overflow the active // segment's `fallocate`'d region, SEAL the current segment (truncate to @@ -407,25 +681,72 @@ impl Shard { seg.write_at(off, &buf)?; { - let mut g = self.inner.lock().unwrap(); + let mut g = super::telemetry::timed_lock( + |ns| self.stats.record_inner_lock_wait(ns), + || self.inner.lock().unwrap(), + ); g.mark_written(lsn); } - self.notify.notify_one(); + self.stats.record_staged(); + // Wake the dedicated committer thread. `work_pending` is set under the + // signal mutex (after `mark_written` above), so this can never be lost + // against the committer's park (see `CommitSignal` docs). + self.commit_signal.signal_work(); Ok(lsn) } /// Register a touched stream's `Arc` into this shard's dirty set /// (spec §7). Called from the append path (`maybe_sync_on_ack`) BEFORE staging /// the WAL record (register-before-stage, CQ-1), since that is where the - /// stream's `Arc` is in hand. Deduped by `stream_id`: re-touching - /// a stream just refreshes the handle, so `checkpoint()` reads each touched - /// stream's current tail + `fdatasync`s its file exactly once. + /// stream's `Arc` is in hand. + /// + /// **Lock-free hot path (Tier-1a).** Dedup is by epoch, not by a per-append + /// map insert under the shard's `dirty` lock: /// - /// `reserve_and_stage` itself stays ignorant of `StreamState` (it only ever - /// needs `stream_id`); the `StreamState` is needed solely by `checkpoint`, to - /// read the durable tail it records and the file it fsyncs. - pub fn register_dirty(&self, stream_id: u64, st: Arc) { - self.dirty.lock().unwrap().insert(stream_id, st); + /// - **Hot path (already dirty this interval):** one relaxed load of the shard + /// epoch + one relaxed load of the stream's `dirty_epoch`; if equal, return + /// immediately. No lock, no push, no clock read, no allocation — this is the + /// per-append common case (a stream is appended to many times between two + /// ~3 s checkpoints). + /// - **Transition (first touch this interval):** CAS the stream's `dirty_epoch` + /// to the current shard epoch. The unique winner of the 0/stale→epoch + /// transition pushes the `Arc` into the shard's `dirty` Vec. + /// Because that happens at most once per stream per checkpoint interval, the + /// Vec's `Mutex` is off the hot path — taken only on the transition. + /// + /// The contention stat (`record_dirty_lock_wait`) is still wired, but only + /// around the rare transition lock, so `dirty_wait_load` collapses toward ~0 + /// (the hot path records nothing). `reserve_and_stage` stays ignorant of + /// `StreamState`; the `StreamState` is needed solely by `checkpoint`. + pub fn register_dirty(&self, _stream_id: u64, st: Arc) { + let epoch = self.dirty_epoch.load(Ordering::Relaxed); + // Hot path: already registered for the current checkpoint interval. Pure + // relaxed loads + a branch — never touches the `dirty` lock. + if st.dirty_epoch.load(Ordering::Relaxed) == epoch { + return; + } + // Transition: claim this stream for the current epoch. Only the thread that + // wins the CAS (0/stale → epoch) pushes; a concurrent racer that lost (or + // already advanced the stream to `epoch`) does nothing. AcqRel so the push + // below is ordered after the claim is published. `compare_exchange` (not + // `_weak`) — we must not spuriously fail and double-push. + let cur = st.dirty_epoch.load(Ordering::Relaxed); + if cur == epoch { + return; + } + if st + .dirty_epoch + .compare_exchange(cur, epoch, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + // Off-hot-path: time the (now rare) dirty-set lock so `dirty_wait_load` + // still reports the residual transition-only contention (≈0). + let mut g = super::telemetry::timed_lock( + |ns| self.stats.record_dirty_lock_wait(ns), + || self.dirty.lock().unwrap(), + ); + g.push(st); + } } /// **Checkpoint** (spec §7), per shard, non-blocking w.r.t. acks: @@ -450,9 +771,9 @@ impl Shard { /// re-registering a stream is never lost (it lands in the *next* checkpoint). /// /// Returns the `checkpoint_lsn` persisted. - pub async fn checkpoint(&self) -> io::Result { + pub async fn checkpoint(self: &Arc) -> io::Result { // 1. Snapshot the recycle floor = the highest durably-acked lsn. - let checkpoint_lsn = *self.durable_tx.borrow(); + let checkpoint_lsn = self.durable_lsn.load(Ordering::Acquire); // 2. Drain the dirty set atomically, then fdatasync each touched file. // Draining first means a concurrent append re-registers into a fresh @@ -464,57 +785,124 @@ impl Shard { // file's cache, so the file is durable up to AT LEAST this tail // afterwards (a later concurrent append only extends past it and lands // in the next checkpoint). Recording this tail is conservative-safe. - let touched: Vec<(u64, u64, Arc)> = { + // LOAD-BEARING ORDERING (Tier-1a): bump the checkpoint epoch and drain + // the dirty Vec in the SAME `dirty`-lock critical section that + // `register_dirty`'s push also takes. This serialization is what makes + // the lock-free hot path safe: a stream registered for the OLD epoch is + // in the Vec we take (and keeps its old `dirty_epoch`, so its next append + // re-registers into the new interval); a stream whose `register_dirty` + // observes the bumped epoch necessarily pushes AFTER we release this + // lock, landing in the fresh post-take Vec for the next checkpoint. So no + // touched stream is ever both drained here AND silently skipped next time. + // The critical section is O(1) — take the Vec and bump the epoch, nothing + // else. The per-stream tail/file capture below runs AFTER the lock is + // released: at high stream cardinality nearly every append is its + // stream's first touch of the interval (the transition path), so any + // O(touched) work under this lock stalls every appender on the shard + // for the whole drain (measured 25–140 ms per tick at 400k streams). + let drained: Vec> = { let mut g = self.dirty.lock().unwrap(); + self.dirty_epoch.fetch_add(1, Ordering::AcqRel); std::mem::take(&mut *g) - .into_iter() - .map(|(id, st)| { + }; + + // Everything below is file IO + O(touched)/O(total-streams) CPU — run it + // in ONE blocking task so none of it ever stalls an async worker thread + // (at 400k streams the capture+fsync+tails phases are tens of ms per tick + // each; on a real disk the fsync fan-out can be far worse). Ordering + // inside the closure is exactly the required hard ordering: capture tails + // → fdatasync per-stream files → persist tails map → persist + // checkpoint_lsn → recycle. Acks never gate on any of this. + let this = Arc::clone(self); + tokio::task::spawn_blocking(move || -> io::Result { + // Phase timing for the `WAL_CKPT` line (`--wal-stats`). One clock + // read per phase, once per ~3 s per shard — nowhere near the hot path. + let t_start = std::time::Instant::now(); + // Capture each touched stream's current logical tail and live file. + // The tail is read BEFORE the fsync; the fdatasync flushes every page + // already in the file's cache, so the file is durable up to AT LEAST + // this tail afterwards (a later concurrent append only extends past + // it and lands in the next checkpoint). Conservative-safe. + let touched: Vec<(u64, u64, Arc)> = drained + .iter() + .map(|st| { let s = st.shared.read().unwrap(); - (id, s.tail, Arc::clone(&s.file)) + (st.id, s.tail, Arc::clone(&s.file)) }) - .collect() - }; - // Run the (potentially blocking) fdatasyncs off the async runtime so a - // slow disk can't stall this task's executor thread. Ordering preserved: - // we await all per-stream fsyncs before touching the WAL. We move only the - // files into the blocking task; the (stream_id, durable_tail) pairs stay - // here to merge into the persisted tail map after the fsyncs succeed. - let tails: Vec<(u64, u64)> = touched.iter().map(|(id, tail, _)| (*id, *tail)).collect(); - let to_sync: Vec> = - touched.into_iter().map(|(_, _, f)| f).collect(); - tokio::task::spawn_blocking(move || -> io::Result<()> { - for f in &to_sync { + .collect(); + let n_touched = touched.len(); + let t_capture = t_start.elapsed(); + + // 2. fdatasync each touched per-stream file. + for (_, _, f) in &touched { crate::store::barrier_fsync(f)?; } - Ok(()) + let t_fsync = t_start.elapsed(); + + // 3a. Persist the CUMULATIVE per-stream durable-tail map (task 11b) + // AFTER the per-stream files are fsync'd, and BEFORE recycle — + // same hard ordering as `checkpoint_lsn`. Merge this checkpoint's + // touched tails into the resident map so a stream touched in an + // earlier checkpoint (but not this one) keeps its last durable + // tail. `tmp` + rename + fsync makes the map itself crash-durable, + // so when recycle deletes the WAL records below the floor, + // recovery can still truncate a recycled stream's torn + // per-stream-file tail to its durable tail. + let tails: Vec<(u64, u64)> = + touched.iter().map(|(id, tail, _)| (*id, *tail)).collect(); + let n_tails = this.persist_durable_tails(&tails)?; + let t_tails = t_start.elapsed(); + + // 3b. Persist checkpoint_lsn (durably) AFTER the per-stream files are + // fsync'd, so the recorded floor only ever covers bytes already on + // disk in their own files. + let ckpt_path = this.dir.join(CHECKPOINT_FILE); + let tmp = this.dir.join(format!("{CHECKPOINT_FILE}.tmp")); + std::fs::write(&tmp, checkpoint_lsn.to_string())?; + std::fs::rename(&tmp, &ckpt_path)?; + + // 4. Recycle: unlink WAL segments fully below checkpoint_lsn. This is + // the LAST step — strictly after the per-stream fsyncs AND the + // durable-tail map persist above. + this.recycle_below(checkpoint_lsn)?; + let t_rest = t_start.elapsed(); + + // 5. Flush the meta sidecar of every touched stream whose append + // path marked it dirty (WAL mode defers the per-append debounced + // flush to here — see handle_append_inner). Strictly AFTER the + // WAL-critical sequence above: sidecar producer/access state is + // non-durable by contract and plays no part in WAL replay, so it + // must never delay the recycle floor. Errors are ignored exactly + // like the debounced flush ignored them. + let mut n_meta = 0u64; + for st in &drained { + if st.meta_dirty.swap(false, Ordering::AcqRel) { + let _ = crate::store::write_meta_sync(st, false); + n_meta += 1; + } + } + + if super::telemetry::stats_enabled() { + let t_total = t_start.elapsed(); + eprintln!( + "WAL_CKPT shard={} touched={} tails_entries={} meta={} capture_us={} fsync_us={} tails_us={} rest_us={} meta_us={} total_us={}", + this.dir.file_name().and_then(|s| s.to_str()).unwrap_or("?"), + n_touched, + n_tails, + n_meta, + t_capture.as_micros(), + (t_fsync - t_capture).as_micros(), + (t_tails - t_fsync).as_micros(), + (t_rest - t_tails).as_micros(), + (t_total - t_rest).as_micros(), + t_total.as_micros(), + ); + } + + Ok(checkpoint_lsn) }) .await - .expect("checkpoint fdatasync task panicked")?; - - // 3a. Persist the CUMULATIVE per-stream durable-tail map (task 11b) AFTER - // the per-stream files are fsync'd, and BEFORE recycle — same hard - // ordering as `checkpoint_lsn`. Merge this checkpoint's touched tails - // into the previously-persisted map so a stream touched in an earlier - // checkpoint (but not this one) keeps its last durable tail. `tmp` + - // rename + fsync makes the map itself crash-durable, so when recycle - // deletes the WAL records below the floor, recovery can still truncate - // a recycled stream's torn per-stream-file tail to its durable tail. - self.persist_durable_tails(&tails)?; - - // 3b. Persist checkpoint_lsn (durably) AFTER the per-stream files are - // fsync'd, so the recorded floor only ever covers bytes already on - // disk in their own files. - let ckpt_path = self.dir.join(CHECKPOINT_FILE); - let tmp = self.dir.join(format!("{CHECKPOINT_FILE}.tmp")); - std::fs::write(&tmp, checkpoint_lsn.to_string())?; - std::fs::rename(&tmp, &ckpt_path)?; - - // 4. Recycle: unlink WAL segments fully below checkpoint_lsn. This is the - // LAST step — strictly after the per-stream fsyncs AND the durable-tail - // map persist above. - self.recycle_below(checkpoint_lsn)?; - - Ok(checkpoint_lsn) + .expect("checkpoint task panicked") } /// Merge `touched` `(stream_id, durable_tail)` pairs into the persisted @@ -524,28 +912,40 @@ impl Shard { /// the WAL is recycled, so a torn per-stream-file tail can always be truncated /// to its durable tail even after its WAL records are gone (task 11b). /// - /// Cumulative-merge: read the existing map, overwrite each touched stream's - /// entry with its newest durable tail (`max`, so a re-checkpointed earlier tail - /// can never regress the recorded value), keep every untouched stream's last - /// recorded tail. - fn persist_durable_tails(&self, touched: &[(u64, u64)]) -> io::Result<()> { + /// Cumulative-merge: merge into the RESIDENT map (`tails_cache`, seeded from + /// disk once on first use), overwrite each touched stream's entry with its + /// newest durable tail (`max`, so a re-checkpointed earlier tail can never + /// regress the recorded value), keep every untouched stream's last recorded + /// tail. Serializing from memory avoids re-reading + re-parsing the whole + /// file every checkpoint (O(total streams per shard) each ~3 s). + /// Returns the number of entries in the persisted map (for `WAL_CKPT`). + fn persist_durable_tails(&self, touched: &[(u64, u64)]) -> io::Result { if touched.is_empty() && !self.dir.join(TAILS_FILE).exists() { // Nothing touched and no prior map: nothing to persist. - return Ok(()); + return Ok(0); } - let mut map = Self::read_durable_tails_at(&self.dir); + let mut cache = self.tails_cache.lock().unwrap(); + let map = cache.get_or_insert_with(|| Self::read_durable_tails_at(&self.dir)); for &(id, tail) in touched { let slot = map.entry(id).or_insert(0); *slot = (*slot).max(tail); } // Serialize as `stream_id durable_tail` lines (sorted for a deterministic, // diff-friendly file). Plain decimal text, matching the `checkpoint` file. - let mut entries: Vec<(u64, u64)> = map.into_iter().collect(); + let mut entries: Vec<(u64, u64)> = map.iter().map(|(&k, &v)| (k, v)).collect(); entries.sort_unstable(); + let n = entries.len(); let mut body = String::with_capacity(entries.len() * 16); - for (id, tail) in entries { - body.push_str(&format!("{id} {tail}\n")); + { + use std::fmt::Write as _; + for (id, tail) in entries { + let _ = writeln!(body, "{id} {tail}"); + } } + // The resident map is fully merged and serialized; release it before the + // file IO below (nothing else contends today, but don't hold a lock over + // a write+fsync+rename gratuitously). + drop(cache); let path = self.dir.join(TAILS_FILE); let tmp = self.dir.join(format!("{TAILS_FILE}.tmp")); std::fs::write(&tmp, &body)?; @@ -553,7 +953,7 @@ impl Shard { // crash-durable BEFORE recycle (the whole point of 11b). std::fs::File::open(&tmp)?.sync_all()?; std::fs::rename(&tmp, &path)?; - Ok(()) + Ok(n) } /// Read the persisted per-shard durable-tail map from `/tails`. Returns @@ -775,12 +1175,12 @@ impl Shard { self.stats.snapshot() } - /// Current `durable_lsn` — the highest lsn made durable + acked. Cheap - /// (`watch::borrow`), read by the emitter and by [`Shard::checkpoint`]. + /// Current `durable_lsn` — the highest lsn made durable + acked. Cheap (one + /// atomic load), read by the emitter and by [`Shard::checkpoint`]. /// Telemetry/test-only as a public accessor (see [`Shard::wal_size_bytes`]). #[cfg_attr(not(any(feature = "telemetry", test)), allow(dead_code))] pub fn durable_lsn_now(&self) -> u64 { - *self.durable_tx.borrow() + self.durable_lsn.load(Ordering::Acquire) } /// The shard's `tail_lsn` — the highest lsn **assigned** so far (`next_lsn - 1`, @@ -793,19 +1193,51 @@ impl Shard { } /// Await until this shard's `durable_lsn >= lsn`. + /// + /// Coalesced wakeup (Tier-1c): instead of subscribing to a broadcast that + /// every commit wakes, we register a single `oneshot` keyed by `lsn` in the + /// min-ordered [`WaiterReg`]. [`Shard::publish_durable`] fires only the + /// waiters whose lsn the new watermark satisfies — so a parked waiter is + /// woken at most once, exactly when its lsn becomes durable. + /// + /// **Cross-thread wakeup (Tier-2a):** `publish_durable` runs on the dedicated + /// committer OS thread; `oneshot::Sender::send` fires the receiver's waker + /// from any thread, so an async `wait_durable` future parked on the runtime is + /// woken across the thread→async boundary with no tokio handle required. + /// + /// **Lost-wakeup safety (the register/publish race):** `publish_durable` + /// stores `durable_lsn` (Release) *before* it locks the heap to drain; we + /// push our waiter *before* re-loading `durable_lsn` (Acquire). The heap + /// `Mutex` orders the two heap critical sections, so for any interleaving + /// either (a) our push precedes the commit's drain — it fires our oneshot — + /// or (b) the commit's drain precedes our push — but then its `durable_lsn` + /// store happens-before our re-check load, which sees it and returns. No + /// commit can land in the gap between the check and the registration and + /// leave us parked forever. pub async fn wait_durable(&self, lsn: u64) { - let mut rx = self.durable_tx.subscribe(); - if *rx.borrow_and_update() >= lsn { + // Fast path: already durable — return without touching the registry. + if self.durable_lsn.load(Ordering::Acquire) >= lsn { return; } - loop { - if rx.changed().await.is_err() { - return; - } - if *rx.borrow_and_update() >= lsn { - return; - } + let rx = { + let mut reg = self.waiters.lock().unwrap(); + let (tx, rx) = oneshot::channel(); + let seq = reg.next_seq; + reg.next_seq = reg.next_seq.wrapping_add(1); + reg.heap.push(DurableWaiter { lsn, seq, tx }); + rx + }; + // Re-check AFTER registering to close the race where a commit landed + // between the fast-path load and the push (see the doc comment). If it is + // now durable, return; our heap entry's receiver is dropped, so a later + // drain firing it is a silent no-op. + if self.durable_lsn.load(Ordering::Acquire) >= lsn { + return; } + // Park until our oneshot fires. A `RecvError` (sender dropped without + // firing) only happens at shard teardown — treat as "return", matching + // the old `watch::changed()`-errored behaviour. + let _ = rx.await; } /// Current contiguous-written watermark (`written_high`). Snapshot this @@ -829,37 +1261,102 @@ impl Shard { /// is the committer's Ok-branch; callers MUST pass a watermark snapshotted /// BEFORE the covering fsync (never re-snapshot afterwards). pub fn publish_durable(&self, watermark: u64) { - let durable = *self.durable_tx.borrow(); + let durable = self.durable_lsn.load(Ordering::Acquire); if watermark <= durable { return; } self.stats.record_batch(watermark - durable); - // Use send_replace (unconditional write) so the watermark is stored even - // when no receivers exist yet — `durable_tx` sits at 0 receivers between - // appends because the constructor drops `_durable_rx` and a waiter only - // subscribes inside `wait_durable` AFTER `reserve_and_stage` returns. - // `send()` silently no-ops when receiver_count == 0, which would drop the - // durable watermark; a waiter that subscribes afterwards reads a stale - // value and waits forever for an already-durable record. - self.durable_tx.send_replace(watermark); + // Publish the new watermark (Release) BEFORE draining the waiter heap. + // This ordering is load-bearing for lost-wakeup safety: a `wait_durable` + // that registers concurrently re-checks `durable_lsn` after pushing, and + // the heap `Mutex` guarantees it either observes this store or is drained + // below (see `wait_durable`'s doc comment). Storing the atomic + firing the + // oneshots below both work from the dedicated committer OS thread (Tier-2a) + // with no tokio runtime handle. + self.durable_lsn.store(watermark, Ordering::Release); + // Coalesced wakeup (Tier-1c): under the short lock, pop ONLY the prefix of + // waiters whose lsn <= the new watermark and fire their oneshots. The heap + // is min-ordered by lsn, so the first entry above the watermark stops the + // drain — every still-parked waiter has a strictly higher lsn. This + // replaces the `watch` broadcast that woke every parked subscriber (most + // of which re-checked and immediately re-parked — the thundering herd). + let mut fired = 0u64; + { + let mut reg = self.waiters.lock().unwrap(); + while let Some(top) = reg.heap.peek() { + if top.lsn > watermark { + break; + } + let w = reg.heap.pop().expect("peek just confirmed a top entry"); + // `send` fails silently if the receiver was dropped (a cancelled + // wait_durable — timeout/connection drop); count only live waiters + // actually woken, so `waiters_woken_avg` reflects real wakeups. + if w.tx.send(()).is_ok() { + fired += 1; + } + } + } + // Record how many waiters this commit ACTUALLY woke — now just the few + // satisfied by this watermark, not every parked subscriber. This is the + // proof of the coalescing: `waiters_woken_avg` collapses toward ~1. + self.stats.record_waiters_woken(fired); let mut g = self.inner.lock().unwrap(); g.sealed_pending.retain(|(end_lsn, _)| *end_lsn > watermark); } - /// The shard's group-commit committer: wait for staged work, `fdatasync` the - /// active segment, advance `durable_lsn` to the contiguous-written watermark, - /// and publish it to waiters. Runs forever (the caller `abort`s it). + /// One group-commit attempt: snapshot the contiguous-written watermark, + /// `fdatasync` the segments that cover it (sealed-pending first, then active), + /// then publish it as `durable_lsn`. Returns `Ok(Some(watermark))` when an + /// advance was published, `Ok(None)` when already caught up (nothing to do), + /// and `Err` when an fsync failed. + /// + /// **No-loss / watermark invariants (load-bearing):** the watermark is + /// snapshotted **before** the covering fsync and published **exactly** (never + /// re-snapshotted after the fsync — see [`Shard::publish_durable`]). On fsync + /// error we return `Err` and do **not** publish, so `durable_lsn` never + /// advances over un-fsync'd bytes (records stay un-acked). `sealed_pending` + /// segments are fsync'd before `publish_durable` retires them. + /// + /// Runs the `fdatasync` **synchronously on the committer's own OS thread** — + /// no `spawn_blocking` round-trip onto the shared runtime (the Tier-2a win). + fn commit_once(&self) -> io::Result> { + let watermark = self.snapshot_watermark(); + let durable = self.durable_lsn.load(Ordering::Acquire); + if watermark <= durable { + return Ok(None); + } + let (seg, sealed) = self.collect_fsync_targets(); + for s in &sealed { + s.fdatasync()?; + } + seg.fdatasync()?; + // Snapshotted-before-fsync watermark, published exactly. + self.publish_durable(watermark); + Ok(Some(watermark)) + } + + /// The shard's group-commit committer, run on a **dedicated OS thread** (one + /// per shard, spawned by [`Shard::spawn_committer`]) — off the shared async + /// runtime, so committers don't time-share the network/reactor worker threads + /// and pay no per-commit `spawn_blocking` hop (Tier-2a). It blocks on the + /// [`CommitSignal`] condvar for staged work, `fdatasync`s synchronously, and + /// advances/publishes `durable_lsn`. Firing the per-waiter `oneshot`s from + /// `publish_durable` (driven from this thread) still wakes async + /// `wait_durable` futures across the thread→async boundary. + /// + /// **Lost-wakeup safety:** when caught up we park in `wait_for_work`, which + /// checks `work_pending` **under the same mutex** a stage sets it under, and + /// we always re-snapshot the watermark off-lock after waking. A stage racing + /// the park is therefore never lost (see [`CommitSignal`]). /// - /// **Lost-wakeup safety:** we register the `Notified` future *before* - /// snapshotting the watermark. `Notify` stores one permit, so a `notify_one` - /// racing between our snapshot and the `await` is captured by the already- - /// registered future and returns immediately on the next iteration — no - /// staged record can sit un-committed waiting for a wakeup that already fired. + /// **fsync-error path:** on `fdatasync` failure we do **not** advance + /// `durable_lsn` (no ack) and back off with bounded exponential delay + /// (interruptible by shutdown), exactly as the no-loss invariant requires. /// - /// **fsync-error path:** if `fdatasync` fails we do **not** advance - /// `durable_lsn` and do **not** publish — the staged records stay un-acked, - /// exactly as the no-loss invariant requires (spec §6). - pub async fn run_committer(self: std::sync::Arc) { + /// **Shutdown:** on a stop signal the loop performs a **final drain** + /// (commits everything already contiguous-written, so in-flight commits are + /// not dropped) and then returns so the thread can be joined. + pub fn run_committer(&self) { // Backoff state for the fsync-error path: a persistently failing disk // (ENOSPC/EIO/read-only volume) must not busy-spin a core, hammer the disk, // and flood stderr. The no-loss invariant holds throughout — `durable_lsn` @@ -870,71 +1367,121 @@ impl Shard { let mut backoff = RETRY_BACKOFF_MIN; let mut consecutive_errors: u64 = 0; loop { - // Register interest BEFORE reading state (lost-wakeup safety). - let notified = self.notify.notified(); - - let watermark = self.snapshot_watermark(); - let durable = *self.durable_tx.borrow(); - - if watermark > durable { - let (seg, sealed) = self.collect_fsync_targets(); - let fsync_res: io::Result<()> = tokio::task::spawn_blocking(move || { - for s in &sealed { - s.fdatasync()?; + match self.commit_once() { + Ok(Some(_)) => { + // Advanced — re-snapshot immediately; more may have arrived + // while we were fsyncing (group commit naturally batches them). + consecutive_errors = 0; + backoff = RETRY_BACKOFF_MIN; + continue; + } + Ok(None) => { + // Caught up — reset the error backoff and park for the next + // stage (or a stop signal). + consecutive_errors = 0; + backoff = RETRY_BACKOFF_MIN; + if self.commit_signal.wait_for_work() { + // Stop requested: drain any records that became + // contiguous-written before/at the stop, then exit. We do + // NOT retry fsync errors forever here — on error we log and + // give up (no-loss holds: durable_lsn is not advanced, so + // the un-drained tail simply stays un-acked, exactly as a + // crash would leave it). + loop { + match self.commit_once() { + Ok(Some(_)) => continue, + Ok(None) => break, + Err(e) => { + eprintln!( + "WAL committer final-drain fdatasync failed: {e}" + ); + break; + } + } + } + return; } - seg.fdatasync()?; - Ok(()) - }) - .await - .unwrap_or_else(|e| Err(io::Error::other(format!("committer fsync task panicked: {e}")))); - match fsync_res { - Ok(()) => { - self.publish_durable(watermark); - consecutive_errors = 0; - backoff = RETRY_BACKOFF_MIN; - continue; + } + Err(e) => { + // Rate-limited log + bounded exponential backoff. The backoff + // parks on the condvar (not a raw sleep) so shutdown can + // interrupt it; a stream of concurrent stages cannot turn the + // retry into a hot loop (we do not clear `work_pending` here, + // so the same un-acked watermark is retried). + consecutive_errors += 1; + if consecutive_errors == 1 || consecutive_errors % LOG_EVERY == 0 { + eprintln!( + "WAL committer fdatasync failed (attempt {consecutive_errors}): {e}" + ); } - Err(e) => { - // Rate-limited log + bounded exponential backoff. We SLEEP - // (rather than await `notified`) so a stream of concurrent - // appends — each leaving a `Notify` permit — cannot turn the - // retry into a hot loop. A recovered disk costs at most one - // backoff interval of extra latency. Dropping the registered - // `notified` here is safe: an un-polled `Notified` does not - // consume the stored permit, so no stage wakeup is lost. - consecutive_errors += 1; - if consecutive_errors == 1 || consecutive_errors % LOG_EVERY == 0 { - eprintln!( - "WAL committer fdatasync failed (attempt {consecutive_errors}): {e}" - ); - } - tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(RETRY_BACKOFF_MAX); - continue; + if self.commit_signal.backoff_wait(backoff) { + // Stop requested during backoff: exit without acking the + // failing watermark (no-loss preserved). + return; } + backoff = (backoff * 2).min(RETRY_BACKOFF_MAX); } } + } + } - // Nothing new to commit — caught up, so reset the error backoff and - // wait for the next stage. - consecutive_errors = 0; - backoff = RETRY_BACKOFF_MIN; - notified.await; + /// Spawn this shard's committer on a dedicated OS thread and return a + /// [`CommitterHandle`] for clean shutdown. The thread holds an `Arc` + /// clone so it stays alive until stopped + joined. + pub fn spawn_committer(self: &Arc) -> CommitterHandle { + let me = Arc::clone(self); + let join = std::thread::Builder::new() + .name("wal-committer".to_string()) + .spawn(move || me.run_committer()) + .expect("spawn WAL committer thread"); + CommitterHandle { + shard: Arc::clone(self), + join: Some(join), } } /// Test-only: current `durable_lsn`. #[cfg(test)] pub fn durable_lsn(&self) -> u64 { - *self.durable_tx.borrow() + self.durable_lsn.load(Ordering::Acquire) + } + + /// Test-only: number of currently-parked durability waiters in the registry. + /// Lets the coalesced-wakeup tests assert that the fast path registers no + /// waiter and that a commit drains exactly the satisfied prefix. + #[cfg(test)] + pub fn waiter_count(&self) -> usize { + self.waiters.lock().unwrap().heap.len() } - /// Test-only: whether `stream_id` is currently in the dirty set. Used to - /// prove the append path registers a stream BEFORE its lsn can become + /// Test-only: whether `stream_id` is currently in the dirty set (i.e. its + /// `Arc` is present in the shard's pending dirty collection). Used + /// to prove the append path registers a stream BEFORE its lsn can become /// durable (the checkpoint recycle-before-fsync ordering invariant, spec §7). + /// Updated for the Tier-1a representation: a scan of the dirty `Vec` (a stream + /// is pushed at most once per checkpoint interval), not a `HashMap` lookup. #[cfg(test)] pub fn is_dirty(&self, stream_id: u64) -> bool { - self.dirty.lock().unwrap().contains_key(&stream_id) + self.dirty + .lock() + .unwrap() + .iter() + .any(|st| st.id == stream_id) + } + + /// Test-only: number of entries currently in the dirty collection. Because the + /// Tier-1a epoch CAS pushes each stream at most once per checkpoint interval, + /// this counts distinct touched streams (modulo a rare drain-race duplicate) — + /// used to prove the already-dirty hot path does NOT re-push. + #[cfg(test)] + pub fn dirty_len(&self) -> usize { + self.dirty.lock().unwrap().len() + } + + /// Test-only: the shard's current checkpoint epoch (Tier-1a). + #[cfg(test)] + pub fn dirty_epoch_now(&self) -> u64 { + self.dirty_epoch.load(Ordering::Relaxed) } /// Test-only: arm the next `reserve_and_stage` to simulate a `write_at` @@ -1022,10 +1569,7 @@ mod tests { const SEG: u64 = 4096; let dir = tmp("roll-multi"); let sh = Shard::open_with_segment_size(dir.clone(), SEG).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); // ~200-byte records → ~20 per 4 KiB segment; 80 records ⇒ ≥3 segments. let payload = vec![b'x'; 200 - crate::wal::codec::HEADER_LEN]; @@ -1034,7 +1578,7 @@ mod tests { last = sh.reserve_and_stage(RecordKind::Append, 1, i * 200, &payload).unwrap(); } sh.wait_durable(last).await; - h.abort(); + h.stop(); let segs = segs_on_disk(&dir); assert!(segs.len() >= 3, "rolled to ≥3 segments, got {}", segs.len()); @@ -1070,10 +1614,7 @@ mod tests { const SEG: u64 = 4096; let dir = tmp("roll-durable"); let sh = Shard::open_with_segment_size(dir.clone(), SEG).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let payload = vec![b'y'; 150]; let mut last = 0; for i in 0..100u64 { @@ -1082,7 +1623,7 @@ mod tests { sh.wait_durable(last).await; assert_eq!(sh.durable_lsn(), last, "every record across rolls is durable"); assert!(segs_on_disk(&dir).len() >= 3, "spanned ≥3 segments"); - h.abort(); + h.stop(); let _ = std::fs::remove_dir_all(&dir); } @@ -1093,10 +1634,7 @@ mod tests { const SEG: u64 = 4096; let dir = tmp("roll-replay"); let sh = Shard::open_with_segment_size(dir.clone(), SEG).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let mut expect: Vec<(u64, u64, Vec)> = Vec::new(); let mut last = 0; for i in 0..120u64 { @@ -1106,7 +1644,7 @@ mod tests { expect.push((last, i * 7, p)); } sh.wait_durable(last).await; - h.abort(); + h.stop(); assert!(segs_on_disk(&dir).len() >= 3, "spanned ≥3 segments"); // Replay in lsn order across segments; collect every record. @@ -1132,10 +1670,7 @@ mod tests { const SEG: u64 = 4096; let dir = tmp("roll-recycle"); let sh = Shard::open_with_segment_size(dir.clone(), SEG).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let payload = vec![b'z'; 180]; let mut last = 0; for i in 0..120u64 { @@ -1156,7 +1691,7 @@ mod tests { assert_eq!(after.len(), 1, "all sealed segments recycled below the floor"); assert_eq!(after[0].0, active_start, "the active segment is retained"); assert!(seg_path(&dir, active_start).exists(), "active segment file remains"); - h.abort(); + h.stop(); let _ = std::fs::remove_dir_all(&dir); } @@ -1173,10 +1708,7 @@ mod tests { assert_eq!(total, 64); const SEG: u64 = 128; // exactly 2 records per segment let sh = Shard::open_with_segment_size(dir.clone(), SEG).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); // 5 records: r1,r2 fill seg0 exactly (write_pos 64 then 128 == SEG, no roll // on r2 since 64+64==128 is NOT > 128). r3 rolls (128+64 > 128). r4 fills // the new seg. r5 rolls again. @@ -1185,7 +1717,7 @@ mod tests { last = sh.reserve_and_stage(RecordKind::Append, 5, i, &payload).unwrap(); } sh.wait_durable(last).await; - h.abort(); + h.stop(); let segs = segs_on_disk(&dir); assert_eq!(segs.len(), 3, "r1r2 | r3r4 | r5 ⇒ 3 segments"); @@ -1227,16 +1759,13 @@ mod tests { let l1 = sh.reserve_and_stage(RecordKind::Append, 1, 0, b"a").unwrap(); let _l2 = sh.reserve_only(); // #[cfg(test)] hook: assigns lsn, no write let l3 = sh.reserve_and_stage(RecordKind::Append, 1, 2, b"c").unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); sh.wait_durable(l1).await; // give the committer a beat to (incorrectly) over-advance if the watermark is broken tokio::time::sleep(std::time::Duration::from_millis(50)).await; assert!(sh.durable_lsn() >= l1, "l1 (and its contiguous prefix) is durable"); assert!(sh.durable_lsn() < l3, "MUST NOT advance past the unwritten l2 gap to l3"); - h.abort(); + h.stop(); } #[tokio::test] @@ -1256,17 +1785,14 @@ mod tests { // advance durable_lsn past the gap at lsn 1. let l2 = sh.reserve_and_stage(RecordKind::Append, 1, 4, b"ok").unwrap(); assert_eq!(l2, 2, "the failed stage still consumed lsn 1 (it stays a gap)"); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); tokio::time::sleep(std::time::Duration::from_millis(50)).await; assert_eq!( sh.durable_lsn(), 0, "durable_lsn cannot advance past the unwritten (failed) lsn-1 gap" ); - h.abort(); + h.stop(); } #[tokio::test] @@ -1316,10 +1842,7 @@ mod tests { // A new append now lands at lsn 1 / offset 0 into the clean segment; the // committer makes it durable and nothing past it decodes as a record. - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let lsn = sh.reserve_and_stage(RecordKind::Append, 1, 0, b"new").unwrap(); assert_eq!(lsn, 1, "fresh WAL starts at lsn 1"); sh.wait_durable(lsn).await; @@ -1337,7 +1860,7 @@ mod tests { } other => panic!("fresh record did not decode: {other:?}"), } - h.abort(); + h.stop(); let _ = std::fs::remove_dir_all(&dir); } @@ -1347,10 +1870,7 @@ mod tests { let sh = Shard::open(dir.clone()).unwrap(); // Spawn the committer so staged records become durable (advances durable_lsn). - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); // Build a real stream via the store so the dirty set can hold its // `Arc` (checkpoint reads `Shared.tail` + `Shared.file`). @@ -1415,7 +1935,7 @@ mod tests { assert!(seg_path(&dir, 1).exists(), "active segment never recycled"); let _ = l1; - h.abort(); + h.stop(); let _ = std::fs::remove_dir_all(&dir); } @@ -1426,10 +1946,7 @@ mod tests { // as records accumulate (a lagging checkpoint only delays recycling). let dir = tmp("ckpt-nonblock"); let sh = Shard::open(dir.clone()).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let size_before = sh.wal_size_bytes(); let mut last = 0; @@ -1448,7 +1965,7 @@ mod tests { ); assert!(seg_path(&dir, 1).exists(), "WAL segment retained (not recycled)"); - h.abort(); + h.stop(); let _ = std::fs::remove_dir_all(&dir); } @@ -1469,12 +1986,9 @@ mod tests { assert_eq!(sh.durable_lsn_now(), 0, "nothing durable until the committer runs"); // Now run the committer: one fdatasync should make all K durable at once. - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); sh.wait_durable(last).await; - h.abort(); + h.stop(); let snap = sh.stats_snapshot(); assert_eq!(snap.fsync_count, 1, "a single fdatasync committed the whole batch"); @@ -1496,10 +2010,7 @@ mod tests { let dir = tmp("shard-pos"); let sh = Shard::open(dir.clone()).unwrap(); - let h = tokio::spawn({ - let s = sh.clone(); - async move { s.run_committer().await } - }); + let h = sh.spawn_committer(); let mut lsns = Vec::new(); let mut payloads = Vec::new(); @@ -1512,7 +2023,7 @@ mod tests { let last = *lsns.last().unwrap(); sh.wait_durable(last).await; assert!(sh.durable_lsn() >= last, "durable_lsn reached the last staged lsn"); - h.abort(); + h.stop(); // Every record's bytes are on disk and decode correctly, back-to-back. let raw = std::fs::read(seg_path(&dir, 1)).unwrap(); @@ -1531,6 +2042,48 @@ mod tests { } } + #[tokio::test] + async fn dedicated_thread_committer_makes_durable_and_stops_cleanly() { + // The committer runs on its OWN dedicated OS thread (Tier-2a), NOT the + // tokio runtime. It must (a) make staged records durable while running, + // and (b) on a stop signal perform a FINAL DRAIN so even a batch staged + // immediately before the stop becomes durable, then exit so the thread + // joins cleanly (no detach, no hang). + let dir = tmp("dedicated-thread"); + let sh = Shard::open(dir.clone()).unwrap(); + let h = sh.spawn_committer(); + + // (a) Records made durable while the committer thread runs. + let mut last = 0; + for i in 0..10u64 { + last = sh.reserve_and_stage(RecordKind::Append, 1, i, b"live").unwrap(); + } + sh.wait_durable(last).await; + assert!(sh.durable_lsn() >= last, "records durable while committer runs"); + + // (b) Stage a final batch, then stop WITHOUT awaiting durability — the + // committer's final drain must still make them durable before it exits. + // `reserve_and_stage` returns only after `mark_written`, so by the time + // this loop finishes every record is contiguous-written; the final drain + // snapshots that watermark and fsyncs it. + let mut last2 = last; + for i in 0..5u64 { + last2 = sh + .reserve_and_stage(RecordKind::Append, 1, 100 + i, b"tail") + .unwrap(); + } + // `stop()` signals stop and JOINS the thread — blocks until it has drained + // and exited. + h.stop(); + assert_eq!( + sh.durable_lsn(), + last2, + "final drain on shutdown made the just-staged batch durable before the thread exited" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + /// CQ-1 invariant: `register_dirty` must happen-before `reserve_and_stage`. /// /// At the moment `reserve_and_stage` begins (before any lsn is assigned), @@ -1606,4 +2159,228 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + /// Tier-1a hot path: re-registering an ALREADY-dirty stream within the same + /// checkpoint epoch must NOT re-push into the dirty collection. The first + /// `register_dirty` wins the 0→epoch CAS and pushes once; every subsequent + /// touch of that stream (the common per-append case) is a pure relaxed-load + /// branch that returns without taking the `dirty` lock or growing the Vec. + #[tokio::test] + async fn register_dirty_is_idempotent_within_an_epoch() { + let dir = tmp("dirty-idem"); + let sh = Shard::open(dir.clone()).unwrap(); + let store = crate::store::Store::new_with_tier( + dir.clone(), + crate::tier::TierConfig::default(), + ) + .unwrap(); + let st = match store.create("idem-stream", ckpt_test_cfg(), None, 0).unwrap() { + crate::store::CreateResult::Created(s) => s, + _ => panic!("create failed"), + }; + let sid = st.id; + + assert_eq!(sh.dirty_len(), 0, "precondition: empty dirty set"); + + // First touch this epoch: wins the CAS, pushes exactly once. + sh.register_dirty(sid, Arc::clone(&st)); + assert_eq!(sh.dirty_len(), 1, "first registration pushes once"); + assert!(sh.is_dirty(sid)); + + // Many re-touches in the SAME epoch: every one is the hot path (no push). + for _ in 0..1000 { + sh.register_dirty(sid, Arc::clone(&st)); + } + assert_eq!( + sh.dirty_len(), + 1, + "re-registering an already-dirty stream in the same epoch must NOT re-push" + ); + + // A DIFFERENT stream in the same epoch is a distinct first-touch → one push. + let st2 = match store.create("idem-stream-2", ckpt_test_cfg(), None, 0).unwrap() { + crate::store::CreateResult::Created(s) => s, + _ => panic!("create failed"), + }; + sh.register_dirty(st2.id, Arc::clone(&st2)); + assert_eq!(sh.dirty_len(), 2, "a distinct stream still registers once"); + + let _ = std::fs::remove_dir_all(&dir); + } + + /// Tier-1a no-loss: a stream re-touched AFTER a checkpoint drains the dirty set + /// must re-register (the checkpoint bumped the epoch, so the stream's stale + /// `dirty_epoch` no longer matches) and therefore land in the NEXT checkpoint. + /// This is the property that makes the lock-free hot path safe: draining never + /// silently swallows a stream's subsequent appends. + #[tokio::test] + async fn touched_after_checkpoint_drain_lands_in_next_checkpoint() { + let dir = tmp("dirty-next"); + let sh = Shard::open(dir.clone()).unwrap(); + let h = sh.spawn_committer(); + let store = crate::store::Store::new_with_tier( + dir.clone(), + crate::tier::TierConfig::default(), + ) + .unwrap(); + let st = match store.create("next-stream", ckpt_test_cfg(), None, 0).unwrap() { + crate::store::CreateResult::Created(s) => s, + _ => panic!("create failed"), + }; + let sid = st.id; + + let epoch0 = sh.dirty_epoch_now(); + + // Interval 1: append + register, make the record durable so checkpoint has + // a non-zero floor, then checkpoint (drains the stream, bumps the epoch). + sh.register_dirty(sid, Arc::clone(&st)); + let l1 = sh.reserve_and_stage(RecordKind::Append, sid, 0, b"first").unwrap(); + // Reflect a logical tail so checkpoint records it (file already exists). + st.shared.write().unwrap().tail = 5; + sh.wait_durable(l1).await; + assert!(sh.is_dirty(sid), "registered in interval 1"); + + sh.checkpoint().await.unwrap(); + assert_eq!( + sh.dirty_epoch_now(), + epoch0 + 1, + "checkpoint bumped the epoch" + ); + assert!(!sh.is_dirty(sid), "checkpoint drained the dirty set"); + assert_eq!(sh.dirty_len(), 0, "dirty set empty after drain"); + + // Interval 2: the SAME stream is touched again after the drain. Its + // dirty_epoch is stale (== old epoch), so register_dirty must re-push it. + sh.register_dirty(sid, Arc::clone(&st)); + assert!( + sh.is_dirty(sid), + "a stream touched after the drain re-registers (not lost)" + ); + assert_eq!(sh.dirty_len(), 1, "re-registered into the next interval's set"); + + // The next checkpoint drains it again — proving the post-drain append is + // covered, never dropped. + let l2 = sh.reserve_and_stage(RecordKind::Append, sid, 5, b"second").unwrap(); + st.shared.write().unwrap().tail = 11; + sh.wait_durable(l2).await; + sh.checkpoint().await.unwrap(); + assert!(!sh.is_dirty(sid), "second checkpoint drained the re-registration"); + assert_eq!(sh.dirty_epoch_now(), epoch0 + 2, "epoch bumped again"); + + h.stop(); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Tier-1c (a): a waiter for lsn=K is woken EXACTLY when durable reaches >= K, + /// and NEVER before. We drive `publish_durable` directly (no committer) to set + /// the watermark precisely: a commit to K-1 must leave the waiter parked; the + /// commit to K must wake it. + #[tokio::test] + async fn waiter_woken_only_when_durable_reaches_its_lsn() { + use std::time::Duration; + let sh = Shard::open(tmp("wd-exact")).unwrap(); + + let waiter = tokio::spawn({ + let s = sh.clone(); + async move { s.wait_durable(10).await } + }); + // Let it register and park. + tokio::time::sleep(Duration::from_millis(30)).await; + assert_eq!(sh.waiter_count(), 1, "waiter parked while durable (0) < lsn (10)"); + assert!(!waiter.is_finished(), "waiter must not be woken before its lsn"); + + // Advance durable BELOW the waiter's lsn: must NOT wake it. + sh.publish_durable(9); + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(!waiter.is_finished(), "durable=9 < lsn=10 ⇒ still parked"); + assert_eq!(sh.waiter_count(), 1, "waiter not drained below its lsn"); + + // Reach the waiter's lsn: it must wake now. + sh.publish_durable(10); + tokio::time::timeout(Duration::from_millis(500), waiter) + .await + .expect("waiter woken when durable reaches its lsn") + .unwrap(); + assert_eq!(sh.waiter_count(), 0, "the satisfied waiter was drained"); + } + + /// Tier-1c (b): committing watermark=K wakes EVERY waiter with lsn <= K in one + /// pass but leaves every lsn > K parked — and `waiters_woken` counts only the + /// satisfied ones (the coalescing proof: not a broadcast to all parked). + #[tokio::test] + async fn commit_wakes_only_satisfied_waiters_and_counts_them() { + use std::time::Duration; + let sh = Shard::open(tmp("wd-coalesce")).unwrap(); + + let lo = tokio::spawn({ + let s = sh.clone(); + async move { s.wait_durable(3).await } + }); + let mid = tokio::spawn({ + let s = sh.clone(); + async move { s.wait_durable(5).await } + }); + let hi = tokio::spawn({ + let s = sh.clone(); + async move { s.wait_durable(8).await } + }); + tokio::time::sleep(Duration::from_millis(30)).await; + assert_eq!(sh.waiter_count(), 3, "all three waiters parked"); + + // Commit watermark = 5: wakes lsn 3 and 5; lsn 8 stays parked. + sh.publish_durable(5); + tokio::time::timeout(Duration::from_millis(500), lo) + .await + .expect("lsn=3 woken (<= watermark 5)") + .unwrap(); + tokio::time::timeout(Duration::from_millis(500), mid) + .await + .expect("lsn=5 woken (== watermark 5)") + .unwrap(); + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(!hi.is_finished(), "lsn=8 stays parked above the watermark"); + assert_eq!(sh.waiter_count(), 1, "only the unsatisfied (lsn=8) waiter remains"); + + // Coalescing proof: this commit woke exactly the 2 satisfied waiters, not + // all 3 parked subscribers (the old broadcast would have recorded 3). + assert_eq!( + sh.stats_snapshot().waiters_woken, + 2, + "waiters_woken counts only the satisfied prefix, not every parked waiter" + ); + + // Advancing to 8 finally wakes the last one. + sh.publish_durable(8); + tokio::time::timeout(Duration::from_millis(500), hi) + .await + .expect("lsn=8 woken once durable reaches it") + .unwrap(); + assert_eq!(sh.waiter_count(), 0, "all waiters drained"); + } + + /// Tier-1c (c): a waiter whose lsn is ALREADY durable returns immediately via + /// the fast path, WITHOUT registering in the heap (no park, no oneshot). + #[tokio::test] + async fn already_durable_waiter_returns_without_parking() { + use std::time::Duration; + let sh = Shard::open(tmp("wd-fast")).unwrap(); + + // Make lsn 5 durable up front (no waiters registered yet → fires nothing). + sh.publish_durable(5); + assert_eq!(sh.waiter_count(), 0, "no waiters before the fast-path call"); + + // A wait for an already-durable lsn must return immediately and register + // nothing in the heap. + tokio::time::timeout(Duration::from_millis(200), sh.wait_durable(3)) + .await + .expect("already-durable (3 <= 5) waiter returns without parking"); + tokio::time::timeout(Duration::from_millis(200), sh.wait_durable(5)) + .await + .expect("already-durable (5 == 5) waiter returns without parking"); + assert_eq!( + sh.waiter_count(), + 0, + "the fast path registers no waiter in the heap" + ); + } } diff --git a/packages/durable-streams-rust/src/wal/sim_tests.rs b/packages/durable-streams-rust/src/wal/sim_tests.rs new file mode 100644 index 0000000000..2757648c3d --- /dev/null +++ b/packages/durable-streams-rust/src/wal/sim_tests.rs @@ -0,0 +1,1083 @@ +//! Randomized crash/recovery simulation — a seeded fuzz harness for the WAL +//! recovery path. Design: docs/superpowers/specs/2026-07-02-wal-crash-simulation-design.md +//! +//! Each seed drives the REAL handler path (`handlers::handle`) with a random +//! workload (creates, appends, cancelled appends, closes, forks, deletes, +//! checkpoints), then simulates a crash by dropping the generation's tokio +//! runtime (aborting in-flight requests at whatever await point they reached), +//! stopping the committers, and optionally injecting disk faults constrained to +//! the documented fault model: +//! +//! - per-stream data files are fsynced only at checkpoint / recovery repair, so +//! any byte past a stream's known-fsynced floor may be truncated, scribbled, +//! or zero-extended (power loss / torn page writeback); +//! - WAL segment bytes belonging to records with `lsn > durable_lsn` at crash +//! time were never fdatasync'd (no ack was released for them), so a random +//! suffix of that region may be zeroed or scribbled. +//! +//! It then re-runs the real boot sequence (`WalSet::open` → sidecar pass → +//! `wal::recovery::recover` → `reset_after_recovery`) and checks the oracle: +//! every ACKED record is present, in order, whole; maybe-applied (cancelled / +//! in-flight-at-crash) records are each present-whole or absent; no torn or +//! foreign bytes; tails/file_base/closed-ness consistent; deleted streams stay +//! deleted; recovery itself never errors or panics. Multiple generations per +//! seed continue the workload on the recovered store to catch +//! recovery-of-recovery bugs. +//! +//! Reproduce a failure with the seed printed in the panic message: +//! `DS_SIM_SEED0= DS_SIM_SEEDS=1 cargo test crash_recovery_randomized`. +//! Scale the hunt with `DS_SIM_SEEDS` / `DS_SIM_GENS` / `DS_SIM_STEPS`. + +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; + +use crate::api::{Method, Req}; +use crate::handlers; +use crate::handlers::test_support::DurabilityGuard; +use crate::store::Store; +use crate::tier::TierConfig; +use crate::wal::codec::{decode_at, Decoded, RecordKind}; +use crate::wal::shard::CommitterHandle; +use crate::wal::walset::WalSet; + +/// Small segments so the workload rolls + recycles segments (the recovery paths +/// with the most history of bugs). Must comfortably exceed the largest payload +/// (8 KiB) + header. +const SEG_BYTES: u64 = 32 * 1024; + +// --------------------------------------------------------------------------- +// Deterministic PRNG (splitmix64) — no dev-dependency on `rand`. +// --------------------------------------------------------------------------- + +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed) + } + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + if n == 0 { + 0 + } else { + self.next_u64() % n + } + } + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } + fn alnum(&mut self, len: usize) -> String { + const CS: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + (0..len) + .map(|_| CS[self.below(CS.len() as u64) as usize] as char) + .collect() + } +} + +// --------------------------------------------------------------------------- +// Oracle model +// --------------------------------------------------------------------------- + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum RecStatus { + /// 2xx ack observed — must survive every crash, byte-identical, in order. + Acked, + /// Cancelled / in-flight at crash / staged-at-crash — may be present + /// (whole) or absent, never torn. + Maybe, +} + +struct Rec { + wire: Vec, + status: RecStatus, + seq: u64, +} + +struct SModel { + name: String, + json: bool, + file_base: u64, + file_path: PathBuf, + id: u64, + recs: Vec, + next_seq: u64, + closed: bool, + deleted: bool, + has_children: bool, + /// Any `Maybe` rec not yet resolved by a recovery (blocks forking, whose + /// offset must be a known record boundary). + unsettled: bool, + /// Logical tail known fsynced into the data file (checkpoint or recovery + /// repair). Data-file faults may only touch bytes at/after this point. + floor: u64, +} + +impl SModel { + /// Record-boundary offsets (logical): file_base, then after each rec. + /// Only meaningful when `!unsettled` (every rec known present). + fn boundaries(&self) -> Vec { + let mut v = vec![self.file_base]; + let mut acc = self.file_base; + for r in &self.recs { + acc += r.wire.len() as u64; + v.push(acc); + } + v + } +} + +// --------------------------------------------------------------------------- +// Request builders (same shapes as e2e_tests.rs) +// --------------------------------------------------------------------------- + +fn put_req(path: &str, content_type: &str, extra: &[(&str, &str)]) -> Req { + let mut headers = vec![("content-type".to_string(), content_type.to_string())]; + for (k, v) in extra { + headers.push((k.to_string(), v.to_string())); + } + Req { + method: Method::Put, + path: path.to_string(), + query: None, + headers, + body: Bytes::new(), + } +} + +fn post_req(path: &str, content_type: &str, body: &[u8], extra: &[(&str, &str)]) -> Req { + let mut headers = vec![("content-type".to_string(), content_type.to_string())]; + for (k, v) in extra { + headers.push((k.to_string(), v.to_string())); + } + Req { + method: Method::Post, + path: path.to_string(), + query: None, + headers, + body: Bytes::copy_from_slice(body), + } +} + +fn delete_req(path: &str) -> Req { + Req { + method: Method::Delete, + path: path.to_string(), + query: None, + headers: vec![], + body: Bytes::new(), + } +} + +fn fork_offset(bytes: u64) -> String { + format!("{:016}_{:016}", 0, bytes) +} + +const OCTET: &str = "application/octet-stream"; +const JSON_CT: &str = "application/json"; + +// --------------------------------------------------------------------------- +// Simulation driver +// --------------------------------------------------------------------------- + +struct Sim { + seed: u64, + rng: Rng, + dir: PathBuf, + shards_n: usize, + models: Vec, + name_ctr: u64, + gen: u64, + log: Vec, +} + +impl Sim { + fn note(&mut self, s: String) { + self.log.push(s); + } + + fn fail(&self, msg: String) -> ! { + let tail: Vec<&String> = self.log.iter().rev().take(40).collect(); + let mut trace = String::new(); + for l in tail.into_iter().rev() { + trace.push_str(l); + trace.push('\n'); + } + panic!( + "\n=== SIM ORACLE VIOLATION ===\nseed={} gen={} dir={}\n{}\n--- last steps ---\n{}", + self.seed, + self.gen, + self.dir.display(), + msg, + trace + ); + } + + /// Build the next record for stream `mi` and return (payload, wire). + /// `wire` is the exact on-disk/read-surface byte image (JSON gets the `,` + /// delimiter `encode_wire` appends). + fn make_record(&mut self, mi: usize) -> (Vec, Vec) { + let seq = self.models[mi].next_seq; + self.models[mi].next_seq += 1; + let r = self.rng.below(100); + let fill_len = if r < 60 { + self.rng.below(120) as usize + } else if r < 90 { + 120 + self.rng.below(800) as usize + } else if r < 99 { + 1024 + self.rng.below(4096) as usize + } else { + 8192 + }; + let filler = self.rng.alnum(fill_len); + let name = self.models[mi].name.clone(); + if self.models[mi].json { + let payload = format!("{{\"s\":\"{name}\",\"i\":\"{seq:08}\",\"p\":\"{filler}\"}}").into_bytes(); + let mut wire = payload.clone(); + wire.push(b','); + (payload, wire) + } else { + let payload = format!("{name}#{seq:08}|{filler}|").into_bytes(); + (payload.clone(), payload) + } + } + + fn push_rec(&mut self, mi: usize, wire: Vec, status: RecStatus) { + let seq = self.models[mi].next_seq - 1; + if status == RecStatus::Maybe { + self.models[mi].unsettled = true; + } + self.models[mi].recs.push(Rec { wire, status, seq }); + } + + fn pick(&mut self, pred: impl Fn(&SModel) -> bool) -> Option { + let idxs: Vec = self + .models + .iter() + .enumerate() + .filter(|(_, m)| pred(m)) + .map(|(i, _)| i) + .collect(); + if idxs.is_empty() { + None + } else { + Some(idxs[self.rng.below(idxs.len() as u64) as usize]) + } + } +} + +/// Boot the exact main.rs WAL startup sequence for `dir` (spec §9). +fn boot( + dir: &std::path::Path, + shards: Option, + shards_n: usize, +) -> (Arc, Arc, Vec) { + let walset = WalSet::open_with_segment_size(dir, shards, shards_n, SEG_BYTES) + .expect("WalSet::open must succeed"); + let store = Arc::new( + Store::new_with_tier(dir.to_path_buf(), TierConfig::default()).expect("Store::new"), + ); + crate::wal::recovery::recover(&store, &walset) + .expect("recovery must not error on in-model faults"); + walset.reset_after_recovery().expect("reset_after_recovery"); + store + .wal + .set(Arc::clone(&walset)) + .unwrap_or_else(|_| panic!("WAL already attached")); + let mut committers = Vec::new(); + for shard in walset.shards() { + committers.push(shard.spawn_committer()); + } + (store, walset, committers) +} + +// --------------------------------------------------------------------------- +// Oracle verification +// --------------------------------------------------------------------------- + +fn escape_snippet(b: &[u8]) -> String { + b.iter() + .take(120) + .map(|&c| { + if (0x20..0x7f).contains(&c) { + (c as char).to_string() + } else { + format!("\\x{c:02x}") + } + }) + .collect() +} + +/// Match the recovered file bytes against the model. Returns per-rec presence. +/// Greedy in-issue-order pass (Acked must match; Maybe may skip), then a +/// permutation pass for the crash-tail: leftover bytes must be some ordering of +/// skipped Maybe wires. Errors describe the violation. +fn match_stream(model: &SModel, bytes: &[u8]) -> Result, String> { + let mut present = vec![false; model.recs.len()]; + let mut cursor = 0usize; + let mut skipped: Vec = Vec::new(); + for (i, rec) in model.recs.iter().enumerate() { + if bytes[cursor..].starts_with(&rec.wire) { + cursor += rec.wire.len(); + present[i] = true; + } else { + match rec.status { + RecStatus::Acked => { + return Err(format!( + "LOST/CORRUPT ACKED record seq={} ({} bytes) at file cursor {}.\n\ + expected: {}\n\ + found: {}", + rec.seq, + rec.wire.len(), + cursor, + escape_snippet(&rec.wire), + escape_snippet(&bytes[cursor.min(bytes.len())..]) + )); + } + RecStatus::Maybe => skipped.push(i), + } + } + } + // Crash-tail permutation: consume any remaining bytes as whole skipped + // Maybe records, in any order. + while cursor < bytes.len() { + let mut matched = false; + for &i in &skipped { + if !present[i] && bytes[cursor..].starts_with(&model.recs[i].wire) { + cursor += model.recs[i].wire.len(); + present[i] = true; + matched = true; + break; + } + } + if !matched { + return Err(format!( + "TORN/FOREIGN bytes at file cursor {} ({} bytes remain of {} total):\n{}", + cursor, + bytes.len() - cursor, + bytes.len(), + escape_snippet(&bytes[cursor..]) + )); + } + } + Ok(present) +} + +/// Full post-recovery oracle over every model. Updates each model to the +/// settled post-recovery truth (present Maybe → Acked; absent Maybe dropped; +/// floor = recovered frontier, which recovery fdatasync'd). +fn verify_recovery(sim: &mut Sim, store: &Arc) { + for mi in 0..sim.models.len() { + let (name, deleted) = (sim.models[mi].name.clone(), sim.models[mi].deleted); + if deleted { + if let Some(st) = store.get(&name) { + let soft = st.shared.read().unwrap().soft_deleted; + if !soft { + sim.fail(format!("stream {name}: DELETED stream resurrected after recovery")); + } + } + continue; + } + let Some(st) = store.get(&name) else { + sim.fail(format!("stream {name}: acked-created stream MISSING after recovery")); + }; + let bytes = std::fs::read(&st.file_path).unwrap_or_else(|e| { + sim.fail(format!("stream {name}: cannot read data file after recovery: {e}")) + }); + + // File/base/tail consistency. + let (file_base, durable_tail, tail) = { + let s = st.shared.read().unwrap(); + (s.file_base, s.durable_tail, s.tail) + }; + if file_base != sim.models[mi].file_base { + sim.fail(format!( + "stream {name}: file_base changed across recovery: expected {}, got {file_base}", + sim.models[mi].file_base + )); + } + if tail != file_base + bytes.len() as u64 { + sim.fail(format!( + "stream {name}: Shared.tail {tail} != file_base {file_base} + file_len {}", + bytes.len() + )); + } + if durable_tail != tail { + sim.fail(format!( + "stream {name}: durable_tail {durable_tail} != tail {tail} after recovery" + )); + } + let watch_tail = st.tail(); + if watch_tail.bytes != tail { + sim.fail(format!( + "stream {name}: watch tail {} != Shared.tail {tail} after recovery", + watch_tail.bytes + )); + } + + // Closed-ness durability. + if sim.models[mi].closed && !watch_tail.closed { + sim.fail(format!( + "stream {name}: close was ACKED but stream recovered open" + )); + } + + // Content: acked ⊆ recovered ⊆ issued, in order, whole records only. + let present = match match_stream(&sim.models[mi], &bytes) { + Ok(p) => p, + Err(e) => sim.fail(format!("stream {name}: {e}")), + }; + + // JSON read-surface validity. + if sim.models[mi].json && !bytes.is_empty() { + let text = String::from_utf8(bytes.clone()).unwrap_or_else(|_| { + sim.fail(format!("stream {name}: JSON stream contains non-UTF8 bytes")) + }); + let wrapped = format!("[{}]", text.trim_end_matches(',')); + if let Err(e) = serde_json::from_str::(&wrapped) { + sim.fail(format!( + "stream {name}: recovered JSON read surface invalid: {e}\n{}", + escape_snippet(&bytes) + )); + } + } + + // Settle the model to post-recovery truth. + let m = &mut sim.models[mi]; + let mut kept = Vec::new(); + for (i, mut r) in std::mem::take(&mut m.recs).into_iter().enumerate() { + if present[i] { + r.status = RecStatus::Acked; // durable now (recovery fdatasync'd the repair) + kept.push(r); + } + } + m.recs = kept; + m.unsettled = false; + m.floor = file_base + bytes.len() as u64; + m.id = st.id; + m.file_path = st.file_path.clone(); + } +} + +// --------------------------------------------------------------------------- +// Fault injection (crash-time, constrained to the fault model) +// --------------------------------------------------------------------------- + +/// Data-file power-loss faults: any byte at/after the stream's fsynced floor +/// may be truncated away, scribbled, or the file zero-extended (size-metadata +/// committed before data pages). +fn inject_data_file_faults(sim: &mut Sim) { + for mi in 0..sim.models.len() { + if sim.models[mi].deleted || !sim.rng.chance(35) { + continue; + } + let path = sim.models[mi].file_path.clone(); + let Ok(meta) = std::fs::metadata(&path) else { continue }; + let len = meta.len(); + let floor_pos = sim.models[mi].floor.saturating_sub(sim.models[mi].file_base); + let kind = sim.rng.below(3); + let name = sim.models[mi].name.clone(); + match kind { + 0 => { + // Truncate to a random point in [floor_pos, len]. + if len <= floor_pos { + continue; + } + let to = floor_pos + sim.rng.below(len - floor_pos + 1); + let f = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + f.set_len(to).unwrap(); + sim.note(format!("FAULT data-trunc {name}: {len} -> {to} (floor_pos {floor_pos})")); + } + 1 => { + // Scribble garbage over a random subrange of [floor_pos, len). + if len <= floor_pos { + continue; + } + let start = floor_pos + sim.rng.below(len - floor_pos); + let max = (len - start).min(256); + let glen = 1 + sim.rng.below(max) as usize; + let garbage: Vec = (0..glen).map(|_| sim.rng.next_u64() as u8).collect(); + use std::io::{Seek, SeekFrom, Write}; + let mut f = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + f.seek(SeekFrom::Start(start)).unwrap(); + f.write_all(&garbage).unwrap(); + sim.note(format!( + "FAULT data-scribble {name}: [{start}, {}) of len {len} (floor_pos {floor_pos})", + start + glen as u64 + )); + } + _ => { + // Zero-fill a subrange of the un-fsynced region: the size (inode) + // update was committed but those data pages never flushed. The + // file's LENGTH never grows — power loss cannot create bytes + // beyond the maximum length ever written. + if len <= floor_pos { + continue; + } + let start = floor_pos + sim.rng.below(len - floor_pos); + let zlen = 1 + sim.rng.below(len - start) as usize; + use std::io::{Seek, SeekFrom, Write}; + let mut f = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + f.seek(SeekFrom::Start(start)).unwrap(); + f.write_all(&vec![0u8; zlen]).unwrap(); + sim.note(format!( + "FAULT data-zero {name}: [{start}, {}) of len {len} (floor_pos {floor_pos})", + start + zlen as u64 + )); + } + } + } +} + +/// WAL suffix faults: bytes of records with `lsn > durable` were never +/// fdatasync'd (nothing gated on them was acked), so a random suffix of that +/// region may be zeroed (lost pages) or scribbled (torn write). +fn inject_wal_faults(sim: &mut Sim, durable: &[u64]) { + for (i, &d) in durable.iter().enumerate() { + if !sim.rng.chance(50) { + continue; + } + let shard_dir = sim.dir.join("wal").join(i.to_string()); + let Ok(rd) = std::fs::read_dir(&shard_dir) else { continue }; + let mut segs: Vec<(u64, PathBuf)> = rd + .flatten() + .filter_map(|e| { + let p = e.path(); + let stem = p.file_stem()?.to_str()?.parse::().ok()?; + (p.extension()?.to_str()? == "wal").then_some((stem, p)) + }) + .collect(); + segs.sort(); + let Some((_, seg_path)) = segs.last() else { continue }; + let Ok(bytes) = std::fs::read(seg_path) else { continue }; + + // Walk records: find the byte range [fault_from, logical_end) holding + // records with lsn > durable. + let mut off = 0usize; + let mut fault_from: Option = None; + while let Decoded::Record { lsn, total, .. } = decode_at(&bytes, off) { + if lsn > d && fault_from.is_none() { + fault_from = Some(off); + } + off += total; + } + let logical_end = off; + let Some(from) = fault_from else { continue }; + if from >= logical_end { + continue; + } + let p = from as u64 + sim.rng.below((logical_end - from) as u64 + 1); + if p as usize >= logical_end { + continue; + } + use std::io::{Seek, SeekFrom, Write}; + let mut f = std::fs::OpenOptions::new().write(true).open(seg_path).unwrap(); + if sim.rng.chance(50) { + // Zero from p to the end of the file's logical region (lost pages). + let zeros = vec![0u8; logical_end - p as usize]; + f.seek(SeekFrom::Start(p)).unwrap(); + f.write_all(&zeros).unwrap(); + sim.note(format!( + "FAULT wal-zero shard {i}: [{p}, {logical_end}) durable_lsn {d}" + )); + } else { + let glen = 1 + sim.rng.below(64) as usize; + let garbage: Vec = (0..glen).map(|_| sim.rng.next_u64() as u8 | 1).collect(); + f.seek(SeekFrom::Start(p)).unwrap(); + f.write_all(&garbage).unwrap(); + sim.note(format!( + "FAULT wal-scribble shard {i}: [{p}, {}) durable_lsn {d}", + p + glen as u64 + )); + } + f.sync_all().unwrap(); + } +} + +// --------------------------------------------------------------------------- +// Workload +// --------------------------------------------------------------------------- + +async fn ack_append(sim: &mut Sim, store: &Arc, mi: usize) { + let (payload, wire) = sim.make_record(mi); + let m = &sim.models[mi]; + let ct = if m.json { JSON_CT } else { OCTET }; + let name = m.name.clone(); + let resp = handlers::handle(Arc::clone(store), post_req(&name, ct, &payload, &[])).await; + if !(200..300).contains(&resp.status) { + sim.fail(format!( + "append to open stream {name} rejected with {} (payload {} bytes)", + resp.status, + payload.len() + )); + } + sim.note(format!("append {name} seq={} {}B acked", sim.models[mi].next_seq - 1, wire.len())); + sim.push_rec(mi, wire, RecStatus::Acked); +} + +async fn cancelled_append(sim: &mut Sim, store: &Arc, mi: usize) { + let (payload, wire) = sim.make_record(mi); + let m = &sim.models[mi]; + let ct = if m.json { JSON_CT } else { OCTET }; + let name = m.name.clone(); + let req = post_req(&name, ct, &payload, &[]); + let jh = tokio::spawn(handlers::handle(Arc::clone(store), req)); + for _ in 0..sim.rng.below(4) { + tokio::task::yield_now().await; + } + jh.abort(); + match jh.await { + Ok(resp) if (200..300).contains(&resp.status) => { + sim.note(format!("append {name} cancelled-but-acked ({}B)", wire.len())); + sim.push_rec(mi, wire, RecStatus::Acked); + } + Ok(resp) => { + sim.note(format!("append {name} cancelled, status {}", resp.status)); + sim.push_rec(mi, wire, RecStatus::Maybe); + } + Err(_) => { + sim.note(format!("append {name} cancelled mid-flight ({}B) -> maybe", wire.len())); + sim.push_rec(mi, wire, RecStatus::Maybe); + } + } +} + +async fn run_generation(sim: &mut Sim, store: &Arc, walset: &Arc, steps: u64) { + for _ in 0..steps { + let op = sim.rng.below(100); + match op { + // ---- create ---- + _ if op < 8 && sim.models.len() < 20 => { + let json = sim.rng.chance(40); + let name = format!("sim-{}", sim.name_ctr); + sim.name_ctr += 1; + let ct = if json { JSON_CT } else { OCTET }; + let resp = handlers::handle(Arc::clone(store), put_req(&name, ct, &[])).await; + if !(200..300).contains(&resp.status) { + sim.fail(format!("create {name} rejected: {}", resp.status)); + } + let st = store.get(&name).unwrap_or_else(|| { + sim.fail(format!("create {name} acked but store.get is None")) + }); + sim.note(format!("create {name} json={json}")); + sim.models.push(SModel { + name, + json, + file_base: 0, + file_path: st.file_path.clone(), + id: st.id, + recs: Vec::new(), + next_seq: 0, + closed: false, + deleted: false, + has_children: false, + unsettled: false, + floor: 0, + }); + } + // ---- cancelled append (network drop) ---- + _ if op < 14 => { + if let Some(mi) = sim.pick(|m| !m.deleted && !m.closed) { + cancelled_append(sim, store, mi).await; + } + } + // ---- close ---- + _ if op < 18 => { + if let Some(mi) = sim.pick(|m| !m.deleted && !m.closed) { + let name = sim.models[mi].name.clone(); + let ct = if sim.models[mi].json { JSON_CT } else { OCTET }; + let resp = handlers::handle( + Arc::clone(store), + post_req(&name, ct, b"", &[("stream-closed", "true")]), + ) + .await; + if !(200..300).contains(&resp.status) { + sim.fail(format!("close {name} rejected: {}", resp.status)); + } + sim.note(format!("close {name}")); + sim.models[mi].closed = true; + } + } + // ---- fork ---- + _ if op < 22 && sim.models.len() < 20 => { + if let Some(pi) = sim.pick(|m| !m.deleted && !m.unsettled) { + let bounds = sim.models[pi].boundaries(); + let at = bounds[sim.rng.below(bounds.len() as u64) as usize]; + let parent = sim.models[pi].name.clone(); + let json = sim.models[pi].json; + let ct = if json { JSON_CT } else { OCTET }; + let name = format!("sim-{}", sim.name_ctr); + sim.name_ctr += 1; + let resp = handlers::handle( + Arc::clone(store), + put_req( + &name, + ct, + &[ + ("stream-forked-from", parent.as_str()), + ("stream-fork-offset", &fork_offset(at)), + ], + ), + ) + .await; + if !(200..300).contains(&resp.status) { + sim.fail(format!( + "fork {name} from {parent}@{at} rejected: {}", + resp.status + )); + } + let st = store.get(&name).unwrap_or_else(|| { + sim.fail(format!("fork {name} acked but store.get is None")) + }); + let fb = st.shared.read().unwrap().file_base; + if fb != at { + sim.fail(format!("fork {name}: file_base {fb} != fork offset {at}")); + } + sim.note(format!("fork {name} from {parent}@{at}")); + sim.models[pi].has_children = true; + sim.models.push(SModel { + name, + json, + file_base: at, + file_path: st.file_path.clone(), + id: st.id, + recs: Vec::new(), + next_seq: 0, + closed: false, + deleted: false, + has_children: false, + unsettled: false, + floor: at, + }); + } + } + // ---- delete ---- + _ if op < 24 => { + if let Some(mi) = sim.pick(|m| !m.deleted && !m.has_children) { + let name = sim.models[mi].name.clone(); + let resp = handlers::handle(Arc::clone(store), delete_req(&name)).await; + if !(200..300).contains(&resp.status) { + sim.fail(format!("delete {name} rejected: {}", resp.status)); + } + sim.note(format!("delete {name}")); + sim.models[mi].deleted = true; + } + } + // ---- checkpoint ---- + _ if op < 30 => { + let si = sim.rng.below(sim.shards_n as u64) as usize; + let shard = Arc::clone(&walset.shards()[si]); + shard.checkpoint().await.unwrap_or_else(|e| { + sim.fail(format!("checkpoint shard {si} failed: {e}")) + }); + let tails = shard.read_durable_tails(); + for m in &mut sim.models { + if let Some(&t) = tails.get(&m.id) { + if t > m.floor { + m.floor = t; + } + } + } + sim.note(format!("checkpoint shard {si} ({} tails)", tails.len())); + } + // ---- live tail sanity ---- + // The reader-visible tail (durable_tail) may lawfully LAG the file + // length: a cancelled append leaves bytes in the file whose publish + // never ran (healed monotonically by the next acked append, or + // exposed by crash recovery). It must never EXCEED the file though. + _ if op < 33 => { + if let Some(mi) = sim.pick(|m| !m.deleted) { + let name = sim.models[mi].name.clone(); + if let Some(st) = store.get(&name) { + let len = std::fs::metadata(&st.file_path).map(|m| m.len()).unwrap_or(0); + let tail = st.tail().bytes; + let fb = sim.models[mi].file_base; + if tail > fb + len { + sim.fail(format!( + "stream {name}: live tail {tail} > file_base {fb} + file_len {len} \ + (published tail covers bytes the file does not have)" + )); + } + } + } + } + // ---- plain acked append ---- + _ => { + if let Some(mi) = sim.pick(|m| !m.deleted && !m.closed) { + ack_append(sim, store, mi).await; + } else { + // Everything closed/deleted: force a create next loop. + } + } + } + } + + // In-flight-at-crash appends: spawned over the real path, never awaited — + // the runtime drop (the crash) aborts them at whatever point they reached. + let inflight = sim.rng.below(3); + for _ in 0..inflight { + if let Some(mi) = sim.pick(|m| !m.deleted && !m.closed) { + let (payload, wire) = sim.make_record(mi); + let ct = if sim.models[mi].json { JSON_CT } else { OCTET }; + let name = sim.models[mi].name.clone(); + sim.note(format!("in-flight append {name} ({}B) at crash", wire.len())); + sim.push_rec(mi, wire, RecStatus::Maybe); + let req = post_req(&name, ct, &payload, &[]); + let store2 = Arc::clone(store); + tokio::spawn(async move { + let _ = handlers::handle(store2, req).await; + }); + } + } + if inflight > 0 { + // Let the spawned appends progress a random amount before the crash. + for _ in 0..sim.rng.below(6) { + tokio::task::yield_now().await; + } + } +} + +/// After the crash (runtime dropped, committers stopped): stage 0..=2 records +/// that reached the data file's page cache + the WAL staging buffer but whose +/// WAL bytes were never fdatasync'd — the classic torn-append shape. These are +/// exactly the bytes `inject_wal_faults` is then allowed to tear. +fn stage_torn_appends(sim: &mut Sim, store: &Arc, walset: &Arc) { + let n = sim.rng.below(3); + for _ in 0..n { + let Some(mi) = sim.pick(|m| !m.deleted && !m.closed) else { return }; + let (_, wire) = sim.make_record(mi); + let name = sim.models[mi].name.clone(); + let Some(st) = store.get(&name) else { continue }; + let cur_len = std::fs::metadata(&st.file_path).map(|m| m.len()).unwrap_or(0); + let offset = sim.models[mi].file_base + cur_len; + { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&st.file_path) + .unwrap(); + f.write_all(&wire).unwrap(); + } + let shard = walset.shard_for(st.id); + shard + .reserve_and_stage(RecordKind::Append, st.id, offset, &wire) + .unwrap(); + sim.note(format!( + "staged-at-crash append {name} @{offset} ({}B, never fdatasync'd)", + wire.len() + )); + sim.push_rec(mi, wire, RecStatus::Maybe); + } +} + +// --------------------------------------------------------------------------- +// Seed runner + test entry +// --------------------------------------------------------------------------- + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +fn run_one_seed(seed: u64, gens: u64, steps: u64) { + use std::time::{SystemTime, UNIX_EPOCH}; + let dir = std::env::temp_dir().join(format!( + "ds-wal-sim-{seed}-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() + )); + let _ = std::fs::remove_dir_all(&dir); + + let mut rng = Rng::new(seed); + let shards_n = 1 + (rng.below(3) as usize); // 1..=3 shards + let mut sim = Sim { + seed, + rng, + dir: dir.clone(), + shards_n, + models: Vec::new(), + name_ctr: 0, + gen: 0, + log: Vec::new(), + }; + + for g in 0..=gens { + sim.gen = g; + sim.note(format!("--- generation {g} boot ---")); + // Forensics: snapshot the pre-boot (post-crash, post-fault) disk state + // so a violation can be inspected before recovery/reset mutate it. + if std::env::var("DS_SIM_SNAPSHOT").is_ok() && g > 0 { + let snap = dir.with_file_name(format!( + "{}-preboot-gen{g}", + dir.file_name().unwrap().to_str().unwrap() + )); + let _ = std::fs::remove_dir_all(&snap); + let _ = std::process::Command::new("cp") + .arg("-R") + .arg(&dir) + .arg(&snap) + .status(); + eprintln!("[sim] snapshot: {}", snap.display()); + } + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let last = g == gens; + let (store, walset, committers) = rt.block_on(async { + let (store, walset, committers) = if g == 0 { + boot(&dir, Some(sim.shards_n), sim.shards_n) + } else { + boot(&dir, None, sim.shards_n) + }; + // Oracle over the recovered state (trivially empty on gen 0). + verify_recovery(&mut sim, &store); + if !last { + run_generation(&mut sim, &store, &walset, steps).await; + } + (store, walset, committers) + }); + if last { + for h in committers { + h.stop(); + } + drop(store); + drop(walset); + drop(rt); + break; + } + + // ---- CRASH ---- + // Dropping the runtime aborts every in-flight task (requests, meta + // flushes) at its current await point; blocking writes already + // submitted finish first (a legal pre-crash ordering). + drop(rt); + // The committer threads are the "disk": stop + join them (their final + // drain fsync is a no-op for anything the oracle relies on). + for h in committers { + h.stop(); + } + // Durable frontier per shard, BEFORE staging the torn tail below — + // everything at/below this is fdatasync'd and must survive faults. + let durable: Vec = walset.shards().iter().map(|s| s.durable_lsn_now()).collect(); + // Stage page-cache-only appends (data file + WAL staging, no fsync). + if sim.rng.chance(60) { + stage_torn_appends(&mut sim, &store, &walset); + } + drop(store); + drop(walset); + + // ---- POWER-LOSS FAULTS ---- + if sim.rng.chance(55) { + inject_data_file_faults(&mut sim); + } + if sim.rng.chance(55) { + inject_wal_faults(&mut sim, &durable); + } + } + + let _ = std::fs::remove_dir_all(&dir); +} + +/// Forensic helper: `DS_DUMP_DIR= cargo test wal_forensic_dump -- --ignored --nocapture` +/// decodes every WAL segment (lsn, kind, stream_id, stream_offset, len) and +/// lists per-stream file sizes, for inspecting a failing sim snapshot. +#[test] +#[ignore] +fn wal_forensic_dump() { + let dir = PathBuf::from(std::env::var("DS_DUMP_DIR").expect("set DS_DUMP_DIR")); + for entry in std::fs::read_dir(dir.join("streams")).unwrap().flatten() { + let p = entry.path(); + if p.extension().is_none() { + eprintln!("stream file {:?}: {} bytes", p.file_name().unwrap(), p.metadata().unwrap().len()); + } + } + let wal_root = dir.join("wal"); + for sh in std::fs::read_dir(&wal_root).unwrap().flatten() { + let sp = sh.path(); + if !sp.is_dir() { + continue; + } + let mut segs: Vec<(u64, PathBuf)> = std::fs::read_dir(&sp) + .unwrap() + .flatten() + .filter_map(|e| { + let p = e.path(); + let stem = p.file_stem()?.to_str()?.parse::().ok()?; + (p.extension()?.to_str()? == "wal").then_some((stem, p)) + }) + .collect(); + segs.sort(); + let ckpt = std::fs::read_to_string(sp.join("checkpoint")).unwrap_or_default(); + let tails = std::fs::read_to_string(sp.join("tails")).unwrap_or_default(); + eprintln!("== shard {:?} checkpoint={} tails={}", sp.file_name().unwrap(), ckpt.trim(), tails.trim()); + for (start, seg) in segs { + let bytes = std::fs::read(&seg).unwrap(); + eprintln!("-- segment {start}.wal ({} bytes)", bytes.len()); + let mut off = 0usize; + loop { + match decode_at(&bytes, off) { + Decoded::Record { lsn, kind, stream_id, stream_offset, len, total, .. } => { + eprintln!( + " off={off:<8} lsn={lsn:<6} kind={kind:?} stream={stream_id} s_off={stream_offset} len={len}" + ); + off += total; + } + Decoded::Incomplete => { + eprintln!(" off={off:<8} "); + break; + } + Decoded::Torn => { + eprintln!(" off={off:<8} "); + break; + } + } + } + } + } +} + +/// Forensic helper: run `replay_from_checkpoint(0)` over DS_DUMP_DIR's shards +/// via the real WalSet::open path and print every record the walk yields. +#[test] +#[ignore] +fn wal_forensic_replay() { + let dir = PathBuf::from(std::env::var("DS_DUMP_DIR").expect("set DS_DUMP_DIR")); + let walset = WalSet::open_with_segment_size(&dir, None, 1, SEG_BYTES).unwrap(); + for (i, shard) in walset.shards().iter().enumerate() { + let mut n = 0usize; + shard + .replay_from_checkpoint(0, |kind, sid, soff, payload| { + n += 1; + eprintln!("shard {i}: {kind:?} stream={sid} s_off={soff} len={}", payload.len()); + }) + .unwrap(); + eprintln!("shard {i}: {n} records replayed"); + } +} + +/// Seeded randomized crash/recovery simulation. Fast deterministic defaults for +/// CI; scale with DS_SIM_SEEDS / DS_SIM_SEED0 / DS_SIM_GENS / DS_SIM_STEPS for +/// a long local hunt. +#[test] +fn crash_recovery_randomized_simulation() { + let _guard = DurabilityGuard::wal(); + let seeds = env_u64("DS_SIM_SEEDS", 4); + let seed0 = env_u64("DS_SIM_SEED0", 0x0001_5EED); + let gens = env_u64("DS_SIM_GENS", 3); + let steps = env_u64("DS_SIM_STEPS", 100); + for k in 0..seeds { + let seed = seed0 + k; + eprintln!("[sim] seed {seed} ({}/{seeds})", k + 1); + run_one_seed(seed, gens, steps); + } +} diff --git a/packages/durable-streams-rust/src/wal/telemetry.rs b/packages/durable-streams-rust/src/wal/telemetry.rs index 145093dadc..ca5f7755b6 100644 --- a/packages/durable-streams-rust/src/wal/telemetry.rs +++ b/packages/durable-streams-rust/src/wal/telemetry.rs @@ -42,13 +42,62 @@ //! commit path (CQ-2). At N shards × 1 Hz this is N `read_dir`s/sec — acceptable //! and off the append/commit path. -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; /// Upper-inclusive batch-size bucket boundaries (records-per-commit). The last /// (overflow) bucket catches everything `> 128`. Mirrors the OTel /// `BATCH_BUCKETS` view so the printed and exported distributions agree. pub const BATCH_BUCKETS: &[u64] = &[1, 2, 4, 8, 16, 32, 64, 128]; +// =========================================================================== +// Runtime contention-stats gate (`--wal-stats`, OFF by default). +// =========================================================================== +// +// The contention counters below (lock-wait nanos, wakeup fan-out) need a clock +// read around each lock acquisition. Reading `Instant::now()` on the per-append +// hot path is NOT free, so the timing is gated behind a runtime flag: when off, +// the hot path pays a single relaxed atomic load and skips the clock entirely; +// when on (`--wal-stats `), it times every contended acquisition and a +// plain stderr emitter prints per-shard contention lines every ``. +// +// This is deliberately INDEPENDENT of the heavy `telemetry` (OTLP) feature: the +// whole point is an always-available, zero-dependency way to *observe the lock +// contention* the saturation findings attribute the write ceiling to — no +// collector, no feature rebuild. + +/// Whether `--wal-stats` armed the contention timing + emitter. +static STATS_ENABLED: AtomicBool = AtomicBool::new(false); + +/// Arm (or disarm) the contention-stats timing on the hot path. Called once at +/// startup from `--wal-stats`; the emitter is spawned separately in `main`. +pub fn set_stats_enabled(on: bool) { + STATS_ENABLED.store(on, Ordering::Relaxed); +} + +/// True when `--wal-stats` armed the contention timing. One relaxed load; the +/// hot path reads this before any `Instant::now()` so a default run pays nothing +/// beyond the load. +#[inline] +pub fn stats_enabled() -> bool { + STATS_ENABLED.load(Ordering::Relaxed) +} + +/// Time `f` (a lock acquisition) and return its result, recording the wait via +/// `record` ONLY when stats are armed. When off this is just `f()` plus one +/// relaxed atomic load — no clock read. Keeps the timing boilerplate out of the +/// shard hot path while guaranteeing the clock is never touched in a default run. +#[inline] +pub fn timed_lock(record: impl FnOnce(u64), f: impl FnOnce() -> T) -> T { + if stats_enabled() { + let t0 = std::time::Instant::now(); + let out = f(); + record(t0.elapsed().as_nanos() as u64); + out + } else { + f() + } +} + /// Per-shard durability + batch-size counters. Cheap to update on the commit /// path (relaxed atomics only); read off-path by the 1 Hz emitter. #[derive(Debug)] @@ -67,6 +116,32 @@ pub struct ShardStats { /// size is `<= BATCH_BUCKETS[i]` and `> BATCH_BUCKETS[i-1]`; the final extra /// slot counts batches `> 128`. buckets: [AtomicU64; BATCH_BUCKETS.len() + 1], + + // ---- contention counters (`--wal-stats`; see the gate above) ---- + /// Σ nanoseconds spent acquiring the shard `inner` Mutex on the APPEND path + /// (phase-1 reserve + phase-2 mark_written), and the number of such + /// acquisitions. `avg_inner_wait = nanos / acquires`. This is the headline + /// contention signal: all of a shard's streams funnel through `inner`, so a + /// rising average here is the per-shard write-serialization the saturation + /// findings attribute the ceiling to. + inner_lock_wait_nanos: AtomicU64, + inner_lock_acquires: AtomicU64, + /// Σ nanoseconds spent acquiring the shard `dirty` Mutex (one acquisition + + /// HashMap insert PER append), and the count. Isolates the per-append global + /// dirty-set lock (Tier-1 target: make it lock-free) from `inner`. + dirty_lock_wait_nanos: AtomicU64, + dirty_lock_acquires: AtomicU64, + /// Number of records staged into this shard (one per `reserve_and_stage`). + /// The append throughput per shard; pairs with `fsync_count` to show the + /// real coalescing ratio (`staged / fsync_count`). + staged: AtomicU64, + /// Σ durability waiters woken across all commits. After the Tier-1c coalesced + /// wakeup this is the count of waiters each `publish_durable` ACTUALLY fires — + /// only those whose lsn the new watermark satisfies, not every parked + /// subscriber. `avg_wakeups = woken / fsync_count` should sit near ~1 (versus + /// the old `watch`-broadcast thundering herd, where it tracked the parked + /// subscriber count). + waiters_woken: AtomicU64, } impl Default for ShardStats { @@ -77,6 +152,12 @@ impl Default for ShardStats { last_batch: AtomicU64::new(0), max_batch: AtomicU64::new(0), buckets: Default::default(), + inner_lock_wait_nanos: AtomicU64::new(0), + inner_lock_acquires: AtomicU64::new(0), + dirty_lock_wait_nanos: AtomicU64::new(0), + dirty_lock_acquires: AtomicU64::new(0), + staged: AtomicU64::new(0), + waiters_woken: AtomicU64::new(0), } } } @@ -100,6 +181,30 @@ impl ShardStats { self.buckets[idx].fetch_add(1, Ordering::Relaxed); } + /// Record one append-path `inner` Mutex acquisition that waited `nanos`. + /// Called twice per append (reserve + mark_written) only when stats armed. + pub fn record_inner_lock_wait(&self, nanos: u64) { + self.inner_lock_wait_nanos.fetch_add(nanos, Ordering::Relaxed); + self.inner_lock_acquires.fetch_add(1, Ordering::Relaxed); + } + + /// Record one `dirty` Mutex acquisition that waited `nanos` (once per append). + pub fn record_dirty_lock_wait(&self, nanos: u64) { + self.dirty_lock_wait_nanos.fetch_add(nanos, Ordering::Relaxed); + self.dirty_lock_acquires.fetch_add(1, Ordering::Relaxed); + } + + /// Record one record staged into this shard (one per `reserve_and_stage`). + pub fn record_staged(&self) { + self.staged.fetch_add(1, Ordering::Relaxed); + } + + /// Record the durability waiters woken by one commit (the coalesced count of + /// oneshots `publish_durable` fired). Called once per successful commit. + pub fn record_waiters_woken(&self, n: u64) { + self.waiters_woken.fetch_add(n, Ordering::Relaxed); + } + /// The bucket index for a batch size (the first boundary it does not exceed, /// else the overflow slot). fn bucket_index(batch: u64) -> usize { @@ -126,6 +231,12 @@ impl ShardStats { last_batch: self.last_batch.load(Ordering::Relaxed), max_batch: self.max_batch.load(Ordering::Relaxed), buckets, + inner_lock_wait_nanos: self.inner_lock_wait_nanos.load(Ordering::Relaxed), + inner_lock_acquires: self.inner_lock_acquires.load(Ordering::Relaxed), + dirty_lock_wait_nanos: self.dirty_lock_wait_nanos.load(Ordering::Relaxed), + dirty_lock_acquires: self.dirty_lock_acquires.load(Ordering::Relaxed), + staged: self.staged.load(Ordering::Relaxed), + waiters_woken: self.waiters_woken.load(Ordering::Relaxed), } } } @@ -145,6 +256,12 @@ pub struct StatsSnapshot { pub last_batch: u64, pub max_batch: u64, pub buckets: [u64; BATCH_BUCKETS.len() + 1], + pub inner_lock_wait_nanos: u64, + pub inner_lock_acquires: u64, + pub dirty_lock_wait_nanos: u64, + pub dirty_lock_acquires: u64, + pub staged: u64, + pub waiters_woken: u64, } /// All derivations are telemetry/test-only (emitter + stats tests); targeted @@ -167,6 +284,36 @@ impl StatsSnapshot { self.max_batch } + /// Mean nanoseconds an append waited to acquire the shard `inner` Mutex. + /// `0` when no acquisition was timed. The headline per-shard write- + /// serialization signal — near-zero when uncontended, climbs under load. + pub fn avg_inner_wait_ns(&self) -> f64 { + if self.inner_lock_acquires == 0 { + 0.0 + } else { + self.inner_lock_wait_nanos as f64 / self.inner_lock_acquires as f64 + } + } + + /// Mean nanoseconds an append waited to acquire the per-append `dirty` Mutex. + pub fn avg_dirty_wait_ns(&self) -> f64 { + if self.dirty_lock_acquires == 0 { + 0.0 + } else { + self.dirty_lock_wait_nanos as f64 / self.dirty_lock_acquires as f64 + } + } + + /// Mean durability waiters woken per commit (`waiters_woken / fsync_count`). + /// The thundering-herd fan-out: how many parked appenders each commit wakes. + pub fn avg_waiters_woken(&self) -> f64 { + if self.fsync_count == 0 { + 0.0 + } else { + self.waiters_woken as f64 / self.fsync_count as f64 + } + } + /// The `q`-quantile (0.0..=1.0) batch size, read off the cumulative buckets. /// Returns the **upper boundary** of the bucket the quantile falls in (the /// conservative, OTel-explicit-bucket convention); the overflow bucket @@ -212,6 +359,12 @@ impl StatsSnapshot { for (a, b) in self.buckets.iter_mut().zip(other.buckets.iter()) { *a += *b; } + self.inner_lock_wait_nanos += other.inner_lock_wait_nanos; + self.inner_lock_acquires += other.inner_lock_acquires; + self.dirty_lock_wait_nanos += other.dirty_lock_wait_nanos; + self.dirty_lock_acquires += other.dirty_lock_acquires; + self.staged += other.staged; + self.waiters_woken += other.waiters_woken; } } @@ -314,6 +467,98 @@ mod emitter { #[cfg(feature = "telemetry")] pub use emitter::spawn_emitter; +// =========================================================================== +// Always-on contention emitter (`--wal-stats `, dependency-free). +// =========================================================================== +// +// Unlike the `telemetry`-feature `WAL_STATS` emitter above (which needs the OTLP +// dep tree only to be COMPILED, and prints batch gauges to stdout), this one is +// ALWAYS compiled and prints DELTA-based contention rates to stderr each tick. +// It is the measurement vehicle for the lock-contention investigation: no +// collector, no feature flag — just `--wal-stats 1` and read the `WAL_CONT` +// lines. Spawned from `main` only when `--wal-stats` is passed (which also arms +// `set_stats_enabled(true)` so the hot path actually times the acquisitions). + +mod stats_emitter { + use std::sync::Arc; + use std::time::Duration; + + use super::StatsSnapshot; + use crate::wal::walset::WalSet; + + /// Spawn the contention emitter, printing aggregate `WAL_CONT` lines every + /// `interval`. Deltas are taken against the previous tick so the numbers + /// reflect the live window, not a warmup-diluted cumulative average. + pub fn spawn_stats_emitter(walset: Arc, interval: Duration) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + ticker.tick().await; // skip the immediate boot tick + + // Per-shard previous snapshots for delta computation. + let n = walset.shards().len(); + let mut prev: Vec = vec![StatsSnapshot::default(); n]; + + loop { + ticker.tick().await; + let dt = interval.as_secs_f64(); + let mut cur_agg = StatsSnapshot::default(); + let mut prev_agg = StatsSnapshot::default(); + for (i, shard) in walset.shards().iter().enumerate() { + let cur = shard.stats_snapshot(); + cur_agg.merge(&cur); + prev_agg.merge(&prev[i]); + prev[i] = cur; + } + print_delta(&cur_agg, &prev_agg, dt); + } + }); + } + + /// Print one aggregate `WAL_CONT` line from the (cur − prev) delta over `dt` + /// seconds. All `avg_*` fields are computed on the DELTA counts so they show + /// the contention *in this window*. + fn print_delta(cur: &StatsSnapshot, prev: &StatsSnapshot, dt: f64) { + let d_staged = cur.staged.saturating_sub(prev.staged); + let d_fsync = cur.fsync_count.saturating_sub(prev.fsync_count); + let d_records = cur.records_committed.saturating_sub(prev.records_committed); + let d_inner_ns = cur + .inner_lock_wait_nanos + .saturating_sub(prev.inner_lock_wait_nanos); + let d_inner_acq = cur + .inner_lock_acquires + .saturating_sub(prev.inner_lock_acquires); + let d_dirty_ns = cur + .dirty_lock_wait_nanos + .saturating_sub(prev.dirty_lock_wait_nanos); + let d_dirty_acq = cur + .dirty_lock_acquires + .saturating_sub(prev.dirty_lock_acquires); + let d_woken = cur.waiters_woken.saturating_sub(prev.waiters_woken); + + let staged_per_s = d_staged as f64 / dt; + let fsync_per_s = d_fsync as f64 / dt; + let batch_avg = if d_fsync == 0 { 0.0 } else { d_records as f64 / d_fsync as f64 }; + let inner_wait_us = if d_inner_acq == 0 { 0.0 } else { d_inner_ns as f64 / d_inner_acq as f64 / 1000.0 }; + let dirty_wait_us = if d_dirty_acq == 0 { 0.0 } else { d_dirty_ns as f64 / d_dirty_acq as f64 / 1000.0 }; + let woken_avg = if d_fsync == 0 { 0.0 } else { d_woken as f64 / d_fsync as f64 }; + // Lock-wait *load*: fraction of one core-second spent purely WAITING on + // each lock in this window (Σ wait nanos / window nanos). >1.0 means more + // than a whole core's worth of time was lost parking on that lock. + let inner_wait_load = d_inner_ns as f64 / (dt * 1e9); + let dirty_wait_load = d_dirty_ns as f64 / (dt * 1e9); + + eprintln!( + "WAL_CONT staged/s={staged_per_s:.0} fsync/s={fsync_per_s:.0} batch_avg={batch_avg:.1} \ + inner_wait_us={inner_wait_us:.2} inner_wait_load={inner_wait_load:.2} \ + dirty_wait_us={dirty_wait_us:.2} dirty_wait_load={dirty_wait_load:.2} \ + waiters_woken_avg={woken_avg:.1}" + ); + } +} + +pub use stats_emitter::spawn_stats_emitter; + /// No-op `spawn_emitter` (feature off): spawns nothing, prints nothing. A default /// build pays only the per-commit atomics in `record_batch`. #[cfg(not(feature = "telemetry"))] @@ -354,6 +599,54 @@ mod tests { assert_eq!(snap.quantile(1.0), 200, "top quantile reaches the overflow max"); } + #[test] + fn contention_counters_average_and_merge() { + let a = ShardStats::default(); + // Two inner acquisitions waiting 100ns and 300ns → avg 200ns. + a.record_inner_lock_wait(100); + a.record_inner_lock_wait(300); + // One dirty acquisition waiting 50ns. + a.record_dirty_lock_wait(50); + a.record_staged(); + a.record_staged(); + // One commit woke 8 waiters. + a.record_batch(2); + a.record_waiters_woken(8); + let sa = a.snapshot(); + assert_eq!(sa.inner_lock_acquires, 2); + assert_eq!(sa.avg_inner_wait_ns(), 200.0); + assert_eq!(sa.avg_dirty_wait_ns(), 50.0); + assert_eq!(sa.staged, 2); + assert_eq!(sa.avg_waiters_woken(), 8.0, "8 woken / 1 commit"); + + // Merge a second shard and confirm the contention fields fold in. + let b = ShardStats::default(); + b.record_inner_lock_wait(200); + b.record_dirty_lock_wait(150); + b.record_staged(); + b.record_batch(4); + b.record_waiters_woken(4); + let mut agg = sa.clone(); + agg.merge(&b.snapshot()); + assert_eq!(agg.inner_lock_acquires, 3); + assert_eq!(agg.inner_lock_wait_nanos, 600); + assert_eq!(agg.avg_inner_wait_ns(), 200.0, "600ns / 3 acquires"); + assert_eq!(agg.dirty_lock_wait_nanos, 200); + assert_eq!(agg.staged, 3); + assert_eq!(agg.waiters_woken, 12); + assert_eq!(agg.avg_waiters_woken(), 6.0, "12 woken / 2 commits"); + } + + #[test] + fn stats_gate_defaults_off_and_toggles() { + // The gate defaults off so default runs read no clock; toggling it is what + // `--wal-stats` does. (Process-global, so just prove the toggle works.) + super::set_stats_enabled(true); + assert!(super::stats_enabled()); + super::set_stats_enabled(false); + assert!(!super::stats_enabled()); + } + #[test] fn merge_aggregates() { let a = ShardStats::default(); diff --git a/packages/durable-streams-rust/src/wal/walset.rs b/packages/durable-streams-rust/src/wal/walset.rs index 669dade2f8..b8fe5609ea 100644 --- a/packages/durable-streams-rust/src/wal/walset.rs +++ b/packages/durable-streams-rust/src/wal/walset.rs @@ -21,9 +21,9 @@ use std::io; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use super::shard::Shard; +use super::shard::{CommitterHandle, Shard}; /// The persisted-`N` file under the data dir: `/wal/shards`. const SHARDS_FILE: &str = "shards"; @@ -46,6 +46,12 @@ fn fnv1a(x: u64) -> u64 { pub struct WalSet { shards: Vec>, n: usize, + /// Handles to the per-shard dedicated committer OS threads (Tier-2a). Spawned + /// by [`WalSet::spawn_committers`], joined by [`WalSet::stop_committers`] on + /// the process shutdown path so in-flight commits complete and the threads are + /// reaped rather than detached. Behind a `Mutex` for interior mutability (the + /// `WalSet` is shared behind an `Arc`). + committers: Mutex>, } impl WalSet { @@ -113,7 +119,11 @@ impl WalSet { segment_size, )?); } - Ok(Arc::new(WalSet { shards, n })) + Ok(Arc::new(WalSet { + shards, + n, + committers: Mutex::new(Vec::new()), + })) } /// The shard a `stream_id` routes to — computed **only** from the persisted @@ -143,11 +153,32 @@ impl WalSet { Ok(()) } - /// Spawn each shard's committer (the tokio-task `run_committer`). + /// Spawn each shard's committer on its own **dedicated OS thread** (Tier-2a), + /// off the shared async runtime. The [`CommitterHandle`]s are retained so + /// [`WalSet::stop_committers`] can drain + join them on shutdown. pub fn spawn_committers(self: &Arc) { + let mut handles = self.committers.lock().unwrap(); for shard in &self.shards { - let shard = Arc::clone(shard); - tokio::spawn(shard.run_committer()); + handles.push(shard.spawn_committer()); + } + } + + /// Stop every committer thread: signal all of them first (so they shut down + /// concurrently), then join all. Each thread performs a **final drain** so any + /// already-contiguous-written records still become durable before it exits — + /// keeping the no-loss invariant across a graceful shutdown. Idempotent (a + /// second call finds the handle list empty). Called from the process shutdown + /// path (`main.rs`, after `engine_raw::drain`). + pub fn stop_committers(&self) { + let handles: Vec = + std::mem::take(&mut *self.committers.lock().unwrap()); + // Signal all, then join all, so a slow shard's final drain overlaps the + // others rather than serializing shutdown. + for h in &handles { + h.signal_stop(); + } + for h in handles { + h.join(); } } }