From 935f7f250de8f2d048c459cc6df2e2cec92062a6 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:11:24 -0700 Subject: [PATCH 01/12] checkpoint: pre-existing uncommitted 0.3.3 work (rocksdb support files, benches, docs) Found uncommitted in the working tree before the export/import feature work began: the snapshot_record codec split, backend benches, nats.rs changes, and doc updates from the unreleased 0.3.3 rocksdb-backend work. Checkpointed verbatim so feature commits stay separable. The rocksdb backend module itself lands with the export/import commit that extends it (the file mixes both). Co-Authored-By: Claude Fable 5 --- ARCHITECTURE.md | 9 +- README.md | 12 +- benches/snapshot_backends.rs | 416 +++++++++++++++++++++++++++++++++++ src/nats.rs | 6 +- src/snapshot_record.rs | 145 ++++++++++++ tests/integration.rs | 4 +- 6 files changed, 581 insertions(+), 11 deletions(-) create mode 100644 benches/snapshot_backends.rs create mode 100644 src/snapshot_record.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3dddaea..57efa51 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -72,6 +72,7 @@ watch_all_from(cursor, tx) | `SnapshotStore` | Trait: the durable-fold contract — atomic `apply(batch, cursor)`, `load`, `get`, `range` | Not a serving index; stops at fold + cursor + query | | `AppendLogSnapshot` | Default `SnapshotStore`: append-only log + in-RAM fold (pure-Rust) | Not for folds larger than RAM | | `FjallSnapshot` | On-disk `SnapshotStore` (fjall LSM, `feature = "fjall"`) for large folds | Not in the pure-Rust core; opt-in feature | +| `RocksDbSnapshot` | On-disk `SnapshotStore` (RocksDB, `feature = "rocksdb"`) for large folds | Not pure-Rust; opt-in feature with a C++ build dep | | `watch_applied` | Combinator: batch → apply → *then* advance cursor / fold into `SnapshotStore` | Not a raw watch; the cursor follows `apply`, not receipt | | `ConnectionCapabilities` | Feature flags for runtime branching (CAS, streaming watch, …) | Not enforced; purely advisory | @@ -95,7 +96,8 @@ watch_all_from(cursor, tx) snapshot.rs (orthogonal, optional) ┌─────────────────────────────────────────────────────────────┐ │ SnapshotStore trait: apply(batch, cursor) │ load │ get │ range -│ AppendLogSnapshot (default, in-RAM) FjallSnapshot (feat) │ +│ AppendLogSnapshot (default, in-RAM) │ +│ FjallSnapshot │ RocksDbSnapshot (feature-gated, on-disk) │ │ (append-only CRC log, tempfile+rename compact) │ └─────────────────────────────────────────────────────────────┘ applied.rs (combinator over KvWatcher + snapshot) @@ -209,8 +211,9 @@ Three invariants bind every implementation: | ------- | ------ | ----- | ---------- | | `AppendLogSnapshot` (default) | `snapshot.rs` | Append-only CRC log + in-RAM `HashMap` fold | `checkpoint` flush (page cache); `fsync` only at `compact` | | `FjallSnapshot` (`feature = "fjall"`) | `snapshot_fjall.rs` | On-disk fjall LSM (`data` + `meta` partitions) | One atomic batch per `apply` (data + cursor); per-commit `fsync` configurable (NO_SYNC default) | +| `RocksDbSnapshot` (`feature = "rocksdb"`) | `snapshot_rocksdb.rs` | On-disk RocksDB (`data` + `meta` column families), tuned for billion-key folds (hit-optimized ribbon filters, partitioned index, zstd bottommost — see the module's Tuning docs) | One atomic `WriteBatch` per `apply` (data + cursor); WAL always on, per-commit `fsync` configurable (NO_SYNC default) | -`FjallSnapshot` keeps the cursor in the same fjall `Batch` as the data it names, so under NO_SYNC a crash can lose the un-synced tail but never desynchronize cursor from data — on reopen the recovered cursor is consistent and the watch re-folds the tail. The rest of this section describes the **append-log backend** (the default), whose on-disk format is below. +The two LSM backends are interchangeable in contract and share the value-record codec (`snapshot_record.rs`); fjall keeps the crate pure-Rust, RocksDB trades a C++ build dependency for the battle-tested engine and its operational tooling (`ldb`, `sst_dump`). Both keep the cursor in the same atomic batch as the data it names, so under NO_SYNC a crash can lose the un-synced tail but never desynchronize cursor from data — on reopen the recovered cursor is consistent and the watch re-folds the tail. The rest of this section describes the **append-log backend** (the default), whose on-disk format is below. ### File Format @@ -371,6 +374,8 @@ Checkpoints are frequent (every N watch events). An fsync per checkpoint would a | `src/nats.rs` | NATS JetStream implementation; bucket creation, scan consumer lifecycle, timeout wrapping, Synadia Cloud workarounds | | `src/snapshot.rs` | `SnapshotStore` trait; append-only log + `AppendLogSnapshot` (default backend): `SnapshotWriter`, `load()`, `replay_log()`, `compact_to_file()` | | `src/snapshot_fjall.rs` | `FjallSnapshot`: on-disk `SnapshotStore` backed by fjall (`feature = "fjall"`) | +| `src/snapshot_rocksdb.rs` | `RocksDbSnapshot`: on-disk `SnapshotStore` backed by RocksDB (`feature = "rocksdb"`) | +| `src/snapshot_record.rs` | Shared `[ver_len][version][value]` value-record codec for the LSM backends | | `src/applied.rs` | `watch_applied` cursor-after-apply combinator, generic over `SnapshotStore`: `WatchScope`, `BatchConfig` | | `src/lib.rs` | Re-exports all public types; no logic | | `benches/` | Criterion benchmarks for snapshot write/checkpoint/load throughput and batch throughput | diff --git a/README.md b/README.md index b042437..667761c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ beyond-slipstream = "0.2" The crate core is pure-Rust. On-disk snapshot backends are opt-in cargo features (no C toolchain is pulled unless you enable one): ```toml -beyond-slipstream = { version = "0.2", features = ["fjall"] } # on-disk SnapshotStore for large folds +beyond-slipstream = { version = "0.2", features = ["fjall"] } # on-disk SnapshotStore, pure-Rust LSM +beyond-slipstream = { version = "0.2", features = ["rocksdb"] } # on-disk SnapshotStore, RocksDB (needs a C++ toolchain + libclang to build) ``` ## Concepts @@ -31,6 +32,7 @@ beyond-slipstream = { version = "0.2", features = ["fjall"] } # on-disk Snapshot | `SnapshotStore` | Trait: the durable-fold contract — `apply` (data + cursor, atomically), `load`, `get`, `range` | | `AppendLogSnapshot` | Default `SnapshotStore`: the append-only log + an in-RAM fold (pure-Rust, small state) | | `FjallSnapshot` | On-disk `SnapshotStore` for folds too large for RAM; queryable (`feature = "fjall"`) | +| `RocksDbSnapshot` | Same contract on RocksDB, for consumers who prefer the C++ LSM (`feature = "rocksdb"`) | | `watch_applied` | Watch loop that advances the cursor only after your `apply` returns, folding into any `SnapshotStore` | | `ConnectionCapabilities` | Feature flags for runtime branching (CAS, streaming watch, global ordering) | @@ -234,7 +236,8 @@ Every backend keeps the same invariants: the fold is a pure function of the log | Backend | When | Notes | | ------- | ---- | ----- | | `AppendLogSnapshot` | **Default.** Fold fits in RAM (edge/tunnel-style services) | Pure-Rust, the append-only log above plus an in-RAM map serving `get`/`range`. No extra dependencies. | -| `FjallSnapshot` | Fold too large for RAM (e.g. routing at ~1B keys) | On-disk [fjall](https://docs.rs/fjall) LSM, `feature = "fjall"`. Each `apply` is one atomic batch (data **and** cursor); durability (NO_SYNC vs fsync) is configurable. | +| `FjallSnapshot` | Fold too large for RAM (e.g. routing at ~1B keys) | On-disk [fjall](https://docs.rs/fjall) LSM, `feature = "fjall"`. Pure-Rust. Each `apply` is one atomic batch (data **and** cursor); durability (NO_SYNC vs fsync) is configurable. | +| `RocksDbSnapshot` | Same as `FjallSnapshot`, preferring the battle-tested C++ LSM and its tooling (`ldb`, `sst_dump`) | On-disk [RocksDB](https://docs.rs/rust-rocksdb), `feature = "rocksdb"`. Each `apply` is one atomic `WriteBatch` (data **and** cursor); WAL always on, per-commit fsync configurable. Tuned for billion-key route folds (hit-optimized ribbon filters, partitioned index, zstd bottommost, batched `multi_get`). Builds C++ (needs a toolchain + libclang). | Pick a backend, then hand it to [`watch_applied`](#applied-watch) — `load` returns the resume cursor alongside the store: @@ -245,7 +248,10 @@ use slipstream::{AppendLogSnapshot, SnapshotStore}; let (resume, store) = AppendLogSnapshot::load(Path::new("/var/lib/svc/state.snap"))?; // Or, behind `feature = "fjall"`, an on-disk fold for a large consumer: -// let (resume, store) = FjallSnapshot::open(dir, FjallConfig { sync: false })?; +// let (resume, store) = FjallSnapshot::open(dir, FjallConfig { sync: false, ..Default::default() })?; + +// Or the same on RocksDB, behind `feature = "rocksdb"`: +// let (resume, store) = RocksDbSnapshot::open(dir, RocksDbConfig { sync: false, ..Default::default() })?; let final_cursor = watch_applied( watcher, WatchScope::All, Some(resume), Some(store), BatchConfig::default(), diff --git a/benches/snapshot_backends.rs b/benches/snapshot_backends.rs new file mode 100644 index 0000000..3e09556 --- /dev/null +++ b/benches/snapshot_backends.rs @@ -0,0 +1,416 @@ +//! Comparative benchmarks for the on-disk [`SnapshotStore`] backends: +//! `FjallSnapshot` vs `RocksDbSnapshot`, on the route-fold workload shape the +//! backends are tuned for (clustered `route.svc-NNNNNN.NNNNNNNN` keys, ~200 B +//! values, batched applies, point gets for existing keys, per-service prefix +//! scans). +//! +//! Run with both backends enabled: +//! +//! ```text +//! cargo bench --bench snapshot_backends --features fjall,rocksdb +//! SLIPSTREAM_BENCH_ENTRIES=4000000 cargo bench --bench snapshot_backends --features fjall,rocksdb +//! ``` +//! +//! Env knobs: `SLIPSTREAM_BENCH_ENTRIES` (default 1_000_000), +//! `SLIPSTREAM_BENCH_VALUE_BYTES` (default 200), and +//! `SLIPSTREAM_BENCH_CACHE_BYTES` (default: each backend's default cache) for +//! measuring cache-size sensitivity, e.g. 32 MiB vs 2 GiB. +//! +//! Caveats for honest numbers: `TempDir` honors `TMPDIR` — point it at real +//! NVMe, not tmpfs, when disk behavior matters. Criterion's repeated iterations +//! measure *warm-cache* reads, which is fair for cross-backend comparison; +//! cold-cache latency needs a manual run against a freshly opened store. + +use std::hint::black_box; +use std::path::Path; + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use slipstream::snapshot::SnapshotStore; +use slipstream::{ + FjallConfig, FjallSnapshot, KvEntry, KvUpdate, RocksDbConfig, RocksDbSnapshot, VersionToken, + WatchCursor, +}; +use tempfile::TempDir; + +/// Routes per service: keys cluster as `route.svc-{service:06}.{route:08}`. +const ROUTES_PER_SERVICE: usize = 1000; +/// Updates per `apply` batch — the `watch_applied` flush-batch shape. +const APPLY_BATCH: usize = 1024; +/// Pseudo-random pool the entry values are sliced from (so compression sees +/// realistic entropy instead of a constant byte). +const VALUE_POOL_BYTES: usize = 1 << 20; + +fn entries() -> usize { + std::env::var("SLIPSTREAM_BENCH_ENTRIES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1_000_000) +} + +fn value_bytes() -> usize { + std::env::var("SLIPSTREAM_BENCH_VALUE_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(200) +} + +/// Deterministic xorshift64* step — repeatable key/value choice across runs. +fn next_rand(state: &mut u64) -> u64 { + let mut x = *state; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + *state = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) +} + +fn key(i: usize) -> String { + format!( + "route.svc-{:06}.{:08}", + i / ROUTES_PER_SERVICE, + i % ROUTES_PER_SERVICE + ) +} + +fn value_pool() -> Vec { + let mut pool = vec![0u8; VALUE_POOL_BYTES]; + let mut state = 0x5EED_5EED_5EED_5EEDu64; + for chunk in pool.chunks_mut(8) { + let bytes = next_rand(&mut state).to_le_bytes(); + chunk.copy_from_slice(&bytes[..chunk.len()]); + } + pool +} + +fn value_for(pool: &[u8], i: usize, len: usize) -> &[u8] { + let off = (i * 7919) % (pool.len() - len); + &pool[off..off + len] +} + +/// Fold `n` entries into `store` in `APPLY_BATCH`-sized applies, the way +/// `watch_applied` would during hydration. +fn hydrate(store: &mut S, n: usize, pool: &[u8], vlen: usize) { + let mut batch = Vec::with_capacity(APPLY_BATCH); + let mut i = 0usize; + while i < n { + batch.clear(); + let end = (i + APPLY_BATCH).min(n); + for j in i..end { + batch.push(KvUpdate::Put(KvEntry { + key: key(j), + value: value_for(pool, j, vlen).to_vec(), + version: VersionToken::from_u64(j as u64 + 1), + })); + } + store + .apply(&batch, &WatchCursor::from_u64(end as u64)) + .expect("apply"); + i = end; + } +} + +/// A key shaped like the fold's keys but in a service range that is never +/// hydrated — the absent-key (unknown service) lookup. +fn miss_key(i: usize) -> String { + format!("route.svc-9{:06}.{:08}", i % 999_999, i % 1000) +} + +/// 10k uniform random single-key probes, reported as a latency distribution. +/// Criterion's mean-of-batches hides exactly the tail this exists to surface +/// (a 200 µs p50 with a 100 ms p999 and "everything is 10 ms" have the same +/// mean and opposite fixes). +fn probe_percentiles(label: &str, mut op: impl FnMut(usize)) { + const PROBES: usize = 10_000; + let n = entries(); + let mut state = 0x0123_4567_89AB_CDEFu64; + let mut lat = Vec::with_capacity(PROBES); + for _ in 0..PROBES { + let i = (next_rand(&mut state) % n as u64) as usize; + let t = std::time::Instant::now(); + op(i); + lat.push(t.elapsed()); + } + lat.sort(); + let p = |q: f64| lat[(((lat.len() as f64) * q) as usize).min(lat.len() - 1)]; + eprintln!( + "{label}: p50 {:.1?} / p90 {:.1?} / p99 {:.1?} / p999 {:.1?} / max {:.1?}", + p(0.50), + p(0.90), + p(0.99), + p(0.999), + lat[lat.len() - 1], + ); +} + +fn cache_bytes() -> Option { + std::env::var("SLIPSTREAM_BENCH_CACHE_BYTES") + .ok() + .and_then(|v| v.parse().ok()) +} + +/// Recursive on-disk size of a store directory, for the post-hydration space +/// report (WAL/journal and not-yet-compacted overhead included — this is what +/// the disk actually holds after a hydration). +fn dir_size_bytes(path: &Path) -> u64 { + let mut total = 0u64; + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(meta) = entry.metadata() else { continue }; + if meta.is_dir() { + stack.push(entry.path()); + } else { + total += meta.len(); + } + } + } + total +} + +fn open_fjall(path: &Path) -> FjallSnapshot { + let mut config = FjallConfig::default(); + if let Some(bytes) = cache_bytes() { + config.cache_size_bytes = bytes; + } + FjallSnapshot::open(path, config).expect("open fjall").1 +} + +fn open_rocksdb(path: &Path) -> RocksDbSnapshot { + let mut config = RocksDbConfig::default(); + if let Some(bytes) = cache_bytes() { + config.cache_size_bytes = bytes; + } + RocksDbSnapshot::open(path, config).expect("open rocksdb").1 +} + +fn bench_apply_hydrate(c: &mut Criterion) { + let n = entries(); + let vlen = value_bytes(); + let pool = value_pool(); + + // Criterion runs at least 10 samples; 10 full hydrations beyond a few + // million entries is hours. At large scale, skip this group — the read + // benchmarks' setup (`bench_reads`) times its one-shot hydration of each + // backend and prints the throughput instead. + if n > 8_000_000 { + eprintln!( + "apply_hydrate: skipped at {n} entries (>8M); see the one-shot \ + hydration timings printed by the read benchmarks' setup" + ); + return; + } + + let mut g = c.benchmark_group("apply_hydrate"); + g.sample_size(10); + g.throughput(Throughput::Elements(n as u64)); + + g.bench_function(BenchmarkId::new("fjall", n), |b| { + b.iter_batched( + || TempDir::new().unwrap(), + |dir| { + let mut s = open_fjall(&dir.path().join("store")); + hydrate(&mut s, n, &pool, vlen); + dir + }, + BatchSize::PerIteration, + ); + }); + g.bench_function(BenchmarkId::new("rocksdb", n), |b| { + b.iter_batched( + || TempDir::new().unwrap(), + |dir| { + let mut s = open_rocksdb(&dir.path().join("store")); + hydrate(&mut s, n, &pool, vlen); + dir + }, + BatchSize::PerIteration, + ); + }); + g.finish(); +} + +fn bench_reads(c: &mut Criterion) { + let n = entries(); + let vlen = value_bytes(); + let pool = value_pool(); + + // One hydrated store per backend, built outside the timers and shared by + // every read benchmark below. The hydrations are timed and printed — at + // large scale this is the apply-throughput measurement (see + // `bench_apply_hydrate`). + let fjall_dir = TempDir::new().unwrap(); + let mut fjall = open_fjall(&fjall_dir.path().join("store")); + let started = std::time::Instant::now(); + hydrate(&mut fjall, n, &pool, vlen); + let secs = started.elapsed().as_secs_f64(); + let bytes = dir_size_bytes(fjall_dir.path()); + eprintln!( + "hydrate/fjall: {n} entries in {secs:.1}s ({:.2}M entries/s); on disk \ + {:.2} GiB ({:.0} B/entry)", + n as f64 / secs / 1e6, + bytes as f64 / (1u64 << 30) as f64, + bytes as f64 / n as f64, + ); + // Settle before reading: a fresh hydration leaves compaction debt that + // inflates cold reads (the unsettled state is reported separately by the + // settle duration itself). + let started = std::time::Instant::now(); + fjall.settle().expect("settle fjall"); + let settled_bytes = dir_size_bytes(fjall_dir.path()); + eprintln!( + "settle/fjall: {:.1}s; on disk after {:.2} GiB", + started.elapsed().as_secs_f64(), + settled_bytes as f64 / (1u64 << 30) as f64, + ); + + let rocks_dir = TempDir::new().unwrap(); + let mut rocks = open_rocksdb(&rocks_dir.path().join("store")); + let started = std::time::Instant::now(); + hydrate(&mut rocks, n, &pool, vlen); + let secs = started.elapsed().as_secs_f64(); + let bytes = dir_size_bytes(rocks_dir.path()); + eprintln!( + "hydrate/rocksdb: {n} entries in {secs:.1}s ({:.2}M entries/s); on disk \ + {:.2} GiB ({:.0} B/entry)", + n as f64 / secs / 1e6, + bytes as f64 / (1u64 << 30) as f64, + bytes as f64 / n as f64, + ); + let started = std::time::Instant::now(); + rocks.settle().expect("settle rocksdb"); + let settled_bytes = dir_size_bytes(rocks_dir.path()); + eprintln!( + "settle/rocksdb: {:.1}s; on disk after {:.2} GiB", + started.elapsed().as_secs_f64(), + settled_bytes as f64 / (1u64 << 30) as f64, + ); + let rocks_reader = rocks.reader(); + + // --- Cold-read latency distributions (percentiles, not criterion means). + // 10k uniform random probes each; "miss" keys share the key shape but name + // services that were never written. Run-order caveat: rocksdb hydrated + // last, so a slice of its store is page-cache-resident that fjall's isn't. + probe_percentiles("probe_hit/fjall", |i| { + let _ = black_box(fjall.get(&key(i)).expect("get")); + }); + probe_percentiles("probe_miss/fjall", |i| { + let _ = black_box(fjall.get(&miss_key(i)).expect("get")); + }); + probe_percentiles("probe_hit/rocksdb", |i| { + let _ = black_box(rocks.get(&key(i)).expect("get")); + }); + probe_percentiles("probe_miss/rocksdb", |i| { + let _ = black_box(rocks.get(&miss_key(i)).expect("get")); + }); + + // --- Point gets for existing keys, uniform over the whole fold. --- + // CRITICAL: the RNG state must live OUTSIDE the benchmark closure. + // Criterion re-invokes that closure once per sample (and per warmup pass); + // state declared inside the body resets to the seed every time, replaying + // the same key prefix — which the page cache then serves warm. At 250M + // that artifact reported 1.5 µs "gets" while criterion's own calibration + // (estimated time / iterations) showed ~860 µs during warmup. Hoisted + // state keeps every draw fresh across samples, so the measured miss rate + // is the true one for uniform access over a fold larger than RAM. + let mut g = c.benchmark_group("get_hit"); + g.throughput(Throughput::Elements(1)); + let mut fjall_state = 0xDEAD_BEEFu64; + g.bench_function("fjall", |b| { + b.iter(|| { + let i = (next_rand(&mut fjall_state) % n as u64) as usize; + black_box(fjall.get(&key(i)).expect("get")) + }); + }); + let mut rocks_state = 0xDEAD_BEEFu64; + g.bench_function("rocksdb", |b| { + b.iter(|| { + let i = (next_rand(&mut rocks_state) % n as u64) as usize; + black_box(rocks.get(&key(i)).expect("get")) + }); + }); + g.finish(); + + // --- One service's routes: the working-set hydration scan. --- + let mid_service = (n / ROUTES_PER_SERVICE) / 2; + let prefix = format!("route.svc-{mid_service:06}."); + let mut g = c.benchmark_group("prefix_scan"); + g.throughput(Throughput::Elements(ROUTES_PER_SERVICE as u64)); + g.bench_function("fjall", |b| { + b.iter(|| { + let mut count = 0usize; + fjall + .for_each_in_range(&prefix, |e| { + count += black_box(e.value.len()); + Ok(()) + }) + .expect("scan"); + black_box(count) + }); + }); + g.bench_function("rocksdb", |b| { + b.iter(|| { + let mut count = 0usize; + rocks + .for_each_in_range(&prefix, |e| { + count += black_box(e.value.len()); + Ok(()) + }) + .expect("scan"); + black_box(count) + }); + }); + g.finish(); + + // --- Batched lookups (rocksdb-only API): one MultiGet vs 100 gets. --- + // Fresh random keys EVERY iteration, generated in `iter_batched` setup so + // key construction is excluded from the timing. A fixed key set would be + // cache-hot after criterion's warmup and measure only per-call overhead — + // MultiGet exists to coalesce filter/index probes and block reads on + // *misses*, so the batches must keep missing. Each arm uses a different + // seed: identical sequences would hand the second arm blocks the first + // arm just pulled into cache. (Page cache still warms globally as the + // group runs — both arms drift faster over time, in run order.) + let make_keys = |state: &mut u64| -> Vec { + (0..100) + .map(|_| key((next_rand(state) % n as u64) as usize)) + .collect() + }; + // RNG state hoisted for the same reason as `get_hit` above: criterion + // re-invokes the closure per sample, and a body-local seed would replay + // the same batches into a warmed page cache. + let mut g = c.benchmark_group("lookup_100"); + g.throughput(Throughput::Elements(100)); + let mut loop_state = 0xFACE_FEEDu64; + g.bench_function("rocksdb_get_loop", |b| { + b.iter_batched( + || make_keys(&mut loop_state), + |keys| { + for k in &keys { + black_box(rocks_reader.get(k).expect("get")); + } + }, + BatchSize::SmallInput, + ); + }); + let mut mg_state = 0xBADC_0FFEu64; + g.bench_function("rocksdb_multi_get", |b| { + b.iter_batched( + || make_keys(&mut mg_state), + |keys| { + black_box( + rocks_reader + .multi_get(keys.iter().map(String::as_str)) + .expect("multi_get"), + ) + }, + BatchSize::SmallInput, + ); + }); + g.finish(); +} + +criterion_group!(benches, bench_apply_hydrate, bench_reads); +criterion_main!(benches); diff --git a/src/nats.rs b/src/nats.rs index 36cf32a..aab50ff 100644 --- a/src/nats.rs +++ b/src/nats.rs @@ -1047,11 +1047,7 @@ impl KvWatcher for NatsKvWatcher { stream_watch(watcher, &tx).await } - async fn watch_prefixes( - &self, - prefixes: &[&str], - tx: Sender, - ) -> Result<(), KvError> { + async fn watch_prefixes(&self, prefixes: &[&str], tx: Sender) -> Result<(), KvError> { if prefixes.is_empty() { // Nothing to watch. Critically, do NOT fall through to `watch_many` // with an empty filter set — an unfiltered ordered consumer would diff --git a/src/snapshot_record.rs b/src/snapshot_record.rs new file mode 100644 index 0000000..4ca89b8 --- /dev/null +++ b/src/snapshot_record.rs @@ -0,0 +1,145 @@ +//! Shared on-disk value-record codec for the LSM-backed [`SnapshotStore`] +//! backends (`FjallSnapshot`, `RocksDbSnapshot`). +//! +//! Both backends store the folded KV state as `key` → `[ver_len:u8][version +//! bytes][value bytes]`. Keeping the codec (and its corruption tests) in one +//! place means the record format cannot drift between backends — a store +//! written by one decodes identically in the other's terms. +//! +//! [`SnapshotStore`]: crate::snapshot::SnapshotStore + +use crate::kv::{KvEntry, VersionToken}; +use crate::snapshot::SnapshotError; + +/// Encode a stored value as `[ver_len:u8][version bytes][value bytes]` into `buf`. +/// +/// `buf` is cleared and refilled (its capacity is reused across a batch). The +/// version is length-prefixed raw bytes for the same reason the append-log format +/// uses it: a backend's token (NATS u64, FDB 10-byte versionstamp) must round-trip +/// intact. +/// +/// `VersionToken` caps inline storage at 10 bytes, so the `u8` length prefix never +/// truncates today. Checking with `try_from` rather than casting surfaces a format +/// error instead of silently writing a wrong length — which would frame a record +/// `decode_entry` then mis-parses — if a future token ever widens past 255 bytes. +/// This mirrors `write_put_record` in `snapshot.rs`. +pub(crate) fn encode_value_into( + buf: &mut Vec, + value: &[u8], + version: &VersionToken, +) -> Result<(), SnapshotError> { + let vb = version.as_bytes(); + let ver_len = u8::try_from(vb.len()).map_err(|_| { + SnapshotError::InvalidFormat(format!( + "version too long: {} bytes (max {})", + vb.len(), + u8::MAX + )) + })?; + buf.clear(); + buf.reserve(1 + vb.len() + value.len()); + buf.push(ver_len); + buf.extend_from_slice(vb); + buf.extend_from_slice(value); + Ok(()) +} + +/// Decode a `[ver_len:u8][version][value]` record back into a [`KvEntry`]. +pub(crate) fn decode_entry(key: &str, raw: &[u8]) -> Result { + let ver_len = *raw.first().ok_or_else(|| { + SnapshotError::InvalidFormat("snapshot value record is empty (no version length)".into()) + })? as usize; + let value_off = 1 + ver_len; + if raw.len() < value_off { + return Err(SnapshotError::InvalidFormat(format!( + "snapshot value record truncated: need {value_off} bytes for version, have {}", + raw.len() + ))); + } + let version = VersionToken::from_raw(&raw[1..value_off]).ok_or_else(|| { + SnapshotError::InvalidFormat(format!( + "version length {ver_len} exceeds version token capacity" + )) + })?; + Ok(KvEntry { + key: key.to_string(), + value: raw[value_off..].to_vec(), + version, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A 10-byte FDB versionstamp has no `u64` form; the length-prefixed value + /// format must carry it intact. A `u64`-only field would flatten it to 0 and + /// silently break every later CAS — so this is the load-bearing reason the + /// record stores a length-prefixed token rather than a fixed 8 bytes. + #[test] + fn encode_decode_round_trips_fdb_versionstamp() { + let vs = VersionToken::from_fdb_versionstamp(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let mut enc = Vec::new(); + encode_value_into(&mut enc, b"payload", &vs).expect("encode"); + let entry = decode_entry("k", &enc).expect("decode"); + + assert_eq!(entry.version.as_bytes(), &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert!( + entry.version.as_u64().is_none(), + "a 10-byte token has no u64 form — it must not be flattened" + ); + assert_eq!(entry.value, b"payload"); + } + + /// An empty value (the CAS-tombstone shape) encodes to just the version prefix + /// and decodes back to a present, empty-valued entry with its version intact. + #[test] + fn encode_decode_round_trips_empty_value() { + let mut enc = Vec::new(); + encode_value_into(&mut enc, b"", &VersionToken::from_u64(7)).expect("encode"); + let entry = decode_entry("k", &enc).expect("decode"); + + assert!(entry.value.is_empty()); + assert_eq!(entry.version.as_u64(), Some(7)); + } + + /// A zero-byte record has no version-length byte — corruption, not a valid + /// record. It must surface as a recoverable `InvalidFormat`, never a panic. + #[test] + fn decode_entry_rejects_empty_record() { + let err = decode_entry("k", &[]).unwrap_err(); + assert!( + matches!(err, SnapshotError::InvalidFormat(_)), + "empty record must be a format error, got {err:?}" + ); + } + + /// A record that claims a longer version than its bytes provide is truncated + /// on-disk corruption — reject it instead of reading past the buffer. + #[test] + fn decode_entry_rejects_truncated_version() { + // Claims a 5-byte version, but only 2 bytes follow the length prefix. + let raw = [5u8, 0xAA, 0xBB]; + let err = decode_entry("k", &raw).unwrap_err(); + assert!( + matches!(err, SnapshotError::InvalidFormat(_)), + "truncated version must be a format error, got {err:?}" + ); + } + + /// A version length beyond `VersionToken`'s 10-byte capacity can't round-trip; + /// `from_raw` rejects it and `decode_entry` maps that to `InvalidFormat` rather + /// than silently truncating to a wrong (CAS-breaking) version. + #[test] + fn decode_entry_rejects_oversized_version() { + // ver_len = 11 with 11 trailing bytes: passes the truncation check, then + // trips the capacity check inside `VersionToken::from_raw`. + let mut raw = vec![11u8]; + raw.extend_from_slice(&[0u8; 11]); + let err = decode_entry("k", &raw).unwrap_err(); + assert!( + matches!(err, SnapshotError::InvalidFormat(_)), + "oversized version must be a format error, got {err:?}" + ); + } +} diff --git a/tests/integration.rs b/tests/integration.rs index a46b72d..05ff40b 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -614,7 +614,9 @@ async fn watch_prefixes_unions_on_a_single_consumer() { ); // vpcC.x must NEVER arrive — server-side filtered, not client-discarded. assert!( - timeout(Duration::from_millis(500), rx.recv()).await.is_err(), + timeout(Duration::from_millis(500), rx.recv()) + .await + .is_err(), "a non-watched prefix leaked through watch_prefixes" ); } From 804036fc1f82dacbb0c1046467e640a35f3c60a8 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:11:49 -0700 Subject: [PATCH 02/12] feat(snapshot)!: artifact codec + SnapshotStore::{cursor,export_to}; append-log export/import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An artifact is a transferable copy of a durable fold: backend files under data/ plus a MANIFEST.json carrying per-file BLAKE3 checksums, backend identity + on-disk format generation, and the watch cursor the payload is exactly consistent with. Export stages, seals (manifest written last, fsynced), and atomically renames — an artifact that exists is complete. Import verifies everything (transport is untrusted), opens the staged copy, gates on cursor equality, and renames — a bad artifact never becomes a fold. Breaking: SnapshotStore gains required cursor() and export_to(); SnapshotError gains ArtifactInvalid; WatchCursor is now PartialEq + Eq. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 263 +++++++++++++++- Cargo.toml | 14 +- src/artifact.rs | 786 ++++++++++++++++++++++++++++++++++++++++++++++++ src/kv.rs | 5 +- src/lib.rs | 8 + src/snapshot.rs | 143 +++++++++ 6 files changed, 1202 insertions(+), 17 deletions(-) create mode 100644 src/artifact.rs diff --git a/Cargo.lock b/Cargo.lock index c52448f..ed0f45a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,18 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-nats" version = "0.46.0" @@ -97,16 +109,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "beyond-slipstream" -version = "0.3.2" +version = "0.3.3" dependencies = [ "async-nats", "async-trait", "base64", + "blake3", "crc32fast", "criterion", "fjall", "futures", - "lsm-tree", + "rust-rocksdb", + "serde", "serde_json", "tempfile", "thiserror 2.0.18", @@ -115,12 +129,44 @@ dependencies = [ "url", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn", +] + [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -157,6 +203,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cast" version = "0.3.0" @@ -170,7 +226,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", - "shlex", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -206,6 +273,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.1" @@ -243,6 +321,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -268,6 +352,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -371,7 +464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -505,7 +598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -676,6 +769,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -684,11 +789,17 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "half" version = "2.7.1" @@ -892,7 +1003,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -910,6 +1021,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.99" @@ -934,6 +1055,27 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -983,6 +1125,16 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.13.1" @@ -998,6 +1150,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.1" @@ -1024,6 +1182,16 @@ dependencies = [ "signatory", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nuid" version = "0.5.0" @@ -1066,6 +1234,16 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + [[package]] name = "parking_lot_core" version = "0.9.12" @@ -1130,6 +1308,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plotters" version = "0.3.7" @@ -1226,6 +1410,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -1334,6 +1524,34 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rust-librocksdb-sys" +version = "0.46.0+11.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8897839fc0d7842aff6eb04e1c2fe8faa160c9af7e9ec3af1522225df2ccc308" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "pkg-config", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f15d775a71972647245bd2c64eb1790e25b02b39fd51a6a4d5c7d940d6e06821" +dependencies = [ + "libc", + "parking_lot", + "rust-librocksdb-sys", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1359,7 +1577,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1574,10 +1792,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -1691,7 +1915,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1961,6 +2185,12 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2114,7 +2344,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2419,3 +2649,14 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "bindgen", + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index e3031b1..826427f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ rust-version = "1.92" [package] name = "beyond-slipstream" -version = "0.3.2" +version = "0.3.3" edition.workspace = true license.workspace = true rust-version.workspace = true @@ -22,10 +22,12 @@ path = "src/lib.rs" async-nats = "0.46" async-trait = "0.1" base64 = "0.22" +blake3 = "1.8.5" crc32fast = "1" fjall = { version = "3", optional = true } -lsm-tree = { version = "3", optional = true } futures = "0.3" +rocksdb = { package = "rust-rocksdb", version = "0.50", optional = true, default-features = false, features = ["lz4", "zstd", "bindgen-runtime"] } +serde = { version = "1", features = ["derive"] } serde_json = "1" tempfile = "3" thiserror = "2" @@ -49,5 +51,11 @@ harness = false name = "applied" harness = false +[[bench]] +name = "snapshot_backends" +harness = false +required-features = ["fjall", "rocksdb"] + [features] -fjall = ["dep:fjall", "dep:lsm-tree"] +fjall = ["dep:fjall"] +rocksdb = ["dep:rocksdb"] diff --git a/src/artifact.rs b/src/artifact.rs new file mode 100644 index 0000000..4dc6c7b --- /dev/null +++ b/src/artifact.rs @@ -0,0 +1,786 @@ +//! Artifact codec and staging for snapshot export/import (replica bootstrap). +//! +//! An **artifact** is a transferable copy of a durable fold: a directory holding +//! the backend's files under `data/` plus a [`MANIFEST_FILE`] recording the +//! backend's identity, its on-disk format generation, per-file checksums, and — +//! the load-bearing part — the **watch cursor the files are consistent with**. +//! Export produces one; import verifies and installs one; the consumer resumes +//! its KV watch from the embedded cursor and replays only the log tail. +//! +//! This module owns what is common across backends: the manifest wire format, +//! checksum verification, and the stage-then-atomic-rename discipline that keeps +//! half-written artifacts and half-imported folds from ever being observable at +//! their final paths. The backend-specific parts (how fjall/RocksDB/the append +//! log get a consistent copy of their files) live with each backend. +//! +//! ## Crash safety +//! +//! - **Export**: payload and manifest are assembled in a hidden temp directory +//! beside `dest`, fsynced, and renamed into place. A crash mid-export leaves a +//! `.slipstream-artifact-*` temp dir (cleaned up by the next tempdir reaper or +//! operator) and **no** artifact at `dest`. An artifact that exists is complete. +//! - **Import**: the payload is copied-and-verified into a temp directory beside +//! the destination, then renamed. A crash mid-import leaves no fold at the +//! destination; a crash after the rename leaves a fully valid fold (a retried +//! import then refuses the non-empty destination — the caller should simply +//! open it). +//! +//! ## Blocking I/O +//! +//! Everything here is synchronous file I/O, same discipline as +//! [`crate::snapshot`]: async callers must offload to `spawn_blocking` (the +//! [`watch_applied`](crate::watch_applied) export path does). + +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +use crate::kv::{VersionToken, WatchCursor}; +use crate::snapshot::SnapshotError; + +/// Version of the artifact layout itself (`MANIFEST.json` schema + `data/` +/// payload convention). Bumped only when the artifact shape changes; the +/// *payload* format is governed separately by [`ExportManifest::backend_version`]. +pub const ARTIFACT_SCHEMA_VERSION: u32 = 1; + +/// Manifest file name at the artifact root. Written last and fsynced, so its +/// presence (after the atomic rename) means the artifact is complete. +pub const MANIFEST_FILE: &str = "MANIFEST.json"; + +/// Directory under the artifact root holding the backend's payload files. +pub(crate) const PAYLOAD_DIR: &str = "data"; + +/// Streaming-hash buffer size. +const HASH_BUF: usize = 1 << 20; + +/// Manifest of an exported artifact: what is in it, which backend wrote it, and +/// the cursor its payload is consistent with. +#[derive(Debug, Clone)] +pub struct ExportManifest { + /// Artifact layout version ([`ARTIFACT_SCHEMA_VERSION`]). + pub schema_version: u32, + /// Backend identity: `"append-log"`, `"fjall"`, or `"rocksdb"`. + pub backend: String, + /// The backend's **on-disk format generation** (not a crate semver): the + /// append log's `FORMAT_VERSION` (`"2"`), fjall's format marker (`"3"`), + /// the rust-rocksdb binding version (`"0.50"`, informational — RocksDB + /// reads older formats and its own open is the arbiter). + pub backend_version: String, + /// The resume cursor the payload is exactly consistent with. Resuming the + /// watch from here replays only the post-export tail. + pub cursor: WatchCursor, + /// Export wall-clock time, seconds since the Unix epoch. Informational. + pub created_at_unix: u64, + /// Every payload file, with size and BLAKE3 digest. Import verifies all of + /// them and rejects undeclared extras. + pub files: Vec, +} + +/// One payload file in an [`ExportManifest`]. +#[derive(Debug, Clone)] +pub struct ArtifactFile { + /// Path relative to the artifact root, `/`-separated, always under `data/`. + pub path: String, + /// Size in bytes. + pub size: u64, + /// Lowercase-hex BLAKE3 digest of the file contents. (BLAKE3 over SHA-256: + /// there is no interop constraint — slipstream writes and reads its own + /// manifests — and artifacts reach GBs, where BLAKE3's SIMD hashing is + /// several times faster.) + pub blake3: String, +} + +// --------------------------------------------------------------------------- +// Wire format (serde) — kept separate from the public types so the public +// surface can hold a real WatchCursor while the JSON stays a stable hex string. +// --------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize)] +struct ManifestWire { + schema_version: u32, + backend: String, + backend_version: String, + /// Raw cursor bytes as lowercase hex; empty string = no cursor + /// ([`WatchCursor::none`]). + cursor_hex: String, + created_at_unix: u64, + files: Vec, +} + +#[derive(Serialize, Deserialize)] +struct FileWire { + path: String, + size: u64, + blake3: String, +} + +fn invalid(msg: impl Into) -> SnapshotError { + SnapshotError::ArtifactInvalid(msg.into()) +} + +// --------------------------------------------------------------------------- +// Hex (cursor encoding) — two tiny helpers instead of a dependency. +// --------------------------------------------------------------------------- + +pub(crate) fn hex_encode(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for b in bytes { + use std::fmt::Write as _; + let _ = write!(out, "{b:02x}"); + } + out +} + +pub(crate) fn hex_decode(s: &str) -> Option> { + if s.len() % 2 != 0 { + return None; + } + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()) + .collect() +} + +fn cursor_to_hex(cursor: &WatchCursor) -> String { + hex_encode(cursor.version().as_bytes()) +} + +fn cursor_from_hex(s: &str) -> Result { + let bytes = hex_decode(s).ok_or_else(|| invalid(format!("malformed cursor_hex: {s:?}")))?; + let token = VersionToken::from_raw(&bytes).ok_or_else(|| { + invalid(format!( + "cursor_hex decodes to {} bytes, exceeds version token capacity", + bytes.len() + )) + })?; + Ok(WatchCursor::from_version(token)) +} + +// --------------------------------------------------------------------------- +// Manifest read/write +// --------------------------------------------------------------------------- + +/// Serialize and write `MANIFEST.json` at the artifact root: tempfile in the +/// same directory, fsync, atomic rename — the manifest is the artifact's +/// completeness marker, so it must never be observable half-written. +pub(crate) fn write_manifest( + artifact_root: &Path, + manifest: &ExportManifest, +) -> Result<(), SnapshotError> { + let wire = ManifestWire { + schema_version: manifest.schema_version, + backend: manifest.backend.clone(), + backend_version: manifest.backend_version.clone(), + cursor_hex: cursor_to_hex(&manifest.cursor), + created_at_unix: manifest.created_at_unix, + files: manifest + .files + .iter() + .map(|f| FileWire { + path: f.path.clone(), + size: f.size, + blake3: f.blake3.clone(), + }) + .collect(), + }; + let json = serde_json::to_vec_pretty(&wire) + .map_err(|e| SnapshotError::Backend(format!("manifest serialization failed: {e}")))?; + + let mut tmp = tempfile::NamedTempFile::new_in(artifact_root)?; + tmp.write_all(&json)?; + tmp.as_file().sync_all()?; + tmp.persist(artifact_root.join(MANIFEST_FILE)) + .map_err(|e| SnapshotError::Io(e.error))?; + Ok(()) +} + +/// Read and validate `MANIFEST.json` from an artifact directory. +/// +/// Validates the schema version and every file path (relative, `/`-separated, +/// no `..`, no `\`, under `data/`) so a hostile or corrupted manifest can never +/// direct a copy outside the staging area (zip-slip). +pub(crate) fn read_manifest(artifact_dir: &Path) -> Result { + let path = artifact_dir.join(MANIFEST_FILE); + let data = fs::read(&path).map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + invalid(format!("no {MANIFEST_FILE} in {}", artifact_dir.display())) + } else { + SnapshotError::Io(e) + } + })?; + let wire: ManifestWire = + serde_json::from_slice(&data).map_err(|e| invalid(format!("malformed manifest: {e}")))?; + + if wire.schema_version != ARTIFACT_SCHEMA_VERSION { + return Err(invalid(format!( + "unsupported artifact schema_version {} (this build supports {})", + wire.schema_version, ARTIFACT_SCHEMA_VERSION + ))); + } + for f in &wire.files { + validate_payload_path(&f.path)?; + } + + Ok(ExportManifest { + schema_version: wire.schema_version, + backend: wire.backend, + backend_version: wire.backend_version, + cursor: cursor_from_hex(&wire.cursor_hex)?, + created_at_unix: wire.created_at_unix, + files: wire + .files + .into_iter() + .map(|f| ArtifactFile { + path: f.path, + size: f.size, + blake3: f.blake3, + }) + .collect(), + }) +} + +/// Reject any manifest path that could escape the artifact when joined: it must +/// be relative, `/`-separated, contain no `..`/`.` components, and live under +/// `data/`. +fn validate_payload_path(p: &str) -> Result<(), SnapshotError> { + let prefix = format!("{PAYLOAD_DIR}/"); + if !p.starts_with(&prefix) || p.len() == prefix.len() { + return Err(invalid(format!( + "manifest path {p:?} is not under {PAYLOAD_DIR}/" + ))); + } + if p.contains('\\') { + return Err(invalid(format!("manifest path {p:?} contains a backslash"))); + } + let path = Path::new(p); + for comp in path.components() { + match comp { + Component::Normal(_) => {} + _ => { + return Err(invalid(format!( + "manifest path {p:?} contains a non-normal component" + ))); + } + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Payload hashing +// --------------------------------------------------------------------------- + +/// Streaming BLAKE3 of one file, returning `(size, hex_digest)`. +fn hash_file(path: &Path) -> Result<(u64, String), SnapshotError> { + let mut file = File::open(path)?; + let mut hasher = blake3::Hasher::new(); + let mut buf = vec![0u8; HASH_BUF]; + let mut size = 0u64; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + size += n as u64; + hasher.update(&buf[..n]); + } + Ok((size, hasher.finalize().to_hex().to_string())) +} + +/// Every regular file under `root/data/`, relative `/`-separated paths, sorted. +fn list_payload_files(root: &Path) -> Result, SnapshotError> { + let payload = root.join(PAYLOAD_DIR); + let mut out = Vec::new(); + let mut stack = vec![payload.clone()]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + stack.push(entry.path()); + } else if ty.is_file() { + out.push(entry.path()); + } else { + // Symlinks etc. have no place in an artifact: a symlink would + // hash as its target's bytes but restore as a link (or escape + // the payload entirely). Refuse at export so import never has + // to trust one. + return Err(invalid(format!( + "payload contains a non-regular file: {}", + entry.path().display() + ))); + } + } + } + out.sort(); + Ok(out) +} + +/// Hash every payload file under `root/data/` and fsync the payload tree +/// (files + directories), returning the manifest file list in sorted order. +pub(crate) fn hash_payload(root: &Path) -> Result, SnapshotError> { + let mut files = Vec::new(); + for abs in list_payload_files(root)? { + let (size, blake3) = hash_file(&abs)?; + // Durability before the rename: the artifact's completeness contract is + // "exists ⇒ verifiable", which only holds if the hashed bytes are the + // on-disk bytes. + File::open(&abs)?.sync_all()?; + let rel = abs + .strip_prefix(root) + .map_err(|_| SnapshotError::Backend("payload path escaped artifact root".into()))?; + let rel = rel + .to_str() + .ok_or_else(|| invalid(format!("non-UTF-8 payload path: {}", rel.display())))?; + files.push(ArtifactFile { + path: rel.to_string(), + size, + blake3, + }); + } + fsync_dir_tree(&root.join(PAYLOAD_DIR))?; + Ok(files) +} + +fn fsync_dir(path: &Path) -> Result<(), SnapshotError> { + File::open(path)?.sync_all()?; + Ok(()) +} + +fn fsync_dir_tree(root: &Path) -> Result<(), SnapshotError> { + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + fsync_dir(&dir)?; + for entry in fs::read_dir(&dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + stack.push(entry.path()); + } + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Destination preconditions +// --------------------------------------------------------------------------- + +/// A destination is available when it does not exist, or is an empty directory +/// (removed just before the final rename). Anything else is refused — never +/// overwrite a fold or an artifact in place. +pub(crate) fn check_dest_available(dest: &Path) -> Result<(), SnapshotError> { + match fs::metadata(dest) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(SnapshotError::Io(e)), + Ok(meta) if meta.is_dir() => { + if fs::read_dir(dest)?.next().is_some() { + Err(invalid(format!( + "destination {} exists and is not empty", + dest.display() + ))) + } else { + Ok(()) + } + } + Ok(_) => Err(invalid(format!( + "destination {} already exists", + dest.display() + ))), + } +} + +/// Remove `dest` if it is an (already-verified-empty) directory so a rename can +/// land on its path, then rename `from` onto it and fsync the parent. +fn rename_into_place(from: &Path, dest: &Path) -> Result<(), SnapshotError> { + if dest.is_dir() { + fs::remove_dir(dest)?; + } + fs::rename(from, dest)?; + if let Some(parent) = dest.parent() { + fsync_dir(parent)?; + } + Ok(()) +} + +fn stage_dir_in(parent: &Path) -> Result { + // Same directory as the destination so the final rename is a same-filesystem + // atomic rename (mirrors `compact_to_file`'s tempfile discipline). The + // hidden prefix keeps half-built stages out of an operator's way. + Ok(tempfile::Builder::new() + .prefix(".slipstream-artifact-") + .tempdir_in(parent)?) +} + +fn dest_parent(dest: &Path) -> Result<&Path, SnapshotError> { + dest.parent().filter(|p| !p.as_os_str().is_empty()).ok_or_else(|| { + invalid(format!( + "destination {} has no parent directory", + dest.display() + )) + }) +} + +// --------------------------------------------------------------------------- +// Export staging +// --------------------------------------------------------------------------- + +/// Assembles an artifact in a temp directory beside `dest`, then seals it +/// (hash → manifest → fsync) and atomically renames it into place. +/// +/// Backends write their payload into [`payload`](Self::payload), then call +/// [`seal_and_finalize`](Self::seal_and_finalize). +pub(crate) struct ExportStage { + dir: tempfile::TempDir, + dest: PathBuf, +} + +impl ExportStage { + /// Create a stage for an artifact that will land at `dest_dir`. Fails fast + /// if `dest_dir` is unavailable (exists non-empty). + pub(crate) fn new(dest_dir: &Path) -> Result { + check_dest_available(dest_dir)?; + let parent = dest_parent(dest_dir)?; + let dir = stage_dir_in(parent)?; + Ok(Self { + dir, + dest: dest_dir.to_path_buf(), + }) + } + + /// The payload directory the backend writes its files into. + /// + /// Deliberately NOT pre-created: RocksDB's checkpoint API requires its + /// target to not exist, so each backend creates (or lets the engine create) + /// this path itself. + pub(crate) fn payload(&self) -> PathBuf { + self.dir.path().join(PAYLOAD_DIR) + } + + /// Hash the payload, write the manifest, fsync, and atomically rename the + /// stage to the destination. Returns the sealed manifest. + pub(crate) fn seal_and_finalize( + self, + backend: &str, + backend_version: &str, + cursor: &WatchCursor, + ) -> Result { + let root = self.dir.path(); + let files = hash_payload(root)?; + let manifest = ExportManifest { + schema_version: ARTIFACT_SCHEMA_VERSION, + backend: backend.to_string(), + backend_version: backend_version.to_string(), + cursor: cursor.clone(), + created_at_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + files, + }; + write_manifest(root, &manifest)?; + fsync_dir(root)?; + + // Re-check the destination (it may have appeared since `new`), then + // rename. `keep()` disarms the TempDir destructor — the path has been + // renamed away, so there is nothing left for it to delete. + check_dest_available(&self.dest)?; + let dest = self.dest.clone(); + let root = self.dir.keep(); + rename_into_place(&root, &dest)?; + Ok(manifest) + } +} + +// --------------------------------------------------------------------------- +// Import staging +// --------------------------------------------------------------------------- + +/// A verified copy of an artifact's payload, staged beside the destination and +/// ready to be renamed into place. +pub(crate) struct ImportStage { + dir: tempfile::TempDir, + dest: PathBuf, +} + +impl ImportStage { + /// The staged payload root (mirrors the artifact's `data/` layout). + pub(crate) fn payload(&self) -> PathBuf { + self.dir.path().join(PAYLOAD_DIR) + } + + /// Rename the whole staged payload directory onto `dest` (directory-shaped + /// backends: fjall, RocksDB). + pub(crate) fn finalize_dir(self) -> Result<(), SnapshotError> { + check_dest_available(&self.dest)?; + rename_into_place(&self.payload(), &self.dest) + // TempDir drop removes the now-payload-less stage directory. + } + + /// Rename a single staged payload file onto `dest` (file-shaped backends: + /// the append log). + pub(crate) fn finalize_file(self, rel: &str) -> Result<(), SnapshotError> { + check_dest_available(&self.dest)?; + rename_into_place(&self.payload().join(rel), &self.dest) + } +} + +/// Validate an artifact against its manifest and stage a verified copy of its +/// payload beside `dest`. +/// +/// Checks, in order: manifest well-formedness ([`read_manifest`]), backend +/// identity, backend version (via the caller's policy closure), destination +/// availability, then every payload file (copied while hashing — size and +/// BLAKE3 digest must match the manifest, and the payload must contain no undeclared +/// extra files). The transport that delivered the artifact is untrusted; this +/// re-verification is the trust boundary. +pub(crate) fn verify_and_stage_import( + artifact_dir: &Path, + dest: &Path, + expected_backend: &str, + check_backend_version: impl Fn(&str) -> Result<(), SnapshotError>, +) -> Result<(ExportManifest, ImportStage), SnapshotError> { + let manifest = read_manifest(artifact_dir)?; + + if manifest.backend != expected_backend { + return Err(invalid(format!( + "artifact backend is {:?}, expected {:?}", + manifest.backend, expected_backend + ))); + } + check_backend_version(&manifest.backend_version)?; + check_dest_available(dest)?; + + // Undeclared extras: a file in the payload that the manifest doesn't list + // was never hashed at export — it cannot be trusted. + let declared: BTreeSet<&str> = manifest.files.iter().map(|f| f.path.as_str()).collect(); + for abs in list_payload_files(artifact_dir)? { + let rel = abs + .strip_prefix(artifact_dir) + .map_err(|_| SnapshotError::Backend("payload path escaped artifact dir".into()))?; + let rel = rel + .to_str() + .ok_or_else(|| invalid(format!("non-UTF-8 payload path: {}", rel.display())))?; + if !declared.contains(rel) { + return Err(invalid(format!( + "payload contains undeclared file: {rel}" + ))); + } + } + + let parent = dest_parent(dest)?; + let dir = stage_dir_in(parent)?; + let stage = ImportStage { + dir, + dest: dest.to_path_buf(), + }; + + // Copy-while-hashing: one read pass per file serves both the copy and the + // verification. + for f in &manifest.files { + let src_path = artifact_dir.join(&f.path); + let dst_path = stage.dir.path().join(&f.path); + if let Some(p) = dst_path.parent() { + fs::create_dir_all(p)?; + } + let mut src = File::open(&src_path).map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + invalid(format!("payload file missing: {}", f.path)) + } else { + SnapshotError::Io(e) + } + })?; + let mut dst = File::create(&dst_path)?; + let mut hasher = blake3::Hasher::new(); + let mut buf = vec![0u8; HASH_BUF]; + let mut size = 0u64; + loop { + let n = src.read(&mut buf)?; + if n == 0 { + break; + } + size += n as u64; + hasher.update(&buf[..n]); + dst.write_all(&buf[..n])?; + } + dst.sync_all()?; + if size != f.size { + return Err(invalid(format!( + "payload file {} is {size} bytes, manifest says {}", + f.path, f.size + ))); + } + let digest = hasher.finalize().to_hex().to_string(); + if digest != f.blake3 { + return Err(invalid(format!( + "payload file {} checksum mismatch (got {digest}, manifest says {})", + f.path, f.blake3 + ))); + } + } + fsync_dir_tree(&stage.payload())?; + + Ok((manifest, stage)) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn manifest_with(files: Vec, cursor: WatchCursor) -> ExportManifest { + ExportManifest { + schema_version: ARTIFACT_SCHEMA_VERSION, + backend: "append-log".into(), + backend_version: "2".into(), + cursor, + created_at_unix: 1_765_400_000, + files, + } + } + + #[test] + fn manifest_round_trips() { + let dir = TempDir::new().unwrap(); + let m = manifest_with( + vec![ArtifactFile { + path: "data/fold.snap".into(), + size: 42, + blake3: "ab".repeat(32), + }], + WatchCursor::from_u64(184_467), + ); + write_manifest(dir.path(), &m).unwrap(); + let got = read_manifest(dir.path()).unwrap(); + assert_eq!(got.schema_version, m.schema_version); + assert_eq!(got.backend, m.backend); + assert_eq!(got.backend_version, m.backend_version); + assert_eq!(got.cursor, m.cursor); + assert_eq!(got.created_at_unix, m.created_at_unix); + assert_eq!(got.files.len(), 1); + assert_eq!(got.files[0].path, "data/fold.snap"); + assert_eq!(got.files[0].size, 42); + } + + #[test] + fn manifest_round_trips_none_cursor() { + let dir = TempDir::new().unwrap(); + let m = manifest_with(vec![], WatchCursor::none()); + write_manifest(dir.path(), &m).unwrap(); + let got = read_manifest(dir.path()).unwrap(); + assert!(got.cursor.is_none(), "none cursor survives the round trip"); + } + + #[test] + fn manifest_round_trips_fdb_width_cursor() { + // 10-byte tokens have no u64 form; the hex path must carry them intact. + let raw = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let cursor = WatchCursor::from_version(VersionToken::from_raw(&raw).unwrap()); + let dir = TempDir::new().unwrap(); + write_manifest(dir.path(), &manifest_with(vec![], cursor.clone())).unwrap(); + let got = read_manifest(dir.path()).unwrap(); + assert_eq!(got.cursor, cursor); + } + + fn write_raw_manifest(dir: &Path, json: &str) { + fs::write(dir.join(MANIFEST_FILE), json).unwrap(); + } + + fn wire_json(cursor_hex: &str, files: &str, schema: u32) -> String { + format!( + r#"{{"schema_version":{schema},"backend":"append-log","backend_version":"2", + "cursor_hex":"{cursor_hex}","created_at_unix":0,"files":{files}}}"# + ) + } + + #[test] + fn rejects_bad_cursor_hex() { + let dir = TempDir::new().unwrap(); + for bad in ["zz", "abc", "0102030405060708090a0b"] { + // non-hex, odd length, 11 bytes (> token capacity) + write_raw_manifest(dir.path(), &wire_json(bad, "[]", ARTIFACT_SCHEMA_VERSION)); + match read_manifest(dir.path()) { + Err(SnapshotError::ArtifactInvalid(_)) => {} + other => panic!("cursor_hex {bad:?}: expected ArtifactInvalid, got {other:?}"), + } + } + } + + #[test] + fn rejects_wrong_schema_version() { + let dir = TempDir::new().unwrap(); + write_raw_manifest(dir.path(), &wire_json("", "[]", ARTIFACT_SCHEMA_VERSION + 1)); + match read_manifest(dir.path()) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("schema_version"), "{msg}"); + } + other => panic!("expected ArtifactInvalid, got {other:?}"), + } + } + + #[test] + fn rejects_path_traversal() { + let dir = TempDir::new().unwrap(); + for bad in [ + "../escape", + "/abs/path", + "data/../escape", + "data/a\\b", + "nondata/x", + "data/", + "data", + ] { + let files = format!( + r#"[{{"path":"{}","size":0,"blake3":""}}]"#, + bad.replace('\\', "\\\\") + ); + write_raw_manifest(dir.path(), &wire_json("", &files, ARTIFACT_SCHEMA_VERSION)); + match read_manifest(dir.path()) { + Err(SnapshotError::ArtifactInvalid(_)) => {} + other => panic!("path {bad:?}: expected ArtifactInvalid, got {other:?}"), + } + } + } + + #[test] + fn hex_round_trips() { + for bytes in [&[][..], &[0u8][..], &[0xde, 0xad, 0xbe, 0xef][..]] { + assert_eq!(hex_decode(&hex_encode(bytes)).unwrap(), bytes); + } + assert!(hex_decode("0g").is_none()); + assert!(hex_decode("a").is_none()); + } + + #[test] + fn dest_preconditions() { + let dir = TempDir::new().unwrap(); + // Absent: fine. + check_dest_available(&dir.path().join("absent")).unwrap(); + // Empty dir: fine. + let empty = dir.path().join("empty"); + fs::create_dir(&empty).unwrap(); + check_dest_available(&empty).unwrap(); + // Non-empty dir: refused. + let full = dir.path().join("full"); + fs::create_dir(&full).unwrap(); + fs::write(full.join("x"), b"x").unwrap(); + assert!(matches!( + check_dest_available(&full), + Err(SnapshotError::ArtifactInvalid(_)) + )); + // Existing file: refused. + let file = dir.path().join("file"); + fs::write(&file, b"x").unwrap(); + assert!(matches!( + check_dest_available(&file), + Err(SnapshotError::ArtifactInvalid(_)) + )); + } +} diff --git a/src/kv.rs b/src/kv.rs index 6f6dc4f..bf7369a 100644 --- a/src/kv.rs +++ b/src/kv.rs @@ -7,7 +7,7 @@ use tokio::sync::mpsc::Sender; /// Backends store whatever they need to resume (NATS: u64 revision). /// Callers should treat this as opaque and only pass it back to /// `watch_all_from` / `watch_prefix_from`. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct WatchCursor(VersionToken); impl WatchCursor { @@ -269,8 +269,7 @@ pub trait KvWatcher: Send + Sync { /// per-stream resource (measured at ~tens of KB of server state each, growing /// super-linearly past a few thousand on one stream), so a watcher scoped to N /// prefixes must not cost N consumers. - async fn watch_prefixes(&self, prefixes: &[&str], tx: Sender) - -> Result<(), KvError>; + async fn watch_prefixes(&self, prefixes: &[&str], tx: Sender) -> Result<(), KvError>; /// Resume watching all keys from a previously saved cursor position. /// diff --git a/src/lib.rs b/src/lib.rs index 7c5e4a3..1e34626 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,14 +24,20 @@ #![deny(unsafe_code)] mod applied; +mod artifact; mod kv; mod nats; pub mod snapshot; #[cfg(feature = "fjall")] mod snapshot_fjall; +#[cfg(any(feature = "fjall", feature = "rocksdb"))] +mod snapshot_record; +#[cfg(feature = "rocksdb")] +mod snapshot_rocksdb; mod stores; pub use applied::{BatchConfig, WatchScope, watch_applied}; +pub use artifact::{ARTIFACT_SCHEMA_VERSION, ArtifactFile, ExportManifest, MANIFEST_FILE}; pub use kv::{ KvEntry, KvError, KvReader, KvTtl, KvUpdate, KvWatcher, KvWriter, VersionToken, WatchCursor, }; @@ -39,4 +45,6 @@ pub use nats::{NatsConnection, NatsConnectionConfig, nats_connect}; pub use snapshot::{AppendLogSnapshot, SnapshotStore}; #[cfg(feature = "fjall")] pub use snapshot_fjall::{FjallConfig, FjallReader, FjallSnapshot}; +#[cfg(feature = "rocksdb")] +pub use snapshot_rocksdb::{RocksDbConfig, RocksDbReader, RocksDbSnapshot}; pub use stores::{Connection, ConnectionCapabilities, KvStore, StorageType, StoreConfig}; diff --git a/src/snapshot.rs b/src/snapshot.rs index 2fa8634..332499a 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -43,6 +43,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; +use crate::artifact::{ExportManifest, ExportStage, verify_and_stage_import}; use crate::kv::{KvEntry, KvUpdate, VersionToken, WatchCursor}; const MAGIC: &[u8; 4] = b"PGSS"; @@ -79,6 +80,16 @@ pub enum SnapshotError { /// error surface. #[error("snapshot backend error: {0}")] Backend(String), + /// An export artifact failed validation: malformed/mismatched manifest, + /// checksum mismatch, version-policy rejection, unavailable destination, or a + /// post-import cursor that disagrees with the manifest. + /// + /// Distinct from [`InvalidFormat`](Self::InvalidFormat) (a *store-file* + /// problem — "my local fold is broken") because the recovery is different: an + /// invalid artifact means "fetch another artifact or fall back to a full + /// scan", never "repair the local store". + #[error("invalid snapshot artifact: {0}")] + ArtifactInvalid(String), } /// Result of loading a snapshot from disk. @@ -378,6 +389,39 @@ pub trait SnapshotStore: Sized + Send { } Ok(()) } + + /// The most recently applied (and durably persisted) resume cursor — + /// [`WatchCursor::none`] when nothing has been applied. + fn cursor(&self) -> WatchCursor; + + /// Export this fold as a transferable **artifact directory** at `dest_dir`: + /// the backend's files under `data/` plus a `MANIFEST.json` carrying + /// per-file checksums, the backend's identity and on-disk format generation, + /// and the watch cursor the payload is exactly consistent with + /// (== [`cursor`](Self::cursor) at the moment of export). + /// + /// This is the bootstrap primitive for replicas of a **bounded** log: ship + /// the artifact to a new/rebuilding node, import it there, and resume the + /// watch from the embedded cursor — replaying only the log tail instead of + /// a full state that the log may no longer hold. + /// + /// ## Contract + /// + /// - `dest_dir` must not exist (or be an empty directory). Never overwrites. + /// - On `Ok`, the artifact at `dest_dir` is **complete and fsynced**: it was + /// assembled in a temp directory, the manifest written last, and atomically + /// renamed into place. A crash mid-export leaves no artifact at `dest_dir`. + /// - `&mut self` is deliberate: export must not run concurrently with + /// [`apply`](Self::apply), which is what makes the embedded cursor exact. + /// Inside [`watch_applied`](crate::watch_applied), use its export-request + /// channel rather than calling this directly. + /// - **Artifacts are transient.** Directory-copy backends hardlink immutable + /// files where possible, so a lingering artifact pins storage the source + /// later compacts away — upload (or move) it, then delete it. Stage and + /// destination should be on the same filesystem as the fold; a + /// cross-filesystem destination silently degrades hardlinks to full copies. + /// - Blocking I/O, like everything on this trait. + fn export_to(&mut self, dest_dir: &Path) -> Result; } // --------------------------------------------------------------------------- @@ -405,6 +449,14 @@ pub struct AppendLogSnapshot { } impl AppendLogSnapshot { + /// Backend identity in artifact manifests. + pub(crate) const BACKEND: &'static str = "append-log"; + /// On-disk format generation in artifact manifests (the append log's + /// [`FORMAT_VERSION`]). + pub(crate) const BACKEND_VERSION: &'static str = "2"; + /// The single payload file inside an artifact, relative to its `data/` dir. + const ARTIFACT_PAYLOAD: &'static str = "fold.snap"; + /// Open or resume the log at `path` with an explicit compaction threshold. /// /// Replays any existing log into the in-RAM fold (and compacts it, exactly as @@ -425,6 +477,63 @@ impl AppendLogSnapshot { }, )) } + + /// Import an exported artifact (see [`SnapshotStore::export_to`]) as a new + /// fold at `dest_path`, returning the embedded resume cursor and the opened + /// store. + /// + /// `dest_path` is a **file** path (this backend is a single log file) and + /// must not already exist. The artifact is fully verified against its + /// manifest — checksums, backend identity, format generation — and the + /// staged copy is loaded and its cursor compared against the manifest's + /// before anything lands at `dest_path`; a bad artifact never becomes a + /// fold. A crash mid-import leaves nothing at `dest_path`; a crash after + /// the final rename leaves a valid fold (a retried import then refuses the + /// existing destination — just [`open`](Self::open) it). + pub fn import( + artifact_dir: &Path, + dest_path: &Path, + compact_threshold: u64, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (manifest, stage) = + verify_and_stage_import(artifact_dir, dest_path, Self::BACKEND, |v| { + if v == Self::BACKEND_VERSION { + Ok(()) + } else { + Err(SnapshotError::ArtifactInvalid(format!( + "append-log artifact has format generation {v:?}, this build reads {:?}", + Self::BACKEND_VERSION + ))) + } + })?; + + // This backend's artifact is exactly one payload file. + let expected = format!("{}/{}", crate::artifact::PAYLOAD_DIR, Self::ARTIFACT_PAYLOAD); + if manifest.files.len() != 1 || manifest.files[0].path != expected { + return Err(SnapshotError::ArtifactInvalid(format!( + "append-log artifact must contain exactly {expected:?}" + ))); + } + + // Verify the staged payload opens and agrees with the manifest cursor + // BEFORE the rename — a bad artifact must never land at the destination. + let staged_file = stage.payload().join(Self::ARTIFACT_PAYLOAD); + let staged_cursor = match load(&staged_file)? { + Some(snap) => snap.cursor, + // An empty fold exports to an artifact whose log holds no entries + // and no cursor; `load` reports that as `None`. + None => WatchCursor::none(), + }; + if staged_cursor != manifest.cursor { + return Err(SnapshotError::ArtifactInvalid(format!( + "payload cursor {staged_cursor:?} disagrees with manifest cursor {:?}", + manifest.cursor + ))); + } + + stage.finalize_file(Self::ARTIFACT_PAYLOAD)?; + Self::open(dest_path, compact_threshold) + } } impl SnapshotStore for AppendLogSnapshot { @@ -473,6 +582,40 @@ impl SnapshotStore for AppendLogSnapshot { out.sort_unstable_by(|a, b| a.key.cmp(&b.key)); Ok(out) } + + fn cursor(&self) -> WatchCursor { + self.cursor.clone() + } + + fn export_to(&mut self, dest_dir: &Path) -> Result { + let stage = ExportStage::new(dest_dir)?; + fs::create_dir(stage.payload())?; + // The in-RAM fold is the complete live state, so the artifact payload is + // simply a compacted log written from it — same sorted, CRC'd, synced v2 + // file `compact_to_file` produces for the store itself. No need to flush + // or read back the live log file. + let payload = stage.payload().join(Self::ARTIFACT_PAYLOAD); + compact_to_file(&payload, &self.entries, &self.cursor)?; + + // Verify the payload by loading it back: cursor and entry count must + // match the live fold. Cheap for the in-RAM backend, and keeps the + // "an artifact that exists is trustworthy" guarantee uniform across + // backends. + let (loaded_cursor, loaded_len) = match load(&payload)? { + Some(snap) => (snap.cursor, snap.entries.len()), + None => (WatchCursor::none(), 0), + }; + if loaded_cursor != self.cursor || loaded_len != self.entries.len() { + return Err(SnapshotError::ArtifactInvalid(format!( + "exported payload disagrees with live fold (cursor {loaded_cursor:?} vs {:?}, \ + {loaded_len} vs {} entries)", + self.cursor, + self.entries.len() + ))); + } + + stage.seal_and_finalize(Self::BACKEND, Self::BACKEND_VERSION, &self.cursor) + } } // --------------------------------------------------------------------------- From 8b6934be6cbdf1a2ec9d96e906be9b059eaa5cf9 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:11:49 -0700 Subject: [PATCH 03/12] feat(rocksdb)!: export via native Checkpoint; import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export = rocksdb::checkpoint::Checkpoint into the artifact stage (engine flushes the memtable, hardlinks SSTs — consistent by construction), then verify-by-reopen + cursor-equality gate, hash after the verify handle drains background work, seal, rename. Import does not gate on backend_version: RocksDB reads older on-disk formats; its own open is the arbiter. Includes the previously-uncommitted RocksDbSnapshot backend itself (found in-tree from the unreleased 0.3.3 work); the inherent cursor() accessor is replaced by the trait method. Co-Authored-By: Claude Fable 5 --- src/snapshot_rocksdb.rs | 734 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 734 insertions(+) create mode 100644 src/snapshot_rocksdb.rs diff --git a/src/snapshot_rocksdb.rs b/src/snapshot_rocksdb.rs new file mode 100644 index 0000000..9b8f9ac --- /dev/null +++ b/src/snapshot_rocksdb.rs @@ -0,0 +1,734 @@ +//! On-disk [`SnapshotStore`] backed by [RocksDB](https://docs.rs/rust-rocksdb) — +//! for consumers whose fold is too large to hold in RAM and who prefer the +//! battle-tested C++ LSM (mature operational tooling: `ldb`/`sst_dump`, +//! statistics, decades of production hardening) over the pure-Rust fjall +//! backend. +//! +//! RocksDB is a C++ library: building this backend needs a C++ toolchain and +//! libclang (for bindgen), and the first `librocksdb-sys` compile takes several +//! minutes. That cost is why it is opt-in behind `feature = "rocksdb"` and the +//! crate core stays pure-Rust. The binding is the maintained `rust-rocksdb` +//! fork, renamed to `rocksdb` in `Cargo.toml` so a future binding switch is a +//! one-line manifest change. +//! +//! ## How it honors the [`SnapshotStore`] invariants +//! +//! - **Atomic data + cursor.** Each [`apply`](SnapshotStore::apply) is a single +//! RocksDB [`WriteBatch`]: every put/delete *and* the resume cursor land in one +//! WAL entry — atomic even across column families. There is no window where the +//! cursor names a revision whose data is missing. +//! - **Self-sufficient under NO_SYNC.** The WAL is always on; `sync` only +//! controls whether each commit fsyncs it. With sync off (the default — same +//! cache philosophy as the append log's no-fsync-per-checkpoint path), a commit +//! reaches the OS but is not fsync'd: it survives a process crash (WAL replay on +//! reopen), while a power-loss crash can lose the un-synced *tail*. That is safe +//! precisely because data and cursor are one atomic batch: whatever survived has +//! its matching cursor, so on reopen the consumer resumes the watch from the +//! recovered cursor and re-folds the tail from NATS. Set `sync = true` to fsync +//! every commit. +//! - **Queryable.** [`get`](SnapshotStore::get) and [`range`](SnapshotStore::range) +//! read straight from RocksDB's block-cached storage — no full-DB +//! deserialization — so a 1B-route consumer can build its serving index from a +//! prefix scan. +//! +//! ## Threading +//! +//! RocksDB is synchronous; [`watch_applied`](crate::watch_applied) already +//! offloads [`apply`](SnapshotStore::apply) to a blocking task, and async callers +//! querying [`get`](SnapshotStore::get)/[`range`](SnapshotStore::range) should use +//! `spawn_blocking` likewise. +//! +//! ## Tuning +//! +//! [`open`](RocksDbSnapshot::open) configures RocksDB for the workload this +//! backend exists for — a route-scale fold (model: 1e9 entries, ~60 B keys, +//! ~200 B values ≈ 270 GB raw ≈ 125 GB on disk after compression, NVMe), bulk +//! hydration through `apply`, then steady-state churn with concurrent serving +//! reads that are overwhelmingly point-gets for keys that *exist*, plus +//! per-service prefix scans. RocksDB's own defaults (no filters at all, +//! index/filter blocks outside the cache, 4 KiB blocks, 64 MiB memtables, two +//! background jobs) are wrong at that scale; the constants below encode the +//! corrected configuration, each with its own rationale. The user-facing knobs +//! stay [`RocksDbConfig`]'s `sync` and `cache_size_bytes` — everything else is +//! an opinionated constant. +//! +//! Deliberately rejected, so nobody re-litigates them silently: +//! +//! - **Prefix extractor + prefix bloom.** Slipstream prefixes are +//! variable-length strings; a fixed/capped extractor mis-set is a famous +//! correctness footgun, and our scans already pass iterate bounds +//! ([`PrefixRange`]) which bound the scan without one. +//! - **Universal compaction.** Leveled + dynamic level bytes wins read and +//! space amplification for a read-heavy fold; write amp during hydration is +//! absorbed by parallel compaction. +//! - **Direct I/O.** The NO_SYNC WAL story leans on OS page-cache semantics; +//! mixing direct-I/O SST reads in changes the caching contract for marginal +//! gain at this cache size. +//! - **`atomic_flush`.** Only needed when the WAL is off; ours is always on, +//! and WAL recovery already keeps the cross-CF batch atomic. +//! - **Statistics.** `enable_statistics` costs ~5–10% on the hot path; turn it +//! on locally when investigating, not in the library default. +//! - **zstd dictionary training.** Dictionaries pay off when compression units +//! are too small to self-contextualize; a 16 KiB block already holds ~60 +//! similar route records, so plain bottommost zstd captures the redundancy. + +use std::path::Path; +use std::sync::Arc; + +use rocksdb::checkpoint::Checkpoint; +use rocksdb::{ + BlockBasedIndexType, BlockBasedOptions, Cache, ColumnFamily, ColumnFamilyDescriptor, DB, + DBCompressionType, ErrorKind, IteratorMode, Options, PrefixRange, ReadOptions, + WaitForCompactOptions, WriteBatch, WriteOptions, +}; + +use crate::artifact::{ExportManifest, ExportStage, verify_and_stage_import}; +use crate::kv::{KvEntry, KvUpdate, VersionToken, WatchCursor}; +use crate::snapshot::{SnapshotError, SnapshotStore}; +use crate::snapshot_record::{decode_entry, encode_value_into}; + +/// Column family holding the folded KV state: `key` → encoded `(version, value)`. +const DATA_CF: &str = "data"; +/// Column family holding fold metadata (just the resume cursor today). +const META_CF: &str = "meta"; +/// Key under [`META_CF`] storing the resume cursor's raw version bytes. +const CURSOR_KEY: &[u8] = b"cursor"; + +// --- Tuned constants (see the module-level `## Tuning` docs for the workload +// model behind the math in each comment). --- + +/// Data-block size for the data CF. RocksDB's 4 KiB default generates one index +/// entry (~64 B) per block over *uncompressed* data: at 270 GB raw that is +/// ~4.2 GB of index. 16 KiB cuts the index to ~1.05 GB, improves compression +/// (more context per block), and speeds scans; the cost — a point-get +/// decompresses 16 KiB instead of 4 KiB — is a few µs, paid only on a cache miss. +const DATA_BLOCK_SIZE: usize = 16 * 1024; + +/// Partition granule for the two-level index and partitioned filters. Leaf +/// index/filter partitions of this size fault through the block cache on +/// demand, so index/filter memory is cache-bounded instead of resident-per-SST. +const METADATA_BLOCK_SIZE: usize = 4096; + +/// Filter bits per key (~1% false positives), all levels (~1.25 GB +/// cache-charged at 1e9 keys — see the rationale where the data CF options +/// keep bottommost filters). +const FILTER_BITS_PER_KEY: f64 = 10.0; + +/// Data CF memtable size. Hydrating 270 GB through RocksDB's 64 MiB default +/// means ~4,300 flushes and 64 MB L0 files; 256 MiB quarters the flush count +/// and gives compaction bigger, fewer units of work. Memtable arena blocks +/// allocate lazily, so tiny stores don't pay this up front. +const DATA_WRITE_BUFFER_BYTES: usize = 256 << 20; + +/// Up to 4 memtables (1 active + 3 immutable draining): ingest keeps moving +/// while flushes ride out compaction I/O bursts. Peak memtable RAM 1 GiB — +/// acceptable for a deliberately on-disk fold. The default of 2 stalls writes +/// whenever a single flush falls behind. +const DATA_MAX_WRITE_BUFFERS: i32 = 4; + +/// Meta CF memtable. It holds exactly one key (the cursor), rewritten every +/// `apply`; 8 MiB is generous. It flushes only under WAL pressure (below). +const META_WRITE_BUFFER_BYTES: usize = 8 << 20; + +/// Hard WAL cap. Every `apply` writes the meta CF, so *every* WAL file holds +/// un-flushed meta data and can only be reclaimed by flushing meta — which only +/// happens under this limit's pressure. The auto default (4× total write +/// buffers ≈ 4.1 GiB here) would mean minutes of WAL replay on a crash reopen; +/// 1 GiB bounds replay to seconds and makes the forced meta flush (a ~KB SST) +/// routine. +const MAX_TOTAL_WAL_BYTES: u64 = 1 << 30; + +/// `sync_file_range` writeback smoothing for SST and WAL writes. Without it, +/// hydration accumulates multi-GB of dirty pages that the OS flushes in latency +/// spiking storms. This is a smoothing hint, NOT durability — the NO_SYNC +/// promise in the module docs is unchanged. +const SYNC_SMOOTHING_BYTES: u64 = 1 << 20; + +/// Cap on flush/compaction parallelism. Background-job throughput shows +/// diminishing returns past this; beyond it, compaction competes with the +/// serving path for CPU. +const MAX_COMPACTION_PARALLELISM: usize = 16; + +/// Durability and read-cache configuration for [`RocksDbSnapshot`]. +/// +/// Defaults to NO_SYNC (`sync: false`) — same cache philosophy as the append +/// log's no-fsync-per-checkpoint path. +#[derive(Debug, Clone, Copy)] +pub struct RocksDbConfig { + /// `fsync` the WAL on every [`apply`](SnapshotStore::apply) commit when + /// `true`. When `false` (the default), commits are written to the WAL but not + /// fsync'd (NO_SYNC): faster, survives a process crash via WAL replay, and a + /// tail lost to power loss is rebuilt by resuming the watch from the recovered + /// cursor — the snapshot is a cache. + pub sync: bool, + + /// Block-cache capacity in bytes. RocksDB's own default is 32 MiB — the + /// same starvation problem as fjall's 32 MiB default + /// ([`FjallConfig::cache_size_bytes`](crate::FjallConfig::cache_size_bytes)): + /// a working-set hydration (a prefix range over one service's keys) misses + /// the cache and hits disk, and the miss rate climbs as the fold grows. + /// + /// Index and filter blocks live *inside* this cache (see the module-level + /// Tuning docs), so the value is an honest bound on the store's read + /// memory. Budget at the 1 GiB default against a 1e9-key fold: + /// ~150–175 MB of metadata (pinned top-level index partitions + upper-level + /// filters + hot leaf index partitions), leaving ~850 MB ≈ 53k × 16 KiB + /// data blocks ≈ ~3M resident entries — keys cluster by service prefix, so + /// hot blocks pack hot services densely. Size it at roughly + /// `resident working set + ~200 MB metadata`; `0` falls back to RocksDB's + /// 32 MiB default. + /// + /// Measured with `benches/snapshot_backends.rs` (NVMe ext4, criterion + /// median, default 1 GiB cache): point-gets against a cache-resident + /// working set run ~2 µs; uniform random gets over a 500M-route fold + /// (114 GiB on disk, 245 B/entry, vs 27 GiB RAM — most reads hit disk) + /// run ~0.9 ms mean, per-service prefix scans ~127 ns/entry, hydration + /// 0.19 M entries/s. The cache buys hot-set residency — the µs-vs-ms gap + /// above — so size it to the working set. + pub cache_size_bytes: u64, +} + +impl Default for RocksDbConfig { + fn default() -> Self { + Self { + sync: false, + // 1 GiB: holds index/data blocks for a ~1e6-service working set + // resident, matching the routing registries' default resident cap. + cache_size_bytes: 1024 * 1024 * 1024, + } + } +} + +/// On-disk durable fold backed by RocksDB. See the [module docs](self). +pub struct RocksDbSnapshot { + // Arc so `reader()` handles share the instance: RocksDB serves reads from + // `&DB` concurrently with writes, and `DB` is `Send + Sync`. + db: Arc, + config: RocksDbConfig, + cursor: WatchCursor, +} + +impl RocksDbSnapshot { + /// Open or resume the store at `path` with explicit durability config. + /// + /// `path` is a directory (RocksDB database), created if absent. Returns the + /// persisted resume cursor — [`WatchCursor::none`] when fresh — and the store. + pub fn open(path: &Path, config: RocksDbConfig) -> Result<(WatchCursor, Self), SnapshotError> { + std::fs::create_dir_all(path)?; + + // --- DB-wide: parallelism, WAL bound, writeback smoothing. --- + // Left at their (good) defaults on purpose: `max_open_files = -1` (a few + // thousand table handles are cheap; table-cache misses are not), + // `format_version` 7, `target_file_size_base` 64 MB, `compaction_pri` + // (min-overlapping-ratio), iterator `auto_readahead_size`, and + // `level_compaction_dynamic_level_bytes = true` — the last is + // load-bearing: `optimize_filters_for_hits` below assumes the bottommost + // level holds ~90% of the data, which dynamic leveling guarantees. + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + let cores = std::thread::available_parallelism() + .map(std::num::NonZero::get) + .unwrap_or(4) + .min(MAX_COMPACTION_PARALLELISM); + // `increase_parallelism` sets `max_background_jobs` internally — do not + // also call `set_max_background_jobs`. + db_opts.increase_parallelism(cores as i32); + // Let big L0→L1 compactions (the hydration bottleneck) split into + // parallel subcompactions instead of running single-threaded. + db_opts.set_max_subcompactions((cores / 2).max(1) as u32); + db_opts.set_max_total_wal_size(MAX_TOTAL_WAL_BYTES); + db_opts.set_bytes_per_sync(SYNC_SMOOTHING_BYTES); + db_opts.set_wal_bytes_per_sync(SYNC_SMOOTHING_BYTES); + + // --- Block cache, shared by both CFs so memory accounting is unified. + // HyperClockCache over LRU: reads are lock-free, where LRU's sharded + // mutexes are the known contention point for many concurrent reader + // handles. Entry charge 0 = auto. (Falling back to LRU is a one-line + // change: `Cache::new_lru_cache(capacity)`.) `cache_size_bytes == 0` + // keeps RocksDB's 32 MiB default LRU but still applies every other + // table option. + let cache = if config.cache_size_bytes > 0 { + let capacity = usize::try_from(config.cache_size_bytes).map_err(|_| { + SnapshotError::InvalidFormat(format!( + "cache_size_bytes {} exceeds usize on this platform", + config.cache_size_bytes + )) + })?; + Some(Cache::new_hyper_clock_cache(capacity, 0)) + } else { + None + }; + + // --- Data CF table format: blocks, filters, partitioned index. --- + let mut data_tbl = BlockBasedOptions::default(); + if let Some(cache) = &cache { + data_tbl.set_block_cache(cache); + } + data_tbl.set_block_size(DATA_BLOCK_SIZE); + // Hybrid ribbon: bloom at L0 (files live minutes; ribbon's ~4× build CPU + // on every memtable flush isn't worth it there), ribbon below (same FP + // rate at ~70% of bloom's memory). RocksDB's default is NO filters at + // all — every miss would probe data blocks in every level. + data_tbl.set_hybrid_ribbon_filter(FILTER_BITS_PER_KEY, 1); + data_tbl.set_optimize_filters_for_memory(true); + // Two-level index + partitioned filters: leaf partitions fault through + // the block cache instead of living whole-and-resident per SST. + // `set_partition_filters` is a silent no-op without + // `TwoLevelIndexSearch` — keep these adjacent. + data_tbl.set_index_type(BlockBasedIndexType::TwoLevelIndexSearch); + data_tbl.set_partition_filters(true); + data_tbl.set_metadata_block_size(METADATA_BLOCK_SIZE); + // Index/filter blocks count against (and live in) the cache, so + // `cache_size_bytes` is an honest bound on the store's read memory; pin + // L0 and the top-level index partitions (~25 MB at 1e9 keys) so the hot + // lookup path never faults its roots. + data_tbl.set_cache_index_and_filter_blocks(true); + data_tbl.set_pin_l0_filter_and_index_blocks_in_cache(true); + // Requires `cache_index_and_filter_blocks(true)` (set above). + data_tbl.set_pin_top_level_index_and_filter(true); + + // --- Data CF: compression, hit-optimized filters, memtables. --- + // Only lz4 and zstd are compiled into the binding (build-time trim of + // the default five compression libs); RocksDB's own default is Snappy, + // which would fail at the first flush/compaction — not at open — if + // left unset. lz4 on the write-hot upper levels, zstd where ~90% of the + // bytes settle. + let mut data_opts = Options::default(); + data_opts.set_compression_type(DBCompressionType::Lz4); + data_opts.set_bottommost_compression_type(DBCompressionType::Zstd); + // Bottommost-level filters are kept (no `optimize_filters_for_hits`): + // they are the only in-memory rejection for absent-key lookups, and on + // a tree carrying compaction debt (mid-hydration, post-bulk-load) they + // also reject the overlapping runs a point-get must otherwise probe on + // disk. ~1.25 GB of cache-charged filter mass at 1e9 keys is the + // cheapest read-latency insurance available at that scale. + data_opts.set_write_buffer_size(DATA_WRITE_BUFFER_BYTES); + data_opts.set_max_write_buffer_number(DATA_MAX_WRITE_BUFFERS); + // Must come after every `data_tbl` mutation — the factory snapshots the + // table options. + data_opts.set_block_based_table_factory(&data_tbl); + + // --- Meta CF: one key (the cursor); tiny memtable, shared cache. --- + let mut meta_opts = Options::default(); + meta_opts.set_compression_type(DBCompressionType::Lz4); + meta_opts.set_write_buffer_size(META_WRITE_BUFFER_BYTES); + if let Some(cache) = &cache { + let mut meta_tbl = BlockBasedOptions::default(); + meta_tbl.set_block_cache(cache); + meta_opts.set_block_based_table_factory(&meta_tbl); + } + + let db = DB::open_cf_descriptors( + &db_opts, + path, + [ + ColumnFamilyDescriptor::new(DATA_CF, data_opts), + ColumnFamilyDescriptor::new(META_CF, meta_opts), + ], + ) + .map_err(map_rocksdb)?; + + let cursor = match db + .get_cf(cf(&db, META_CF)?, CURSOR_KEY) + .map_err(map_rocksdb)? + { + Some(raw) => VersionToken::from_raw(&raw) + .map(WatchCursor::from_version) + .ok_or_else(|| { + SnapshotError::InvalidFormat(format!( + "stored cursor is {} bytes, exceeds version token capacity", + raw.len() + )) + })?, + None => WatchCursor::none(), + }; + + Ok(( + cursor.clone(), + Self { + db: Arc::new(db), + config, + cursor, + }, + )) + } + + /// A cheap, concurrent-read-safe handle to the fold's data column family. + /// + /// RocksDB serves readers concurrently with the writer, so a consumer can + /// clone this out *before* handing the fold to + /// [`watch_applied`](crate::watch_applied) (which takes the store by value, + /// `apply` being `&mut self`) and then `get`/`range` the fold from a separate + /// serving task. That is the working-set-serving pattern for a fold too large + /// to hold resident: seed the hot set, serve it from RAM, and `range` the cold + /// tail from the fold on a cache miss — without the serving path ever touching + /// the writer. + pub fn reader(&self) -> RocksDbReader { + RocksDbReader { + db: Arc::clone(&self.db), + } + } + + /// Flush memtables and block until background compaction debt is fully + /// drained. + /// + /// A bulk hydration leaves the tree with pending compactions that inflate + /// cold-read latency until they drain. Call this after hydrating and + /// before latency-sensitive serving begins; steady-state folding does not + /// need it. + pub fn settle(&self) -> Result<(), SnapshotError> { + let mut opts = WaitForCompactOptions::default(); + opts.set_flush(true); + self.db.wait_for_compact(&opts).map_err(map_rocksdb) + } + + /// Import an exported artifact (see [`SnapshotStore::export_to`]) as a new + /// fold at `dest_dir`, returning the embedded resume cursor and the opened + /// store. + /// + /// `dest_dir` must not exist (or be an empty directory). The artifact is + /// fully verified against its manifest — checksums, backend identity — and + /// the staged copy is **opened** (running RocksDB's own recovery) and its + /// cursor compared against the manifest's before anything lands at + /// `dest_dir`; a bad artifact never becomes a fold. A crash mid-import + /// leaves nothing at `dest_dir`; a crash after the final rename leaves a + /// fully valid fold (a retried import then refuses the existing + /// destination — just [`open`](Self::open) it). + /// + /// Unlike the fjall/append-log imports, the manifest's `backend_version` is + /// **not** gated: it records the rust-rocksdb binding version for + /// observability, but RocksDB reads older on-disk formats and its own open + /// is the real arbiter of compatibility. + pub fn import( + artifact_dir: &Path, + dest_dir: &Path, + config: RocksDbConfig, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (manifest, stage) = + verify_and_stage_import(artifact_dir, dest_dir, Self::BACKEND, |_| Ok(()))?; + + // Verify by opening the staged copy — RocksDB's own recovery (WAL + // replay, MANIFEST/CURRENT validation) is the consistency oracle — and + // gate on cursor agreement with the manifest BEFORE the rename. The + // verify handle uses a minimal cache and drains background work before + // dropping so the staged files are final. + { + let (staged_cursor, verify) = Self::open( + &stage.payload(), + RocksDbConfig { + sync: config.sync, + cache_size_bytes: 0, + }, + )?; + verify.db.cancel_all_background_work(true); + if staged_cursor != manifest.cursor { + return Err(SnapshotError::ArtifactInvalid(format!( + "payload cursor {staged_cursor:?} disagrees with manifest cursor {:?}", + manifest.cursor + ))); + } + } + + stage.finalize_dir()?; + Self::open(dest_dir, config) + } +} + +// --- Export internals ------------------------------------------------------- + +impl RocksDbSnapshot { + /// Backend identity in artifact manifests. + pub(crate) const BACKEND: &'static str = "rocksdb"; + /// The rust-rocksdb binding version, recorded in artifact manifests for + /// observability (import does not gate on it — RocksDB reads older on-disk + /// formats and its own open is the arbiter). Bump in lockstep with the + /// `rocksdb` version in `Cargo.toml`. + pub(crate) const BACKEND_VERSION: &'static str = "0.50"; +} + +/// A concurrent read handle over a [`RocksDbSnapshot`]'s data column family, +/// cloned via [`RocksDbSnapshot::reader`]. Reads share the same on-disk fold as +/// the writer and are safe to run concurrently with it. +#[derive(Clone)] +pub struct RocksDbReader { + db: Arc, +} + +impl RocksDbReader { + /// Live entry for `key`, or `None` if absent/deleted. + pub fn get(&self, key: &str) -> Result, SnapshotError> { + get_entry(&self.db, key) + } + + /// Batched point lookups: one RocksDB `MultiGet` instead of N independent + /// `get`s. RocksDB sorts the keys internally and coalesces filter probes + /// and index lookups per SST. + /// + /// Measured (`benches/snapshot_backends.rs`, 100-key uniform random + /// batches against a 500M-route fold, most reads hitting NVMe): 18.5 ms + /// per batch vs 103 ms for a loop of [`get`](Self::get)s — 5.5× — because + /// `MultiGet` overlaps the cold block reads the loop pays sequentially. + /// Against a cache-resident working set the loop is marginally *faster* + /// (per-batch marshaling, nothing left to coalesce). Use this when + /// batches are likely to miss; use the loop when they're hot. + /// + /// Results are positionally aligned with the input; `None` means + /// absent/deleted, exactly as [`get`](Self::get). + pub fn multi_get<'k>( + &self, + keys: impl IntoIterator, + ) -> Result>, SnapshotError> { + let keys: Vec<&str> = keys.into_iter().collect(); + let data = cf(&self.db, DATA_CF)?; + // `sorted_input = false`: callers pass arbitrary route keys; RocksDB + // sorts a copy internally. + let results = self + .db + .batched_multi_get_cf(data, keys.iter().map(|k| k.as_bytes()), false); + keys.iter() + .zip(results) + .map(|(key, res)| match res.map_err(map_rocksdb)? { + // The pinnable slice borrows the block cache; `decode_entry` + // copies out of it, so nothing is held past this closure. + Some(raw) => Ok(Some(decode_entry(key, &raw)?)), + None => Ok(None), + }) + .collect() + } + + /// Stream every live entry whose key starts with `prefix`, ascending, without + /// buffering the whole match set — the memory-bounded scan for an on-disk fold. + pub fn for_each_in_range( + &self, + prefix: &str, + f: impl FnMut(KvEntry) -> Result<(), SnapshotError>, + ) -> Result<(), SnapshotError> { + scan_prefix(&self.db, prefix, f) + } + + /// Buffered counterpart to [`for_each_in_range`](Self::for_each_in_range) for + /// bounded prefixes (e.g. one service's routes, or the whole `node.` map). + pub fn range(&self, prefix: &str) -> Result, SnapshotError> { + let mut out = Vec::new(); + self.for_each_in_range(prefix, |e| { + out.push(e); + Ok(()) + })?; + Ok(out) + } +} + +impl SnapshotStore for RocksDbSnapshot { + fn load(path: &Path) -> Result<(WatchCursor, Self), SnapshotError> { + Self::open(path, RocksDbConfig::default()) + } + + fn apply(&mut self, batch: &[KvUpdate], cursor: &WatchCursor) -> Result<(), SnapshotError> { + let data = cf(&self.db, DATA_CF)?; + let meta = cf(&self.db, META_CF)?; + // One atomic batch: a WriteBatch is a single WAL entry even across column + // families, so every data mutation AND the cursor commit together. Either + // the whole fold step is durable or none of it is — the cursor never + // outraces its data. + let mut wb = WriteBatch::default(); + // One scratch buffer reused across the whole batch. `put_cf` copies the + // bytes into the batch's internal representation before returning, so the + // buffer is free to be refilled for the next entry. That turns N per-`Put` + // assembly allocations into one amortized allocation. + let mut scratch = Vec::new(); + for update in batch { + match update { + KvUpdate::Put(entry) => { + encode_value_into(&mut scratch, &entry.value, &entry.version)?; + wb.put_cf(data, entry.key.as_bytes(), scratch.as_slice()); + } + KvUpdate::Delete { key, .. } | KvUpdate::Purge { key, .. } => { + wb.delete_cf(data, key.as_bytes()); + } + } + } + // Cursor in the SAME batch as the data it names. + wb.put_cf(meta, CURSOR_KEY, cursor.version().as_bytes()); + + // The WAL is always on; `set_sync` only toggles the per-commit fsync. + // NO_SYNC (sync: false) reaches the OS — survives a process crash via WAL + // replay, not a power loss — exactly the cache semantics the module docs + // promise. + let mut wo = WriteOptions::default(); + wo.set_sync(self.config.sync); + self.db.write_opt(&wb, &wo).map_err(map_rocksdb)?; + + self.cursor = cursor.clone(); + Ok(()) + } + + fn get(&self, key: &str) -> Result, SnapshotError> { + get_entry(&self.db, key) + } + + fn range(&self, prefix: &str) -> Result, SnapshotError> { + // Collect the streaming scan — same decode path as `for_each_in_range`, + // just buffered. RocksDB yields keys in ascending byte order, so the + // result is already sorted (unlike the HashMap-backed append log). + let mut out = Vec::new(); + self.for_each_in_range(prefix, |entry| { + out.push(entry); + Ok(()) + })?; + Ok(out) + } + + fn for_each_in_range( + &self, + prefix: &str, + f: impl FnMut(KvEntry) -> Result<(), SnapshotError>, + ) -> Result<(), SnapshotError> { + scan_prefix(&self.db, prefix, f) + } + + fn cursor(&self) -> WatchCursor { + self.cursor.clone() + } + + /// Export via RocksDB's native [`Checkpoint`]: the engine flushes the + /// memtable, hardlinks the immutable SSTs, and copies + /// MANIFEST/CURRENT/OPTIONS + WAL into the stage — consistent by + /// construction, the engine's own blessed snapshot mechanism. The copy is + /// then verified by reopening it (cursor must equal the live fold's — a + /// complete tail-loss detector, since every `apply` writes the cursor in + /// the same batch as its data), hashed **after** the verify handle drops + /// (recovery may rewrite the stage), sealed, and atomically renamed. + fn export_to(&mut self, dest_dir: &Path) -> Result { + let stage = ExportStage::new(dest_dir)?; + + // `create_checkpoint` requires its target to NOT exist — hand it the + // not-yet-created payload path inside the stage. + Checkpoint::new(&self.db) + .and_then(|cp| cp.create_checkpoint(stage.payload())) + .map_err(map_rocksdb)?; + + { + let (staged_cursor, verify) = Self::open( + &stage.payload(), + RocksDbConfig { + sync: true, + cache_size_bytes: 0, + }, + )?; + verify.db.cancel_all_background_work(true); + if staged_cursor != self.cursor { + return Err(SnapshotError::ArtifactInvalid(format!( + "checkpoint recovered cursor {staged_cursor:?}, live fold is at {:?}", + self.cursor + ))); + } + } // verify handle dropped: background work drained, staged files final + + stage.seal_and_finalize(Self::BACKEND, Self::BACKEND_VERSION, &self.cursor) + } +} + +/// Resolve a column family handle, mapping the impossible-after-`open` absence +/// to a backend error rather than a panic. +fn cf<'a>(db: &'a DB, name: &str) -> Result<&'a ColumnFamily, SnapshotError> { + db.cf_handle(name) + .ok_or_else(|| SnapshotError::Backend(format!("missing column family: {name}"))) +} + +/// Point lookup shared by [`RocksDbSnapshot::get`] and [`RocksDbReader::get`]. +fn get_entry(db: &DB, key: &str) -> Result, SnapshotError> { + match db + .get_cf(cf(db, DATA_CF)?, key.as_bytes()) + .map_err(map_rocksdb)? + { + Some(raw) => Ok(Some(decode_entry(key, &raw)?)), + None => Ok(None), + } +} + +/// Streaming prefix scan shared by the snapshot and its reader handles. +/// +/// The iterator is lazy — entries are decoded and handed to `f` one at a time, +/// so a 1B-route consumer building a serving index never holds more than a +/// single `KvEntry` in memory at once. `PrefixRange` sets both iterate bounds +/// from the prefix, so RocksDB terminates the scan internally at the bound +/// (never scanning tombstones past the prefix) and handles the all-`0xFF` +/// successor edge case; an empty prefix is an unbounded full scan. +fn scan_prefix( + db: &DB, + prefix: &str, + mut f: impl FnMut(KvEntry) -> Result<(), SnapshotError>, +) -> Result<(), SnapshotError> { + let mut read_opts = ReadOptions::default(); + read_opts.set_iterate_range(PrefixRange(prefix.as_bytes())); + // An empty prefix is the build-a-serving-index full scan: streaming ~the + // whole fold through the block cache would evict the entire hot set, so + // don't populate it. Bounded prefix scans keep `fill_cache = true` + // deliberately — a per-service hydration *defines* the working set. + if prefix.is_empty() { + read_opts.fill_cache(false); + } + for item in db.iterator_cf_opt(cf(db, DATA_CF)?, read_opts, IteratorMode::Start) { + let (raw_key, raw_val) = item.map_err(map_rocksdb)?; + let key = std::str::from_utf8(&raw_key).map_err(|e| { + SnapshotError::InvalidFormat(format!("non-UTF-8 key in rocksdb store: {e}")) + })?; + f(decode_entry(key, &raw_val)?)?; + } + Ok(()) +} + +/// Map a [`rocksdb::Error`] into the backend-agnostic [`SnapshotError`]. +fn map_rocksdb(e: rocksdb::Error) -> SnapshotError { + match e.kind() { + // Keep I/O failures (disk full, permission denied, …) in the variant + // operators already match across backends. RocksDB statuses are strings — + // there is no real errno to preserve — so wrap the message rather than + // flatten it into the opaque backend bucket. + ErrorKind::IOError => SnapshotError::Io(std::io::Error::other(e.into_string())), + // Everything else keeps RocksDB's own status text. Deliberately NOT + // mapping `Corruption` to `SnapshotError::Corrupted`: that variant's + // `Display` is the append log's hardcoded "CRC mismatch" text, which would + // mask RocksDB's far more detailed corruption status. + _ => SnapshotError::Backend(e.into_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// A persisted cursor blob larger than the version-token capacity must surface + /// as a recoverable `InvalidFormat` at `open`, not a panic or a silently + /// truncated cursor that would resume the watch from the wrong position. + #[test] + fn open_rejects_corrupted_cursor() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + + { + let (_c, store) = + RocksDbSnapshot::open(&path, RocksDbConfig::default()).expect("initial open"); + // Write an 11-byte blob straight into the meta column family under + // the cursor key, bypassing the apply path's bounded encoding. The + // default WriteOptions keep the WAL on, so the write survives the + // drop below via WAL replay. + store + .db + .put_cf( + cf(&store.db, META_CF).expect("meta cf"), + CURSOR_KEY, + [0u8; 11], + ) + .expect("insert oversized cursor"); + } + + match RocksDbSnapshot::open(&path, RocksDbConfig::default()) { + Err(SnapshotError::InvalidFormat(_)) => {} + Err(other) => panic!("expected InvalidFormat, got {other:?}"), + Ok(_) => panic!("expected open to reject the oversized cursor"), + } + } +} From 49efa249da9abbde9535771823ea66e73a871ea8 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:11:49 -0700 Subject: [PATCH 04/12] feat(fjall)!: export via persist+quiesce+copy+verify-reopen; import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fjall has no checkpoint API, so export is built from parts: persist(SyncAll) (journal complete; &mut self means nothing commits after), best-effort quiesce (rotate memtables, drain flushes/compactions, bounded 10s — uses doc(hidden)-but-pub fjall APIs, compile-checked), copy the DB dir (hardlink immutable tables/blobs with copy fallback, byte-copy journal+metadata+lock, retry ×3 on background-GC races), then verify by reopening the copy — fjall's own recovery is the consistency oracle and cursor equality is a complete tail-loss detector since every apply commits cursor+data in one batch. Hash runs only after the verify handle drops. Inherent cursor() replaced by the trait method. Co-Authored-By: Claude Fable 5 --- src/snapshot_fjall.rs | 460 +++++++++++++++++++++++++++++------------- 1 file changed, 317 insertions(+), 143 deletions(-) diff --git a/src/snapshot_fjall.rs b/src/snapshot_fjall.rs index af39e7c..10b1f98 100644 --- a/src/snapshot_fjall.rs +++ b/src/snapshot_fjall.rs @@ -29,13 +29,43 @@ //! [`apply`](SnapshotStore::apply) to a blocking task, and async callers querying //! [`get`](SnapshotStore::get)/[`range`](SnapshotStore::range) should use //! `spawn_blocking` likewise. +//! +//! ## Tuning +//! +//! [`open`](FjallSnapshot::open) applies the same route-scale workload tuning +//! as the RocksDB backend (see `snapshot_rocksdb.rs`'s Tuning docs for the +//! full model: ~1e9 entries, bulk hydration, point-gets that are ~always +//! hits, per-service prefix scans). fjall's defaults are already closer to +//! that workload than RocksDB's — bloom filters on by default (0.01% FP at +//! L0, 10 bits/key deeper), index blocks pinned at L0/L1, index and filter +//! partitioning from L3 down, lz4 from L2 down, journal capped at 512 MiB — +//! so the constants below adjust only the three levers that aren't: +//! worker-thread count (fjall caps at 4 by default), memtable size, and data +//! block size — plus pinning L0+L1 filter blocks. Skipping last-level filter +//! construction (`expect_point_read_hits`, fjall's twin of RocksDB's +//! `optimize_filters_for_hits`) was tried and rejected: on a tree carrying +//! compaction debt it multiplied cold point-get cost (~10 ms at 500M routes), +//! and it makes every absent-key lookup a guaranteed disk probe. Everything +//! else is deliberately left at fjall's defaults. +//! +//! Measured at 500M routes (NVMe, `benches/snapshot_backends.rs`): tuned +//! hydration runs 0.42 M entries/s — 2.2× the RocksDB backend — at +//! 226 B/entry on disk. The trade is cold uniform point-gets once the fold +//! dwarfs RAM: multi-millisecond here vs sub-millisecond on the RocksDB +//! backend, whose partitioned, cache-pinned metadata bounds a cold get to +//! fewer disk reads. Write-heavy or hot-set-served folds favor fjall; +//! uniform cold-read folds favor RocksDB. -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; -use fjall::{Config, Database, Keyspace, KeyspaceCreateOptions, PersistMode}; +use fjall::config::{BlockSizePolicy, PinningPolicy}; +use fjall::{Database, Keyspace, KeyspaceCreateOptions, PersistMode}; +use crate::artifact::{ExportManifest, ExportStage, verify_and_stage_import}; use crate::kv::{KvEntry, KvUpdate, VersionToken, WatchCursor}; use crate::snapshot::{SnapshotError, SnapshotStore}; +use crate::snapshot_record::{decode_entry, encode_value_into}; /// Partition holding the folded KV state: `key` → encoded `(version, value)`. const DATA_PARTITION: &str = "data"; @@ -44,6 +74,30 @@ const META_PARTITION: &str = "meta"; /// Key under [`META_PARTITION`] storing the resume cursor's raw version bytes. const CURSOR_KEY: &[u8] = b"cursor"; +// --- Tuned constants (see the module-level `## Tuning` docs). --- + +/// Flush/compaction worker threads. fjall's default is `min(cores, 4)`, +/// which starves a multi-GB hydration on a many-core box; this matches the +/// RocksDB backend's parallelism (also capped at 16 — diminishing returns, +/// and beyond it compaction competes with the serving path for CPU). +const MAX_WORKER_THREADS: usize = 16; + +/// Data-partition memtable. fjall's 64 MiB default means 4× the flush (and +/// L0 compaction) count of a 256 MiB buffer during a route-scale hydration; +/// matches the RocksDB backend's write buffer. Memtables fill lazily, so +/// small stores don't pay this up front. +const DATA_MEMTABLE_BYTES: u64 = 256 << 20; + +/// Meta partition holds exactly one key (the cursor), rewritten every +/// `apply`; 8 MiB is generous (parity with the RocksDB meta CF). +const META_MEMTABLE_BYTES: u64 = 8 << 20; + +/// Data block size. Same math as the RocksDB backend: 4 KiB blocks at a +/// 1e9-key fold produce multi-GB block indexes; 16 KiB quarters that and +/// gives compression more context, at the cost of decompressing 16 KiB +/// instead of 4 KiB on a cache-miss point read. +const DATA_BLOCK_SIZE: u32 = 16 * 1024; + /// Durability and read-cache configuration for [`FjallSnapshot`]. /// /// Defaults to NO_SYNC (`sync: false`) — same cache philosophy as the append @@ -86,6 +140,9 @@ pub struct FjallSnapshot { meta: Keyspace, config: FjallConfig, cursor: WatchCursor, + // fjall's `Config.path` is `pub(crate)`, so the export path keeps its own + // copy of the DB directory for the artifact copy. + path: PathBuf, } impl FjallSnapshot { @@ -95,20 +152,39 @@ impl FjallSnapshot { /// persisted resume cursor — [`WatchCursor::none`] when fresh — and the store. pub fn open(path: &Path, config: FjallConfig) -> Result<(WatchCursor, Self), SnapshotError> { std::fs::create_dir_all(path)?; - let mut db_config = Config::new(path); + let workers = std::thread::available_parallelism() + .map(std::num::NonZero::get) + .unwrap_or(4) + .min(MAX_WORKER_THREADS); + let mut builder = Database::builder(path).worker_threads(workers); // Size the LSM block cache to the working set (default 1 GiB). fjall's own - // default is 32 MiB, far too small for the fold — see `FjallConfig::cache_size_bytes`. + // default is 32 MiB, far too small for the fold — see + // `FjallConfig::cache_size_bytes`. `0` keeps fjall's default. if config.cache_size_bytes > 0 { - db_config.cache = std::sync::Arc::new(lsm_tree::Cache::with_capacity_bytes( - config.cache_size_bytes, - )); + builder = builder.cache_size(config.cache_size_bytes); } - let db = Database::open(db_config).map_err(map_fjall)?; + let db: Database = builder.open().map_err(map_fjall)?; let data = db - .keyspace(DATA_PARTITION, KeyspaceCreateOptions::default) + .keyspace(DATA_PARTITION, || { + KeyspaceCreateOptions::default() + .max_memtable_size(DATA_MEMTABLE_BYTES) + .data_block_size_policy(BlockSizePolicy::all(DATA_BLOCK_SIZE)) + // Last-level filters are kept (no `expect_point_read_hits`): + // they are the only in-memory rejection for absent-key + // lookups, and on a tree carrying compaction debt they + // reject the overlapping runs a point-get must otherwise + // probe on disk (measured: skipping them cost ~10 ms cold + // gets on an unsettled 500M fold). + // + // Pin L0+L1 filters so the hot lookup path never faults its + // filter roots (fjall's default pins L0 only). + .filter_block_pinning_policy(PinningPolicy::new([true, true, false])) + }) .map_err(map_fjall)?; let meta = db - .keyspace(META_PARTITION, KeyspaceCreateOptions::default) + .keyspace(META_PARTITION, || { + KeyspaceCreateOptions::default().max_memtable_size(META_MEMTABLE_BYTES) + }) .map_err(map_fjall)?; let cursor = match meta.get(CURSOR_KEY).map_err(map_fjall)? { @@ -131,15 +207,11 @@ impl FjallSnapshot { meta, config, cursor, + path: path.to_path_buf(), }, )) } - /// The most recently applied resume cursor. - pub fn cursor(&self) -> &WatchCursor { - &self.cursor - } - /// A cheap, concurrent-read-safe handle to the fold's data partition. /// /// fjall serves readers concurrently with the writer, so a consumer can clone @@ -154,6 +226,166 @@ impl FjallSnapshot { data: self.data.clone(), } } + + /// Force a major compaction of the data partition, blocking until done. + /// + /// fjall's background compaction is write-driven: after a bulk hydration + /// stops, residual overlapping runs can persist indefinitely and inflate + /// cold-read latency (every unrejected run costs an extra disk probe). + /// Call this after hydrating and before latency-sensitive serving begins; + /// steady-state folding does not need it. + pub fn settle(&self) -> Result<(), SnapshotError> { + self.data.major_compact().map_err(map_fjall) + } + + /// Import an exported artifact (see [`SnapshotStore::export_to`]) as a new + /// fold at `dest_dir`, returning the embedded resume cursor and the opened + /// store. + /// + /// `dest_dir` must not exist (or be an empty directory). The artifact is + /// fully verified against its manifest — checksums, backend identity, + /// on-disk format generation — and the staged copy is **opened** (running + /// fjall's own recovery) and its cursor compared against the manifest's + /// before anything lands at `dest_dir`; a bad artifact never becomes a + /// fold. A crash mid-import leaves nothing at `dest_dir`; a crash after + /// the final rename leaves a fully valid fold (a retried import then + /// refuses the existing destination — just [`open`](Self::open) it). + pub fn import( + artifact_dir: &Path, + dest_dir: &Path, + config: FjallConfig, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (manifest, stage) = + verify_and_stage_import(artifact_dir, dest_dir, Self::BACKEND, |v| { + if v == Self::BACKEND_VERSION { + Ok(()) + } else { + Err(SnapshotError::ArtifactInvalid(format!( + "fjall artifact has on-disk format generation {v:?}, this build reads {:?}", + Self::BACKEND_VERSION + ))) + } + })?; + + // Verify by opening the staged copy — fjall's own recovery (journal + // CRCs, version checksums) is the consistency oracle — and gate on + // cursor agreement with the manifest BEFORE the rename. The verify + // handle uses a minimal cache; it is dropped (joining fjall's worker + // threads) before the rename. + { + let (staged_cursor, _verify) = Self::open( + &stage.payload(), + FjallConfig { + sync: true, + cache_size_bytes: 0, + }, + )?; + if staged_cursor != manifest.cursor { + return Err(SnapshotError::ArtifactInvalid(format!( + "payload cursor {staged_cursor:?} disagrees with manifest cursor {:?}", + manifest.cursor + ))); + } + } + + stage.finalize_dir()?; + Self::open(dest_dir, config) + } +} + +// --- Export internals ------------------------------------------------------- + +impl FjallSnapshot { + /// Backend identity in artifact manifests. + pub(crate) const BACKEND: &'static str = "fjall"; + /// On-disk format generation in artifact manifests: fjall's V3 format + /// marker, which fjall itself re-enforces at open (`check_version` rejects + /// anything but V3). + pub(crate) const BACKEND_VERSION: &'static str = "3"; + + /// Bound on the best-effort flush/compaction drain before the copy. + const QUIESCE_TIMEOUT: Duration = Duration::from_secs(10); + + /// Best-effort quiesce before the artifact copy: rotate memtables into SSTs + /// and wait (bounded) for background flushes/compactions to drain, so the + /// copy is dominated by immutable, hardlinkable table files. + /// + /// Correctness never depends on this — `export_to` takes `&mut self` (no + /// concurrent commits), the copy retries on files deleted under it, and the + /// verify-by-reopen + cursor-equality gate catches anything torn. The + /// quiesce only makes the copy converge fast. + /// + /// `rotate_memtable_and_wait` / `outstanding_flushes` / `active_compactions` + /// are `pub` but `#[doc(hidden)]` in fjall 3.1.4 ("used in tests" / + /// "experimental"; verified in fjall source: `keyspace/mod.rs:708`, + /// `db.rs:220`, `db.rs:247`). They are compile-checked: a fjall upgrade + /// that removes them fails the build loudly — in that case delete this + /// method; persist + retry + verify remains correct, just retry-prone + /// under churn. + fn quiesce(&self) { + for ks in [&self.data, &self.meta] { + if let Err(e) = ks.rotate_memtable_and_wait() { + tracing::warn!(error = %e, "fjall export quiesce: memtable rotation failed; proceeding"); + return; + } + } + let deadline = Instant::now() + Self::QUIESCE_TIMEOUT; + while (self.db.outstanding_flushes() > 0 || self.db.active_compactions() > 0) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(50)); + } + } +} + +/// `true` for paths inside fjall's immutable-file directories (`tables/`, +/// `blobs/`): created with `create_new`, never mutated, only unlinked — safe to +/// hardlink into an artifact. Everything else (journal, version markers, lock) +/// is byte-copied: the source keeps appending to its journal after export +/// returns, and a hardlinked journal would mutate the artifact under its +/// recorded checksums. +fn is_immutable_payload(rel: &Path) -> bool { + use std::path::Component; + rel.components() + .any(|c| matches!(c, Component::Normal(n) if n == "tables" || n == "blobs")) +} + +/// Copy a fjall DB directory into `dst`: hardlink immutable table/blob files +/// (copy-fallback on any error, e.g. EXDEV), byte-copy everything else. +/// Everything is included — notably the `lock` file, which fjall's recovery +/// `File::open`s and therefore must exist (its lock state is advisory, not in +/// the content). +fn copy_db_dir(src: &Path, dst: &Path) -> Result<(), SnapshotError> { + std::fs::create_dir_all(dst)?; + let mut stack = vec![src.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let ty = entry.file_type()?; + let rel = entry + .path() + .strip_prefix(src) + .map_err(|_| SnapshotError::Backend("fjall copy escaped the DB root".into()))? + .to_path_buf(); + let to = dst.join(&rel); + if ty.is_dir() { + std::fs::create_dir_all(&to)?; + stack.push(entry.path()); + } else if ty.is_file() { + if is_immutable_payload(&rel) { + if std::fs::hard_link(entry.path(), &to).is_err() { + // EXDEV (stage on another filesystem) or anything else: + // fall back to a byte-copy. Correct either way; the + // hardlink is only the cheap path. + std::fs::copy(entry.path(), &to)?; + } + } else { + std::fs::copy(entry.path(), &to)?; + } + } + } + } + Ok(()) } /// A concurrent read handle over a [`FjallSnapshot`]'s data partition, cloned via @@ -275,6 +507,76 @@ impl SnapshotStore for FjallSnapshot { } Ok(()) } + + fn cursor(&self) -> WatchCursor { + self.cursor.clone() + } + + /// Export sequence (fjall has **no checkpoint API**, so this is built from + /// parts; correctness rests on `&mut self` exclusivity + verify-by-reopen, + /// never on the quiesce): + /// + /// 1. `persist(SyncAll)` — the journal is complete and durable. With + /// `&mut self` nothing commits after this. + /// 2. Best-effort quiesce (memtables → SSTs, drain flush/compaction, + /// bounded) so the copy is dominated by immutable hardlinkable files. + /// 3. Copy the DB dir into the stage: hardlink `tables/`/`blobs/`, + /// byte-copy journal + metadata. Background GC can still delete a file + /// between enumerate and link — retried, bounded. + /// 4. **Verify by reopening the copy**: fjall's own recovery is the + /// consistency oracle, and the recovered cursor must equal the live + /// cursor. Because every `apply` writes the cursor in the same batch as + /// its data, cursor equality is a complete tail-loss detector. The + /// verify handle is dropped (joining fjall's workers) before hashing. + /// 5. Hash the staged files **after** the verify-open (recovery may + /// legitimately rewrite the stage), write the manifest, fsync, rename. + fn export_to(&mut self, dest_dir: &Path) -> Result { + let stage = ExportStage::new(dest_dir)?; + let payload = stage.payload(); + + self.db + .persist(PersistMode::SyncAll) + .map_err(map_fjall)?; + self.quiesce(); + + let mut attempt = 0; + loop { + attempt += 1; + match copy_db_dir(&self.path, &payload) { + Ok(()) => break, + Err(SnapshotError::Io(e)) + if e.kind() == std::io::ErrorKind::NotFound && attempt < 3 => + { + // A straggler flush/compaction GC'd a file under the copy. + // Clear and re-copy from the now-quieter tree. + tracing::warn!( + attempt, + "fjall export copy raced background GC; retrying" + ); + std::fs::remove_dir_all(&payload)?; + } + Err(e) => return Err(e), + } + } + + { + let (staged_cursor, _verify) = Self::open( + &payload, + FjallConfig { + sync: true, + cache_size_bytes: 0, + }, + )?; + if staged_cursor != self.cursor { + return Err(SnapshotError::ArtifactInvalid(format!( + "exported copy recovered cursor {staged_cursor:?}, live fold is at {:?}", + self.cursor + ))); + } + } // verify handle dropped: fjall workers joined, staged files final + + stage.seal_and_finalize(Self::BACKEND, Self::BACKEND_VERSION, &self.cursor) + } } impl FjallSnapshot { @@ -294,63 +596,6 @@ impl FjallSnapshot { } } -/// Encode a stored value as `[ver_len:u8][version bytes][value bytes]` into `buf`. -/// -/// `buf` is cleared and refilled (its capacity is reused across a batch). The -/// version is length-prefixed raw bytes for the same reason the append-log format -/// uses it: a backend's token (NATS u64, FDB 10-byte versionstamp) must round-trip -/// intact. -/// -/// `VersionToken` caps inline storage at 10 bytes, so the `u8` length prefix never -/// truncates today. Checking with `try_from` rather than casting surfaces a format -/// error instead of silently writing a wrong length — which would frame a record -/// `decode_entry` then mis-parses — if a future token ever widens past 255 bytes. -/// This mirrors `write_put_record` in `snapshot.rs`. -fn encode_value_into( - buf: &mut Vec, - value: &[u8], - version: &VersionToken, -) -> Result<(), SnapshotError> { - let vb = version.as_bytes(); - let ver_len = u8::try_from(vb.len()).map_err(|_| { - SnapshotError::InvalidFormat(format!( - "version too long: {} bytes (max {})", - vb.len(), - u8::MAX - )) - })?; - buf.clear(); - buf.reserve(1 + vb.len() + value.len()); - buf.push(ver_len); - buf.extend_from_slice(vb); - buf.extend_from_slice(value); - Ok(()) -} - -/// Decode a `[ver_len:u8][version][value]` record back into a [`KvEntry`]. -fn decode_entry(key: &str, raw: &[u8]) -> Result { - let ver_len = *raw.first().ok_or_else(|| { - SnapshotError::InvalidFormat("fjall value record is empty (no version length)".into()) - })? as usize; - let value_off = 1 + ver_len; - if raw.len() < value_off { - return Err(SnapshotError::InvalidFormat(format!( - "fjall value record truncated: need {value_off} bytes for version, have {}", - raw.len() - ))); - } - let version = VersionToken::from_raw(&raw[1..value_off]).ok_or_else(|| { - SnapshotError::InvalidFormat(format!( - "version length {ver_len} exceeds version token capacity" - )) - })?; - Ok(KvEntry { - key: key.to_string(), - value: raw[value_off..].to_vec(), - version, - }) -} - /// Map a [`fjall::Error`] into the backend-agnostic [`SnapshotError`]. fn map_fjall(e: fjall::Error) -> SnapshotError { match e { @@ -371,77 +616,6 @@ mod tests { use super::*; use tempfile::TempDir; - /// A 10-byte FDB versionstamp has no `u64` form; the length-prefixed value - /// format must carry it intact. A `u64`-only field would flatten it to 0 and - /// silently break every later CAS — so this is the load-bearing reason the - /// record stores a length-prefixed token rather than a fixed 8 bytes. - #[test] - fn encode_decode_round_trips_fdb_versionstamp() { - let vs = VersionToken::from_fdb_versionstamp(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - let mut enc = Vec::new(); - encode_value_into(&mut enc, b"payload", &vs).expect("encode"); - let entry = decode_entry("k", &enc).expect("decode"); - - assert_eq!(entry.version.as_bytes(), &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - assert!( - entry.version.as_u64().is_none(), - "a 10-byte token has no u64 form — it must not be flattened" - ); - assert_eq!(entry.value, b"payload"); - } - - /// An empty value (the CAS-tombstone shape) encodes to just the version prefix - /// and decodes back to a present, empty-valued entry with its version intact. - #[test] - fn encode_decode_round_trips_empty_value() { - let mut enc = Vec::new(); - encode_value_into(&mut enc, b"", &VersionToken::from_u64(7)).expect("encode"); - let entry = decode_entry("k", &enc).expect("decode"); - - assert!(entry.value.is_empty()); - assert_eq!(entry.version.as_u64(), Some(7)); - } - - /// A zero-byte record has no version-length byte — corruption, not a valid - /// record. It must surface as a recoverable `InvalidFormat`, never a panic. - #[test] - fn decode_entry_rejects_empty_record() { - let err = decode_entry("k", &[]).unwrap_err(); - assert!( - matches!(err, SnapshotError::InvalidFormat(_)), - "empty record must be a format error, got {err:?}" - ); - } - - /// A record that claims a longer version than its bytes provide is truncated - /// on-disk corruption — reject it instead of reading past the buffer. - #[test] - fn decode_entry_rejects_truncated_version() { - // Claims a 5-byte version, but only 2 bytes follow the length prefix. - let raw = [5u8, 0xAA, 0xBB]; - let err = decode_entry("k", &raw).unwrap_err(); - assert!( - matches!(err, SnapshotError::InvalidFormat(_)), - "truncated version must be a format error, got {err:?}" - ); - } - - /// A version length beyond `VersionToken`'s 10-byte capacity can't round-trip; - /// `from_raw` rejects it and `decode_entry` maps that to `InvalidFormat` rather - /// than silently truncating to a wrong (CAS-breaking) version. - #[test] - fn decode_entry_rejects_oversized_version() { - // ver_len = 11 with 11 trailing bytes: passes the truncation check, then - // trips the capacity check inside `VersionToken::from_raw`. - let mut raw = vec![11u8]; - raw.extend_from_slice(&[0u8; 11]); - let err = decode_entry("k", &raw).unwrap_err(); - assert!( - matches!(err, SnapshotError::InvalidFormat(_)), - "oversized version must be a format error, got {err:?}" - ); - } - /// A persisted cursor blob larger than the version-token capacity must surface /// as a recoverable `InvalidFormat` at `open`, not a panic or a silently /// truncated cursor that would resume the watch from the wrong position. From 92b10bca73e4cccee7b4bc5eb885ad2c3eb26013 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:11:49 -0700 Subject: [PATCH 05/12] test(conformance): export/import suite, instantiated for all three backends Round-trip (cursor + state equality), import-then-tail-replay equals the continuous fold, tampered/missing/undeclared payload rejection with nothing landing at the destination, wrong backend / schema / format-generation / doctored-cursor rejection, non-empty-destination refusal (artifact survives), empty-fold round-trip. rocksdb deliberately skips the format-generation gate. Co-Authored-By: Claude Fable 5 --- tests/snapshot_store.rs | 697 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 694 insertions(+), 3 deletions(-) diff --git a/tests/snapshot_store.rs b/tests/snapshot_store.rs index bcd5c78..76a4569 100644 --- a/tests/snapshot_store.rs +++ b/tests/snapshot_store.rs @@ -2,13 +2,15 @@ //! //! Every check is written generically over a backend and an `open` closure, then //! instantiated for each shipped backend: [`AppendLogSnapshot`] (always) and -//! `FjallSnapshot` (behind `--features fjall`). New backends get the whole suite -//! by adding two wrapper lines. +//! `FjallSnapshot` (behind `--features fjall`), and `RocksDbSnapshot` (behind +//! `--features rocksdb`). New backends get the whole suite by adding two +//! wrapper lines. //! //! Run the full matrix with: //! ```text //! cargo test --test snapshot_store //! cargo test --test snapshot_store --features fjall +//! cargo test --test snapshot_store --features rocksdb //! ``` use std::path::Path; @@ -360,7 +362,7 @@ fn check_empty_value_round_trip(open: impl Fn(&Path) -> (Watch } /// Remove a store at `path`, whether it is a single file (append log) or a -/// directory (fjall keyspace). +/// directory (fjall keyspace, RocksDB database). fn wipe(path: &Path) { if path.is_dir() { std::fs::remove_dir_all(path).expect("rm store dir"); @@ -369,6 +371,291 @@ fn wipe(path: &Path) { } } +// --------------------------------------------------------------------------- +// Export / import — generic checks +// --------------------------------------------------------------------------- + +use slipstream::snapshot::SnapshotError; +use slipstream::{ExportManifest, MANIFEST_FILE}; + +type ImportFn = fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>; + +/// Export a fold of [`stream`] and return `(artifact_dir, manifest, tempdir)`. +fn exported_stream_artifact( + open: &impl Fn(&Path) -> (WatchCursor, S), +) -> (PathBuf, ExportManifest, TempDir) { + let dir = TempDir::new().unwrap(); + let store_path = dir.path().join("store"); + let artifact = dir.path().join("artifact"); + let (_r, mut s) = open(&store_path); + fold(&mut s, &stream()); + let manifest = s.export_to(&artifact).expect("export"); + assert_eq!( + manifest.cursor, + s.cursor(), + "manifest cursor equals the live fold's cursor" + ); + (artifact, manifest, dir) +} + +use std::path::PathBuf; + +/// Round-trip: export a fold, import it elsewhere, and the imported store has +/// the same cursor and byte-identical state. +fn check_export_import_round_trip( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, manifest, dir) = exported_stream_artifact(&open); + assert_eq!(manifest.cursor.as_u64(), Some(7)); + assert!(!manifest.files.is_empty(), "manifest lists payload files"); + assert!(artifact.join(MANIFEST_FILE).is_file()); + + let dest = dir.path().join("imported"); + let (cursor, s) = import(&artifact, &dest).expect("import"); + assert_eq!(cursor.as_u64(), Some(7), "imported cursor is the manifest's"); + assert_eq!(cursor, manifest.cursor); + assert_eq!(dump(&s), expected_state(), "imported state is identical"); +} + +/// The scaling property behind import: fold the post-cursor delta into the +/// imported store and it converges to the continuous-fold reference — import + +/// tail replay, never a full re-fold. +fn check_import_resume_continues_fold( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let updates = stream(); + let dir = TempDir::new().unwrap(); + + // Export mid-stream, after revs 1..=4. + let store_path = dir.path().join("store"); + let artifact = dir.path().join("artifact"); + let (_r, mut s) = open(&store_path); + fold(&mut s, &updates[..4]); + let manifest = s.export_to(&artifact).expect("export"); + assert_eq!(manifest.cursor.as_u64(), Some(4)); + + // Import on "another node" and fold ONLY the tail (revs 5..=7). + let dest = dir.path().join("imported"); + let (cursor, mut imported) = import(&artifact, &dest).expect("import"); + assert_eq!(cursor.as_u64(), Some(4)); + fold(&mut imported, &updates[4..]); + + assert_eq!( + dump(&imported), + expected_state(), + "import + tail replay equals the continuous fold" + ); +} + +/// A flipped byte in the payload fails checksum verification, and nothing is +/// ever created at the destination (stage-then-rename crash safety). +fn check_import_rejects_tampered_payload( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, manifest, dir) = exported_stream_artifact(&open); + + // Tamper with the largest payload file (most likely to hold real data). + let victim = manifest + .files + .iter() + .max_by_key(|f| f.size) + .expect("at least one payload file"); + let victim_path = artifact.join(&victim.path); + let mut bytes = std::fs::read(&victim_path).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 0xFF; + std::fs::write(&victim_path, &bytes).unwrap(); + + let dest = dir.path().join("imported"); + match import(&artifact, &dest) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!( + msg.contains("checksum") || msg.contains("recover") || msg.contains("cursor"), + "rejection names the failure: {msg}" + ); + } + Err(other) => panic!("expected ArtifactInvalid, got {other:?}"), + Ok(_) => panic!("tampered artifact must not import"), + } + assert!(!dest.exists(), "nothing lands at the destination"); +} + +/// A payload file missing from the artifact is rejected. +fn check_import_rejects_missing_payload_file( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, manifest, dir) = exported_stream_artifact(&open); + std::fs::remove_file(artifact.join(&manifest.files[0].path)).unwrap(); + + let dest = dir.path().join("imported"); + assert!( + matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + ), + "missing payload file must be rejected" + ); + assert!(!dest.exists()); +} + +/// A payload file the manifest never declared is rejected (it was never hashed +/// at export, so it cannot be trusted). +fn check_import_rejects_undeclared_extra_file( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _manifest, dir) = exported_stream_artifact(&open); + std::fs::write(artifact.join("data").join("smuggled"), b"x").unwrap(); + + let dest = dir.path().join("imported"); + assert!( + matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + ), + "undeclared extra payload file must be rejected" + ); + assert!(!dest.exists()); +} + +/// Rewrite one top-level manifest field (checksums untouched) and re-import. +fn with_doctored_manifest(artifact: &Path, field: &str, value: serde_json::Value) { + let raw = std::fs::read(artifact.join(MANIFEST_FILE)).unwrap(); + let mut json: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + json[field] = value; + std::fs::write( + artifact.join(MANIFEST_FILE), + serde_json::to_vec(&json).unwrap(), + ) + .unwrap(); +} + +/// Wrong backend identity in the manifest is rejected. +fn check_import_rejects_wrong_backend( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _m, dir) = exported_stream_artifact(&open); + with_doctored_manifest(&artifact, "backend", "bogus-backend".into()); + let dest = dir.path().join("imported"); + assert!(matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + )); + assert!(!dest.exists()); +} + +/// Unsupported artifact schema version is rejected. +fn check_import_rejects_schema_version( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _m, dir) = exported_stream_artifact(&open); + with_doctored_manifest(&artifact, "schema_version", 999.into()); + let dest = dir.path().join("imported"); + assert!(matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + )); + assert!(!dest.exists()); +} + +/// Mismatched on-disk format generation is rejected (strict backends only: +/// append-log and fjall gate on it; RocksDB deliberately does not). +fn check_import_rejects_backend_version( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _m, dir) = exported_stream_artifact(&open); + with_doctored_manifest(&artifact, "backend_version", "999".into()); + let dest = dir.path().join("imported"); + assert!(matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + )); + assert!(!dest.exists()); +} + +/// A doctored cursor (valid hex, valid checksums) is caught by the post-open +/// cursor-equality gate — the payload's recovered cursor is the authority. +fn check_import_rejects_cursor_mismatch( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _m, dir) = exported_stream_artifact(&open); + // rev 999 as 8-byte big-endian hex — well-formed, just wrong. + with_doctored_manifest(&artifact, "cursor_hex", "00000000000003e7".into()); + let dest = dir.path().join("imported"); + match import(&artifact, &dest) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("cursor"), "rejection names the cursor: {msg}"); + } + Err(other) => panic!("expected ArtifactInvalid, got {other:?}"), + Ok(_) => panic!("cursor mismatch must not import"), + } + assert!(!dest.exists()); +} + +/// Import refuses a non-empty destination, and the artifact survives untouched +/// (a retry against a fresh destination succeeds). +fn check_import_rejects_nonempty_dest( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _m, dir) = exported_stream_artifact(&open); + + let dest = dir.path().join("occupied"); + std::fs::create_dir(&dest).unwrap(); + std::fs::write(dest.join("stray"), b"x").unwrap(); + assert!(matches!( + import(&artifact, &dest), + Err(SnapshotError::ArtifactInvalid(_)) + )); + assert!(dest.join("stray").exists(), "occupied dest untouched"); + + let fresh = dir.path().join("fresh"); + let (cursor, s) = import(&artifact, &fresh).expect("artifact survives a refused import"); + assert_eq!(cursor.as_u64(), Some(7)); + assert_eq!(dump(&s), expected_state()); +} + +/// Export refuses a non-empty destination. +fn check_export_rejects_nonempty_dest(open: impl Fn(&Path) -> (WatchCursor, S)) { + let dir = TempDir::new().unwrap(); + let (_r, mut s) = open(&dir.path().join("store")); + fold(&mut s, &stream()); + + let dest = dir.path().join("occupied"); + std::fs::create_dir(&dest).unwrap(); + std::fs::write(dest.join("stray"), b"x").unwrap(); + assert!(matches!( + s.export_to(&dest), + Err(SnapshotError::ArtifactInvalid(_)) + )); + assert!(dest.join("stray").exists(), "occupied dest untouched"); +} + +/// An empty fold (nothing applied, no cursor) exports and imports cleanly. +fn check_export_empty_store( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let dir = TempDir::new().unwrap(); + let (_r, mut s) = open(&dir.path().join("store")); + let artifact = dir.path().join("artifact"); + let manifest = s.export_to(&artifact).expect("export empty fold"); + assert!(manifest.cursor.is_none(), "empty fold has no cursor"); + + let dest = dir.path().join("imported"); + let (cursor, imported) = import(&artifact, &dest).expect("import empty fold"); + assert!(cursor.is_none()); + assert!(dump(&imported).is_empty()); +} + // --------------------------------------------------------------------------- // AppendLogSnapshot — the default backend // --------------------------------------------------------------------------- @@ -419,6 +706,70 @@ fn append_log_empty_value_round_trip() { check_empty_value_round_trip(open_append_log); } +fn import_append_log(artifact: &Path, dest: &Path) -> Result<(WatchCursor, AppendLogSnapshot), SnapshotError> { + AppendLogSnapshot::import(artifact, dest, u64::MAX) +} + +#[test] +fn append_log_export_import_round_trip() { + check_export_import_round_trip(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_resume_continues_fold() { + check_import_resume_continues_fold(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_tampered_payload() { + check_import_rejects_tampered_payload(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_missing_payload_file() { + check_import_rejects_missing_payload_file(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_undeclared_extra_file() { + check_import_rejects_undeclared_extra_file(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_wrong_backend() { + check_import_rejects_wrong_backend(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_schema_version() { + check_import_rejects_schema_version(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_backend_version() { + check_import_rejects_backend_version(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_cursor_mismatch() { + check_import_rejects_cursor_mismatch(open_append_log, import_append_log); +} + +#[test] +fn append_log_import_rejects_nonempty_dest() { + check_import_rejects_nonempty_dest(open_append_log, import_append_log); +} + +#[test] +fn append_log_export_rejects_nonempty_dest() { + check_export_rejects_nonempty_dest(open_append_log); +} + +#[test] +fn append_log_export_empty_store() { + check_export_empty_store(open_append_log, import_append_log); +} + /// Backwards-compat: a file written by the existing [`SnapshotWriter`] API loads /// through the new [`AppendLogSnapshot`] store (the on-disk v2 format is unchanged). #[test] @@ -500,6 +851,91 @@ mod fjall_backend { check_empty_value_round_trip(open_no_sync); } + fn import_fjall( + artifact: &Path, + dest: &Path, + ) -> Result<(WatchCursor, FjallSnapshot), SnapshotError> { + FjallSnapshot::import( + artifact, + dest, + FjallConfig { + sync: false, + ..Default::default() + }, + ) + } + + #[test] + fn fjall_export_import_round_trip() { + check_export_import_round_trip(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_resume_continues_fold() { + check_import_resume_continues_fold(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_tampered_payload() { + check_import_rejects_tampered_payload(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_missing_payload_file() { + check_import_rejects_missing_payload_file(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_undeclared_extra_file() { + check_import_rejects_undeclared_extra_file(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_wrong_backend() { + check_import_rejects_wrong_backend(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_schema_version() { + check_import_rejects_schema_version(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_backend_version() { + check_import_rejects_backend_version(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_cursor_mismatch() { + check_import_rejects_cursor_mismatch(open_no_sync, import_fjall); + } + + #[test] + fn fjall_import_rejects_nonempty_dest() { + check_import_rejects_nonempty_dest(open_no_sync, import_fjall); + } + + #[test] + fn fjall_export_rejects_nonempty_dest() { + check_export_rejects_nonempty_dest(open_no_sync); + } + + #[test] + fn fjall_export_empty_store() { + check_export_empty_store(open_no_sync, import_fjall); + } + + /// `settle` (major compaction) must preserve the fold byte-for-byte. + #[test] + fn fjall_settle_preserves_state() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + let (_r, mut s) = open_no_sync(&path); + fold(&mut s, &stream()); + s.settle().expect("settle"); + assert_eq!(dump(&s), expected_state(), "state survives settle"); + } + /// NO_SYNC crash-tail recovery. With sync off, commits are not fsync'd, but /// data and cursor share one atomic batch, so whatever survives is mutually /// consistent. We can't deterministically simulate a power-loss (a clean drop @@ -566,3 +1002,258 @@ mod fjall_backend { assert_eq!(dump(&s), expected_state()); } } + +// --------------------------------------------------------------------------- +// RocksDbSnapshot — on-disk backend (feature-gated) +// --------------------------------------------------------------------------- + +#[cfg(feature = "rocksdb")] +mod rocksdb_backend { + use super::*; + use slipstream::{RocksDbConfig, RocksDbSnapshot}; + + fn open_no_sync(path: &Path) -> (WatchCursor, RocksDbSnapshot) { + RocksDbSnapshot::open( + path, + RocksDbConfig { + sync: false, + ..Default::default() + }, + ) + .expect("open rocksdb") + } + + #[test] + fn rocksdb_round_trip() { + check_round_trip(open_no_sync); + } + + #[test] + fn rocksdb_get_range() { + check_get_range(open_no_sync); + } + + #[test] + fn rocksdb_for_each_in_range() { + check_for_each_in_range(open_no_sync); + } + + #[test] + fn rocksdb_cursor_resume() { + check_cursor_resume(open_no_sync); + } + + #[test] + fn rocksdb_pure_fold_property() { + check_property_pure_fold(open_no_sync); + } + + #[test] + fn rocksdb_purge() { + check_purge(open_no_sync); + } + + #[test] + fn rocksdb_empty_batch_advances_cursor() { + check_empty_batch_advances_cursor(open_no_sync); + } + + #[test] + fn rocksdb_empty_value_round_trip() { + check_empty_value_round_trip(open_no_sync); + } + + fn import_rocksdb( + artifact: &Path, + dest: &Path, + ) -> Result<(WatchCursor, RocksDbSnapshot), SnapshotError> { + RocksDbSnapshot::import( + artifact, + dest, + RocksDbConfig { + sync: false, + ..Default::default() + }, + ) + } + + #[test] + fn rocksdb_export_import_round_trip() { + check_export_import_round_trip(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_resume_continues_fold() { + check_import_resume_continues_fold(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_tampered_payload() { + check_import_rejects_tampered_payload(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_missing_payload_file() { + check_import_rejects_missing_payload_file(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_undeclared_extra_file() { + check_import_rejects_undeclared_extra_file(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_wrong_backend() { + check_import_rejects_wrong_backend(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_schema_version() { + check_import_rejects_schema_version(open_no_sync, import_rocksdb); + } + + // No `rocksdb_import_rejects_backend_version`: deliberately — the manifest's + // backend_version is informational for RocksDB (the engine reads older + // formats; its own open is the arbiter). See `RocksDbSnapshot::import`. + + #[test] + fn rocksdb_import_rejects_cursor_mismatch() { + check_import_rejects_cursor_mismatch(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_import_rejects_nonempty_dest() { + check_import_rejects_nonempty_dest(open_no_sync, import_rocksdb); + } + + #[test] + fn rocksdb_export_rejects_nonempty_dest() { + check_export_rejects_nonempty_dest(open_no_sync); + } + + #[test] + fn rocksdb_export_empty_store() { + check_export_empty_store(open_no_sync, import_rocksdb); + } + + /// `settle` (flush + wait-for-compact) must preserve the fold byte-for-byte. + #[test] + fn rocksdb_settle_preserves_state() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + let (_r, mut s) = open_no_sync(&path); + fold(&mut s, &stream()); + s.settle().expect("settle"); + assert_eq!(dump(&s), expected_state(), "state survives settle"); + } + + /// NO_SYNC crash-tail recovery. With sync off, commits reach the WAL but are + /// not fsync'd; data and cursor still share one atomic WriteBatch, so whatever + /// survives is mutually consistent. We can't deterministically simulate a + /// power-loss (a clean drop flushes the WAL), so this asserts the load-bearing + /// invariants: after reopen the recovered cursor matches the recovered data, + /// and re-folding the tail from that cursor is idempotent (safe to replay). + #[test] + fn rocksdb_no_sync_tail_is_consistent_and_idempotent() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + let updates = stream(); + + { + let (_r, mut s) = open_no_sync(&path); + fold(&mut s, &updates); + } // drop flushes the WAL + + // Recovered cursor names rev 7, and the data it names is all present. + let (cursor, s) = open_no_sync(&path); + assert_eq!( + cursor.as_u64(), + Some(7), + "cursor recovered, never ahead of data" + ); + assert_eq!(dump(&s), expected_state()); + drop(s); + + // Re-folding the tail (the last batch, revs 6..=7) from the recovered + // cursor is idempotent — replaying the un-synced tail never corrupts state. + let (_r, mut s) = open_no_sync(&path); + fold(&mut s, &updates[5..]); + assert_eq!( + dump(&s), + expected_state(), + "re-folding the tail is idempotent" + ); + } + + /// `RocksDbReader::multi_get` is positionally aligned with its input and + /// agrees with `get` for every key — hits, misses, deleted keys, and the + /// empty-value CAS-tombstone shape — including duplicates and empty input. + #[test] + fn rocksdb_multi_get_matches_get() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + let (_r, mut s) = open_no_sync(&path); + fold(&mut s, &stream()); + // One more apply: an empty-valued put (the CAS-tombstone shape). + s.apply(&[put("lock", b"", 8)], &WatchCursor::from_u64(8)) + .expect("apply lock"); + let reader = s.reader(); + + let keys = [ + "node.a", // live + "missing", // never written + "node.b", // deleted by the stream + "lock", // present with an empty value (CAS tombstone) + "node.a", // duplicate of a live key + ]; + let got = reader.multi_get(keys.iter().copied()).expect("multi_get"); + + assert_eq!(got.len(), keys.len(), "positionally aligned with input"); + for (key, entry) in keys.iter().zip(&got) { + let single = reader.get(key).expect("get"); + assert_eq!( + entry.as_ref().map(|e| (&e.key, &e.value)), + single.as_ref().map(|e| (&e.key, &e.value)), + "multi_get and get disagree for {key:?}" + ); + } + assert!(got[0].is_some(), "live key resolves"); + assert!(got[1].is_none(), "missing key is None"); + assert!(got[2].is_none(), "deleted key is None"); + assert!( + got[3].as_ref().is_some_and(|e| e.value.is_empty()), + "empty value is present, not confused with a delete" + ); + + let empty = reader.multi_get(std::iter::empty()).expect("empty input"); + assert!(empty.is_empty()); + } + + /// With sync on, every commit fsyncs the WAL — round-trip must still hold. + #[test] + fn rocksdb_sync_mode_round_trips() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + { + let (_r, mut s) = RocksDbSnapshot::open( + &path, + RocksDbConfig { + sync: true, + ..Default::default() + }, + ) + .expect("open rocksdb sync"); + fold(&mut s, &stream()); + } + let (cursor, s) = RocksDbSnapshot::open( + &path, + RocksDbConfig { + sync: true, + ..Default::default() + }, + ) + .expect("reopen rocksdb sync"); + assert_eq!(cursor.as_u64(), Some(7)); + assert_eq!(dump(&s), expected_state()); + } +} From 83838923c005527e046470347082a9b2fa88344d Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:15:45 -0700 Subject: [PATCH 06/12] =?UTF-8?q?feat(applied)!:=20ExportRequest=20channel?= =?UTF-8?q?=20=E2=80=94=20export=20a=20live=20fold=20from=20inside=20watch?= =?UTF-8?q?=5Fapplied?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit watch_applied owns its snapshot store by value, so consumers could never export a live fold. It now takes an optional mpsc::Receiver; each request is handled between flushes (pending batch flushed first), so the artifact's embedded cursor is exactly the applied cursor. The export runs on a blocking task with the store moved in and taken back — the same offload as the flush path. Export errors reply to the requester only; the watch keeps running. Dropping the sender disarms the arm. Breaking: watch_applied gains the exports parameter (pass None). Co-Authored-By: Claude Fable 5 --- benches/applied.rs | 1 + src/applied.rs | 291 ++++++++++++++++++++++++++++++++++++++++++++- src/artifact.rs | 1 + src/lib.rs | 2 +- 4 files changed, 292 insertions(+), 3 deletions(-) diff --git a/benches/applied.rs b/benches/applied.rs index 029dfa0..34651e1 100644 --- a/benches/applied.rs +++ b/benches/applied.rs @@ -90,6 +90,7 @@ fn bench_batch_throughput(c: &mut Criterion) { WatchScope::All, None, None::, + None, BatchConfig { window: Duration::from_millis(10), max: 100, diff --git a/src/applied.rs b/src/applied.rs index 3f31bd4..ff0271e 100644 --- a/src/applied.rs +++ b/src/applied.rs @@ -50,15 +50,38 @@ //! of [`watch_applied`] and aborts the watch — that is the caller's contract, //! the same as a panic in any other supplied closure. +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; -use tokio::sync::watch; +use tokio::sync::{oneshot, watch}; use tracing::warn; +use crate::artifact::ExportManifest; use crate::kv::{KvError, KvUpdate, KvWatcher, WatchCursor}; -use crate::snapshot::SnapshotStore; +use crate::snapshot::{SnapshotError, SnapshotStore}; + +/// A request, sent into a running [`watch_applied`] loop, to export the fold it +/// owns (see [`SnapshotStore::export_to`]). +/// +/// `watch_applied` takes its snapshot store **by value**, so a consumer that +/// wants periodic artifacts of a live fold cannot call `export_to` itself. It +/// instead passes an `mpsc::Receiver` to [`watch_applied`] and +/// sends requests through the paired sender. The loop handles a request +/// between batch flushes — after flushing any pending batch — so the artifact's +/// embedded cursor is exactly the applied cursor at the moment of export. +/// +/// The export result (or error) comes back on `reply`; an export failure is +/// reported there and the watch keeps running (the snapshot is a cache — a +/// failed artifact is the requester's problem, not the fold's). +pub struct ExportRequest { + /// Where the artifact directory will be created. Must not exist (or be an + /// empty directory); same filesystem as the fold for cheap hardlinks. + pub dest_dir: PathBuf, + /// Receives the sealed manifest on success. A dropped receiver is ignored. + pub reply: oneshot::Sender>, +} /// What to watch: every key, or every key under a prefix. /// @@ -139,6 +162,12 @@ pub async fn watch_applied( scope: WatchScope, resume: Option, mut store: Option, + // `Some(rx)` arms an export-request arm in the select loop: each + // [`ExportRequest`] is handled between flushes (pending batch flushed + // first), so the exported artifact's cursor is exactly the applied cursor. + // `None` (or dropping the paired sender) leaves the loop's behavior + // unchanged. + mut exports: Option>, config: BatchConfig, mut parse: P, mut apply: A, @@ -290,6 +319,57 @@ where flush!(); } + // Export request. Placed after shutdown/window (they stay prompt) + // and before `rx.recv()` so a firehose of updates cannot starve an + // export indefinitely. The pending batch is flushed first, so the + // exported cursor is exactly the applied cursor; the export itself + // runs on a blocking task with the store moved in and taken back — + // the same offload the flush path uses. + req = async { exports.as_mut().expect("arm guarded by is_some").recv().await }, + if exports.is_some() => { + match req { + Some(ExportRequest { dest_dir, reply }) => { + flush!(); + match store.take() { + Some(mut st) => { + match tokio::task::spawn_blocking(move || { + let res = st.export_to(&dest_dir); + (st, res) + }) + .await + { + // Hand the store back on any clean return; an + // export failure goes to the requester only — + // the watch keeps running (the snapshot is a + // cache). A panicked task lost the store, + // which breaks the resume guarantee: fatal, + // same as the flush path's apply panic. + Ok((st, res)) => { + store = Some(st); + let _ = reply.send(res); + } + Err(e) => { + warn!(error = %e, "snapshot export task panicked; aborting watch"); + handle.abort(); + return Err(KvError::WatchError(format!( + "snapshot export task panicked: {e}" + ))); + } + } + } + None => { + let _ = reply.send(Err(SnapshotError::Backend( + "watch_applied runs without a snapshot store; nothing to export" + .into(), + ))); + } + } + } + // Sender dropped: disarm the arm for the rest of the run. + None => exports = None, + } + } + update = rx.recv() => { match update { Some(u) => { @@ -559,6 +639,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), parse_put, move |batch| ab.lock().unwrap().push(batch), @@ -593,6 +674,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), parse_put, move |batch: Vec>| { @@ -635,6 +717,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig { window: Duration::from_secs(3600), // effectively never max, @@ -674,6 +757,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig { window: Duration::from_secs(3600), // window won't fire max: 100, @@ -724,6 +808,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig { window: Duration::from_secs(3600), max, @@ -770,6 +855,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), // Reject everything — simulates corrupt/irrelevant entries. |_u: &KvUpdate| -> Option> { None }, @@ -813,6 +899,7 @@ mod tests { WatchScope::All, Some(WatchCursor::from_u64(5)), // resume position that "expired" None::, + None, BatchConfig::default(), parse_put, move |batch: Vec>| ab.lock().unwrap().extend(batch), @@ -849,6 +936,7 @@ mod tests { WatchScope::All, None, Some(store), + None, BatchConfig::default(), parse_put, move |_batch: Vec>| {}, @@ -896,6 +984,7 @@ mod tests { WatchScope::All, Some(WatchCursor::from_u64(9)), // resume past rev 9 — not expired None::, + None, BatchConfig::default(), parse_put, move |batch: Vec>| ab.lock().unwrap().extend(batch), @@ -934,6 +1023,7 @@ mod tests { WatchScope::Prefix("node.".to_string()), None, None::, + None, BatchConfig::default(), parse_put, move |batch: Vec>| ab.lock().unwrap().extend(batch), @@ -972,6 +1062,7 @@ mod tests { WatchScope::Prefix("node.".to_string()), Some(WatchCursor::from_u64(5)), // resume position that "expired" None::, + None, BatchConfig::default(), parse_put, move |batch: Vec>| ab.lock().unwrap().extend(batch), @@ -1001,6 +1092,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), parse_put, move |_batch: Vec>| {}, @@ -1040,6 +1132,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), // Keep only keys under "keep."; reject everything else. |u: &KvUpdate| -> Option> { @@ -1086,6 +1179,7 @@ mod tests { WatchScope::All, None, None::, + None, BatchConfig::default(), parse_put, move |_batch: Vec>| { @@ -1115,6 +1209,198 @@ mod tests { ); } + /// An [`ExportRequest`] flushes the pending batch first, so the artifact's + /// cursor is exactly the applied cursor — and the artifact is importable + /// with the batched entries in it. + #[tokio::test(start_paused = true)] + async fn export_request_flushes_pending_batch_first() { + let dir = tempfile::TempDir::new().unwrap(); + let store_path = dir.path().join("fold.snap"); + let artifact = dir.path().join("artifact"); + let (_r, store) = AppendLogSnapshot::open(&store_path, u64::MAX).unwrap(); + + let updates = vec![put("a", b"1", 1), put("b", b"2", 2)]; + let watcher = Arc::new(MockWatcher::new(updates, true)); // hold open + let (sd_tx, sd_rx) = watch::channel(false); + let (ex_tx, ex_rx) = mpsc::channel(1); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + Some(store), + Some(ex_rx), + BatchConfig { + window: Duration::from_secs(3600), // window never fires + max: 100, + }, + parse_put, + move |_batch: Vec>| {}, + move |_| {}, + sd_rx, + )); + + // Let both updates land in the (unflushed) pending batch, then export. + tokio::time::sleep(Duration::from_millis(1)).await; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + ex_tx + .send(ExportRequest { + dest_dir: artifact.clone(), + reply: reply_tx, + }) + .await + .unwrap(); + + let manifest = reply_rx.await.unwrap().expect("export succeeds"); + assert_eq!( + manifest.cursor.as_u64(), + Some(2), + "pending batch flushed before export: artifact cursor is the applied cursor" + ); + + // The artifact is importable and holds both batched entries. + let (cursor, imported) = + AppendLogSnapshot::import(&artifact, &dir.path().join("imported.snap"), u64::MAX) + .unwrap(); + assert_eq!(cursor.as_u64(), Some(2)); + assert_eq!(imported.get("a").unwrap().unwrap().value, b"1"); + assert_eq!(imported.get("b").unwrap().unwrap().value, b"2"); + + sd_tx.send(true).unwrap(); + task.await.unwrap().unwrap(); + } + + /// An export request against a store-less watch replies with an error and + /// the watch keeps running. + #[tokio::test(start_paused = true)] + async fn export_without_store_replies_error() { + let watcher = Arc::new(MockWatcher::new(vec![put("a", b"1", 1)], true)); + let (sd_tx, sd_rx) = watch::channel(false); + let (ex_tx, ex_rx) = mpsc::channel(1); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + None::, + Some(ex_rx), + BatchConfig::default(), + parse_put, + move |_batch: Vec>| {}, + move |_| {}, + sd_rx, + )); + + tokio::time::sleep(Duration::from_millis(1)).await; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + ex_tx + .send(ExportRequest { + dest_dir: std::env::temp_dir().join("never-created"), + reply: reply_tx, + }) + .await + .unwrap(); + assert!( + reply_rx.await.unwrap().is_err(), + "no store → export errors via the reply" + ); + + // The watch is still alive and returns its applied cursor on shutdown. + sd_tx.send(true).unwrap(); + let cursor = task.await.unwrap().unwrap(); + assert_eq!(cursor.as_u64(), Some(1)); + } + + /// An export failure (unavailable destination) is reported on the reply and + /// the watch keeps applying later updates. + #[tokio::test(start_paused = true)] + async fn export_error_does_not_kill_watch() { + let dir = tempfile::TempDir::new().unwrap(); + let store_path = dir.path().join("fold.snap"); + let (_r, store) = AppendLogSnapshot::open(&store_path, u64::MAX).unwrap(); + + // Occupied destination → export fails. + let occupied = dir.path().join("occupied"); + std::fs::create_dir(&occupied).unwrap(); + std::fs::write(occupied.join("stray"), b"x").unwrap(); + + let watcher = Arc::new(MockWatcher::new(vec![put("a", b"1", 1)], true)); + let (sd_tx, sd_rx) = watch::channel(false); + let (ex_tx, ex_rx) = mpsc::channel(1); + + let applied = Arc::new(AtomicU64::new(0)); + let a = Arc::clone(&applied); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + Some(store), + Some(ex_rx), + BatchConfig::default(), + parse_put, + move |_batch: Vec>| {}, + move |cur| a.store(cur.as_u64().unwrap(), Ordering::SeqCst), + sd_rx, + )); + + tokio::time::sleep(Duration::from_millis(1)).await; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + ex_tx + .send(ExportRequest { + dest_dir: occupied, + reply: reply_tx, + }) + .await + .unwrap(); + match reply_rx.await.unwrap() { + Err(crate::snapshot::SnapshotError::ArtifactInvalid(_)) => {} + other => panic!("expected ArtifactInvalid, got {other:?}"), + } + + // Watch still folds: a clean shutdown returns the applied cursor. + sd_tx.send(true).unwrap(); + let cursor = task.await.unwrap().unwrap(); + assert_eq!(cursor.as_u64(), Some(1), "watch survived the failed export"); + assert_eq!(applied.load(Ordering::SeqCst), 1); + } + + /// Dropping the export sender disarms the arm; the loop keeps batching and + /// flushing normally. + #[tokio::test(start_paused = true)] + async fn export_sender_dropped_disarms_channel() { + let watcher = Arc::new(MockWatcher::new(vec![put("a", b"1", 1)], true)); + let (sd_tx, sd_rx) = watch::channel(false); + let (ex_tx, ex_rx) = mpsc::channel::(1); + + let applied = Arc::new(AtomicU64::new(0)); + let a = Arc::clone(&applied); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + None::, + Some(ex_rx), + BatchConfig::default(), + parse_put, + move |_batch: Vec>| {}, + move |cur| a.store(cur.as_u64().unwrap(), Ordering::SeqCst), + sd_rx, + )); + + drop(ex_tx); // disarm + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + applied.load(Ordering::SeqCst), + 1, + "loop keeps flushing after the export sender is gone" + ); + + sd_tx.send(true).unwrap(); + task.await.unwrap().unwrap(); + } + /// With a low `compact_threshold`, the flush path's `spawn_blocking` /// compaction actually fires (every other snapshot test pins the threshold /// at `u64::MAX`, leaving that branch dead). After a compacting run the @@ -1144,6 +1430,7 @@ mod tests { WatchScope::All, None, Some(store), + None, BatchConfig { window: Duration::from_secs(3600), max: 1, // one update per flush → a compaction per update diff --git a/src/artifact.rs b/src/artifact.rs index 4dc6c7b..875e854 100644 --- a/src/artifact.rs +++ b/src/artifact.rs @@ -514,6 +514,7 @@ impl ImportStage { /// Rename the whole staged payload directory onto `dest` (directory-shaped /// backends: fjall, RocksDB). + #[cfg(any(feature = "fjall", feature = "rocksdb"))] pub(crate) fn finalize_dir(self) -> Result<(), SnapshotError> { check_dest_available(&self.dest)?; rename_into_place(&self.payload(), &self.dest) diff --git a/src/lib.rs b/src/lib.rs index 1e34626..d823045 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,7 @@ mod snapshot_record; mod snapshot_rocksdb; mod stores; -pub use applied::{BatchConfig, WatchScope, watch_applied}; +pub use applied::{BatchConfig, ExportRequest, WatchScope, watch_applied}; pub use artifact::{ARTIFACT_SCHEMA_VERSION, ArtifactFile, ExportManifest, MANIFEST_FILE}; pub use kv::{ KvEntry, KvError, KvReader, KvTtl, KvUpdate, KvWatcher, KvWriter, VersionToken, WatchCursor, From d7308f74c5d4b2a040f277316bd5cafbeae4ca93 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:20:12 -0700 Subject: [PATCH 07/12] =?UTF-8?q?feat(lease):=20ExportLease=20=E2=80=94=20?= =?UTF-8?q?at=20most=20one=20export=20per=20round,=20fleet-wide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single KV key coordinates the checkpoint loop across every replica of a fold: acquisition is create-only (one winner by construction), the lease lifetime is an expires_at embedded in the value (portable to any KvWriter — no per-message TTL, no server-version or bucket-flag requirements), and an expired/corrupt lease is taken over with a CAS update so the steal race also has one winner. The ttl IS the round period; complete() publishes the exported cursor on the key, making it the fleet-visible last-export record. Clock-skew worst case is a duplicate export — safe, the lease is dedup, not a correctness gate. Integration-tested against a real nats-server: 8-way concurrent acquire has exactly one winner; held lease blocks; expiry allows takeover; completion is fleet-visible. Co-Authored-By: Claude Fable 5 --- src/export_lease.rs | 227 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + tests/integration.rs | 120 +++++++++++++++++++++++ 3 files changed, 349 insertions(+) create mode 100644 src/export_lease.rs diff --git a/src/export_lease.rs b/src/export_lease.rs new file mode 100644 index 0000000..090b871 --- /dev/null +++ b/src/export_lease.rs @@ -0,0 +1,227 @@ +//! Fleet-wide "export at most once per round" lease. +//! +//! Every replica of a fold runs the same checkpoint loop; without coordination, +//! N nodes would produce N identical artifacts per round. [`ExportLease`] makes +//! exactly one of them do the work: each candidate calls +//! [`try_acquire`](ExportLease::try_acquire) when its trigger fires; one wins, +//! the rest skip the round. +//! +//! ## Mechanism: CAS + embedded expiry — no TTL machinery +//! +//! The lease is a single KV key. Acquisition is a **create-only** write +//! (`KvWriter::create`): exactly one caller fleet-wide can create a missing +//! key, so the race has one winner by construction. The lease's lifetime is an +//! `expires_at_unix` timestamp **inside the value** — not a server-side TTL — +//! and an expired (or unparseable) lease is taken over with a CAS +//! [`update`](crate::KvWriter::update) against the observed version, so the +//! steal race also has exactly one winner. +//! +//! Embedding the expiry rather than using per-message TTL keeps the lease +//! portable to any [`KvWriter`] backend and free of server-version/bucket-flag +//! requirements. The cost is wall-clock comparison across nodes: with +//! NTP-sane clocks and round periods measured in minutes, skew is noise — and +//! a premature steal is *safe* anyway (two exporters produce two identical +//! artifacts; the upload is last-write-wins on the same key). The lease is a +//! work-deduplication optimization, never a correctness gate. +//! +//! ## Lifecycle +//! +//! A successful round leaves the key in place until it expires — that is the +//! "at most once per `ttl`" semantic: `ttl` IS the round period. The winner +//! calls [`LeaseGuard::complete`] after its upload succeeds, which (best +//! effort) rewrites the value with the exported cursor and completion time, so +//! the lease key doubles as the fleet-visible "last export" record. A crash +//! mid-round simply lets the key expire; the next trigger elects someone else. + +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use tracing::{debug, warn}; + +use crate::artifact::hex_encode; +use crate::kv::{KvError, KvReader, KvWriter, VersionToken, WatchCursor}; +use crate::stores::KvStore; + +/// The lease key's value: who holds (or last held) the round, until when, and +/// — after [`LeaseGuard::complete`] — what was exported. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LeaseRecord { + /// Identity of the node that won the round (caller-chosen, e.g. node id). + pub holder_id: String, + /// When the round was won, seconds since the Unix epoch. + pub acquired_at_unix: u64, + /// When the lease lapses and the next round may be won. This is the round + /// period: the "at most once per `ttl`" bound. + pub expires_at_unix: u64, + /// Hex of the exported artifact's cursor, set by [`LeaseGuard::complete`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_cursor_hex: Option, + /// When the round completed (artifact uploaded), set by + /// [`LeaseGuard::complete`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_at_unix: Option, +} + +/// Coordinates "at most one export per round" across every replica of a fold. +/// See the [module docs](self). +pub struct ExportLease { + reader: Arc, + writer: Arc, + key: String, + holder_id: String, +} + +/// Proof of a won round, returned by [`ExportLease::try_acquire`]. +/// +/// There is nothing to release — the round runs until the lease expires (that +/// is the round period). Call [`complete`](Self::complete) after the artifact +/// is safely uploaded to publish what was exported. +pub struct LeaseGuard { + writer: Arc, + key: String, + record: LeaseRecord, + version: VersionToken, +} + +impl ExportLease { + /// A lease on `key` in `store`, identifying this node as `holder_id`. + /// + /// Fails with [`KvError::OperationFailed`] if the store has no writer. + pub fn new( + store: &dyn KvStore, + key: impl Into, + holder_id: impl Into, + ) -> Result { + let writer = store.writer().ok_or_else(|| { + KvError::OperationFailed(format!( + "store {:?} has no writer; an export lease needs create/update", + store.name() + )) + })?; + Ok(Self { + reader: store.reader(), + writer, + key: key.into(), + holder_id: holder_id.into(), + }) + } + + /// Try to win this export round. Exactly one caller fleet-wide gets + /// `Ok(Some(guard))` per round; everyone else gets `Ok(None)` and skips. + /// + /// `ttl` is the round period: the winner's lease suppresses further rounds + /// until it lapses, whether or not the winner survives. Crash mid-round → + /// the key expires → the next trigger elects someone else. + pub async fn try_acquire(&self, ttl: Duration) -> Result, KvError> { + let now = unix_now(); + let record = LeaseRecord { + holder_id: self.holder_id.clone(), + acquired_at_unix: now, + expires_at_unix: now.saturating_add(ttl.as_secs()), + completed_cursor_hex: None, + completed_at_unix: None, + }; + let bytes = serde_json::to_vec(&record) + .map_err(|e| KvError::SerializationError(e.to_string()))?; + + // Fast path: the round is open — create-only, one winner. + match self.writer.create(&self.key, &bytes).await { + Ok(version) => { + debug!(key = %self.key, holder = %self.holder_id, "export lease acquired (create)"); + return Ok(Some(self.guard(record, version))); + } + Err(KvError::AlreadyExists) => {} + Err(e) => return Err(e), + } + + // The key exists. `entry` (not `get`): a CAS-deleted lease is an + // empty-value tombstone that `get` hides, but its version is exactly + // what the takeover CAS needs. + let Some(entry) = self.reader.entry(&self.key).await? else { + // Deleted between create and read; treat as lost — the next + // trigger retries cleanly rather than looping here. + return Ok(None); + }; + + // A live, parseable, unexpired lease wins; anything else (expired, + // tombstone, unparseable garbage) is taken over. Unparseable leases + // MUST be stealable or one corrupt write wedges exports fleet-wide. + if let Ok(existing) = serde_json::from_slice::(&entry.value) + && existing.expires_at_unix > now + { + return Ok(None); + } + + // Takeover: CAS against the version we read — one winner. + match self.writer.update(&self.key, &bytes, &entry.version).await { + Ok(version) => { + debug!(key = %self.key, holder = %self.holder_id, "export lease acquired (takeover)"); + Ok(Some(self.guard(record, version))) + } + // Someone else's create/update landed first: their round. + Err(KvError::RevisionMismatch | KvError::AlreadyExists | KvError::KeyNotFound) => { + Ok(None) + } + Err(e) => Err(e), + } + } + + /// Read the current lease record, if any — the fleet-visible "last export" + /// state. `None` when no round has ever run (or the key was tombstoned). + pub async fn current(&self) -> Result, KvError> { + match self.reader.get(&self.key).await? { + Some(entry) => Ok(serde_json::from_slice(&entry.value).ok()), + None => Ok(None), + } + } + + fn guard(&self, record: LeaseRecord, version: VersionToken) -> LeaseGuard { + LeaseGuard { + writer: Arc::clone(&self.writer), + key: self.key.clone(), + record, + version, + } + } +} + +impl LeaseGuard { + /// The record this guard wrote when it won the round. + pub fn record(&self) -> &LeaseRecord { + &self.record + } + + /// Publish the round's outcome: rewrite the lease value with the exported + /// cursor and completion time (expiry unchanged — the round still runs its + /// full period). + /// + /// Best-effort observability: a CAS conflict means the lease was already + /// taken over (this round overran its ttl) and is logged, not surfaced — + /// the artifact is already safe wherever the caller put it. + pub async fn complete(mut self, cursor: &WatchCursor) -> Result<(), KvError> { + self.record.completed_cursor_hex = Some(hex_encode(cursor.version().as_bytes())); + self.record.completed_at_unix = Some(unix_now()); + let bytes = serde_json::to_vec(&self.record) + .map_err(|e| KvError::SerializationError(e.to_string()))?; + match self.writer.update(&self.key, &bytes, &self.version).await { + Ok(_) => Ok(()), + Err(KvError::RevisionMismatch) => { + warn!( + key = %self.key, + holder = %self.record.holder_id, + "export round overran its lease; completion record skipped" + ); + Ok(()) + } + Err(e) => Err(e), + } + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} diff --git a/src/lib.rs b/src/lib.rs index d823045..24903ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ mod applied; mod artifact; +mod export_lease; mod kv; mod nats; pub mod snapshot; @@ -38,6 +39,7 @@ mod stores; pub use applied::{BatchConfig, ExportRequest, WatchScope, watch_applied}; pub use artifact::{ARTIFACT_SCHEMA_VERSION, ArtifactFile, ExportManifest, MANIFEST_FILE}; +pub use export_lease::{ExportLease, LeaseGuard, LeaseRecord}; pub use kv::{ KvEntry, KvError, KvReader, KvTtl, KvUpdate, KvWatcher, KvWriter, VersionToken, WatchCursor, }; diff --git a/tests/integration.rs b/tests/integration.rs index 05ff40b..1cb832d 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1228,6 +1228,7 @@ fn spawn_applied( WatchScope::All, baseline, snapshot, + None, BatchConfig::default(), parse, move |batch: Vec| { @@ -1491,3 +1492,122 @@ async fn applied_survives_compacted_resume_cursor() { "combinator returned a live cursor after recovering from compaction" ); } + +// --- Export lease -------------------------------------------------------------- + +/// N concurrent acquirers race one round: exactly one wins, everyone else +/// cleanly loses. This is the load-bearing claim — create-only CAS gives the +/// race one winner by construction, against a real JetStream bucket. +#[tokio::test(flavor = "multi_thread")] +async fn export_lease_one_winner_among_concurrent_acquirers() { + let nats = TestNats::start().await; + let (_conn, store) = nats.store("lease-race").await; + + let mut tasks = Vec::new(); + for i in 0..8 { + let lease = slipstream::ExportLease::new(store.as_ref(), "export.edge.us-east", format!("node-{i}")) + .expect("lease"); + tasks.push(tokio::spawn(async move { + lease + .try_acquire(Duration::from_secs(60)) + .await + .expect("try_acquire") + .is_some() + })); + } + + let mut winners = 0; + for t in tasks { + if t.await.expect("join") { + winners += 1; + } + } + assert_eq!(winners, 1, "exactly one node wins the round"); +} + +/// A held (unexpired) lease blocks every later acquirer until it lapses. +#[tokio::test(flavor = "multi_thread")] +async fn export_lease_held_blocks_next_round() { + let nats = TestNats::start().await; + let (_conn, store) = nats.store("lease-held").await; + + let a = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-a").expect("lease"); + let b = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-b").expect("lease"); + + let guard = a + .try_acquire(Duration::from_secs(60)) + .await + .expect("acquire") + .expect("a wins the open round"); + assert!( + b.try_acquire(Duration::from_secs(60)) + .await + .expect("try_acquire") + .is_none(), + "a live lease blocks the round" + ); + drop(guard); // dropping the guard does NOT release: the ttl is the round period + assert!( + b.try_acquire(Duration::from_secs(60)) + .await + .expect("try_acquire") + .is_none(), + "the round persists past the winner's guard" + ); +} + +/// An expired lease is taken over via CAS — and only by one taker. +#[tokio::test(flavor = "multi_thread")] +async fn export_lease_expiry_allows_takeover() { + let nats = TestNats::start().await; + let (_conn, store) = nats.store("lease-expiry").await; + + let a = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-a").expect("lease"); + let b = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-b").expect("lease"); + + let _ = a + .try_acquire(Duration::from_secs(1)) + .await + .expect("acquire") + .expect("a wins"); + tokio::time::sleep(Duration::from_millis(1100)).await; + + let guard = b + .try_acquire(Duration::from_secs(60)) + .await + .expect("try_acquire") + .expect("expired lease is taken over"); + assert_eq!(guard.record().holder_id, "node-b"); + + let current = b.current().await.expect("read").expect("record exists"); + assert_eq!(current.holder_id, "node-b", "takeover is visible fleet-wide"); +} + +/// `complete` publishes the exported cursor on the lease key — the +/// fleet-visible "last export" record. +#[tokio::test(flavor = "multi_thread")] +async fn export_lease_complete_publishes_outcome() { + let nats = TestNats::start().await; + let (_conn, store) = nats.store("lease-complete").await; + + let lease = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-a").expect("lease"); + let guard = lease + .try_acquire(Duration::from_secs(60)) + .await + .expect("acquire") + .expect("wins"); + + guard + .complete(&WatchCursor::from_u64(42)) + .await + .expect("complete"); + + let record = lease.current().await.expect("read").expect("record"); + assert_eq!( + record.completed_cursor_hex.as_deref(), + Some("000000000000002a"), + "exported cursor is published" + ); + assert!(record.completed_at_unix.is_some()); + assert_eq!(record.holder_id, "node-a"); +} From ce83fcb8476c5601182a098c8a33a394ca845649 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:28:54 -0700 Subject: [PATCH 08/12] feat(transport): ArtifactTransport + ObjectStoreTransport + run_export_round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire format: plain tar of the artifact dir (payload files are already lz4/zstd-compressed) at /, manifest duplicated as a sibling .manifest.json uploaded LAST (the remote completeness marker, and a cheap cursor peek without downloading the payload). Download cross-checks the embedded manifest against the sibling; the backend import re-verifies every file hash regardless — transport is untrusted. ObjectStoreTransport wraps any object_store backend (S3/GCS/Azure/local-fs; feature 'transport'), multipart-streams uploads memory-bounded, and unpacks downloads stage-then-rename so a torn download never lands at the dest. run_export_round composes the at-most-once round: lease -> export through the watch_applied channel -> upload -> publish completion (only after the upload) -> delete the local artifact (transience enforced). Failures abandon the lease (new LeaseGuard::abandon) so the fleet retries promptly instead of waiting out the ttl. Per-backend import_remote bootstraps a fold from the remote artifact in one call. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 704 +++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 5 + src/artifact.rs | 15 +- src/export_lease.rs | 28 ++ src/lib.rs | 4 + src/transport.rs | 454 ++++++++++++++++++++++++++++ tests/transport.rs | 409 +++++++++++++++++++++++++ 7 files changed, 1598 insertions(+), 21 deletions(-) create mode 100644 src/transport.rs create mode 100644 tests/transport.rs diff --git a/Cargo.lock b/Cargo.lock index ed0f45a..7b69167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -56,10 +65,10 @@ dependencies = [ "once_cell", "pin-project", "portable-atomic", - "rand", + "rand 0.8.6", "regex", "ring", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "rustls-pki-types", "rustls-webpki 0.102.8", "serde", @@ -89,6 +98,12 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" @@ -119,9 +134,11 @@ dependencies = [ "criterion", "fjall", "futures", + "object_store", "rust-rocksdb", "serde", "serde_json", + "tar", "tempfile", "thiserror 2.0.18", "tokio", @@ -138,7 +155,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -246,6 +263,35 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -337,6 +383,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -382,7 +438,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -403,7 +459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -613,6 +669,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -645,6 +711,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -765,8 +837,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -776,9 +850,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -790,6 +866,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -800,6 +877,25 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -860,12 +956,125 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs 0.8.4", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -995,6 +1204,12 @@ dependencies = [ "compare", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is-terminal" version = "0.4.17" @@ -1015,6 +1230,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1103,6 +1327,12 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lsm-tree" version = "3.1.4" @@ -1144,6 +1374,16 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.1" @@ -1178,7 +1418,7 @@ dependencies = [ "ed25519-dalek", "getrandom 0.2.17", "log", - "rand", + "rand 0.8.6", "signatory", ] @@ -1198,7 +1438,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83" dependencies = [ - "rand", + "rand 0.8.6", ] [[package]] @@ -1216,6 +1456,44 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1234,6 +1512,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1391,6 +1675,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.6.23" @@ -1401,6 +1695,61 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -1429,8 +1778,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -1440,7 +1810,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1452,6 +1832,21 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rayon" version = "1.12.0" @@ -1510,6 +1905,48 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs 0.8.4", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -1600,11 +2037,23 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", ] [[package]] @@ -1622,6 +2071,7 @@ version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ + "web-time", "zeroize", ] @@ -1652,6 +2102,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -1683,7 +2139,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1774,6 +2243,18 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sfa" version = "1.0.0" @@ -1815,7 +2296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "zeroize", ] @@ -1827,7 +2308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1894,6 +2375,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1905,6 +2395,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -2009,6 +2510,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -2081,7 +2597,7 @@ dependencies = [ "futures-sink", "http", "httparse", - "rand", + "rand 0.8.6", "ring", "rustls-pki-types", "tokio", @@ -2090,6 +2606,51 @@ dependencies = [ "webpki-roots 0.26.11", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -2121,6 +2682,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tryhard" version = "0.5.2" @@ -2207,6 +2774,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2244,6 +2820,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.122" @@ -2298,6 +2884,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -2320,6 +2919,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2347,12 +2956,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2535,6 +3197,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 826427f..864bb05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,11 @@ blake3 = "1.8.5" crc32fast = "1" fjall = { version = "3", optional = true } futures = "0.3" +object_store = { version = "0.13.2", optional = true, features = ["aws", "fs"] } rocksdb = { package = "rust-rocksdb", version = "0.50", optional = true, default-features = false, features = ["lz4", "zstd", "bindgen-runtime"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +tar = { version = "0.4.46", optional = true } tempfile = "3" thiserror = "2" tokio = { version = "1", features = ["macros", "rt", "sync", "time"] } @@ -59,3 +61,6 @@ required-features = ["fjall", "rocksdb"] [features] fjall = ["dep:fjall"] rocksdb = ["dep:rocksdb"] +# Artifact transport: tar wire format + object_store (S3/GCS/Azure/local-fs) +# upload/download, plus run_export_round. tokio/fs streams downloads to disk. +transport = ["dep:object_store", "dep:tar", "tokio/fs"] diff --git a/src/artifact.rs b/src/artifact.rs index 875e854..6f8942c 100644 --- a/src/artifact.rs +++ b/src/artifact.rs @@ -199,10 +199,6 @@ pub(crate) fn write_manifest( } /// Read and validate `MANIFEST.json` from an artifact directory. -/// -/// Validates the schema version and every file path (relative, `/`-separated, -/// no `..`, no `\`, under `data/`) so a hostile or corrupted manifest can never -/// direct a copy outside the staging area (zip-slip). pub(crate) fn read_manifest(artifact_dir: &Path) -> Result { let path = artifact_dir.join(MANIFEST_FILE); let data = fs::read(&path).map_err(|e| { @@ -212,8 +208,17 @@ pub(crate) fn read_manifest(artifact_dir: &Path) -> Result Result { let wire: ManifestWire = - serde_json::from_slice(&data).map_err(|e| invalid(format!("malformed manifest: {e}")))?; + serde_json::from_slice(data).map_err(|e| invalid(format!("malformed manifest: {e}")))?; if wire.schema_version != ARTIFACT_SCHEMA_VERSION { return Err(invalid(format!( diff --git a/src/export_lease.rs b/src/export_lease.rs index 090b871..65631dd 100644 --- a/src/export_lease.rs +++ b/src/export_lease.rs @@ -192,6 +192,34 @@ impl LeaseGuard { &self.record } + /// Give the round back early: a failed export/upload should not suppress + /// the fleet for the rest of the ttl. CAS-deletes the lease (tombstone) + /// against this guard's version, so the next trigger on any node can win a + /// fresh round immediately. + /// + /// Best-effort: a CAS conflict (someone already took over) or write error + /// is logged, not surfaced — worst case the round waits out its ttl, which + /// is the no-abandon behavior anyway. + pub async fn abandon(self) { + match self + .writer + .delete_with_version(&self.key, &self.version) + .await + { + Ok(_) => { + debug!(key = %self.key, holder = %self.record.holder_id, "export lease abandoned"); + } + Err(e) => { + warn!( + key = %self.key, + holder = %self.record.holder_id, + error = %e, + "failed to abandon export lease; next round waits for expiry" + ); + } + } + } + /// Publish the round's outcome: rewrite the lease value with the exported /// cursor and completion time (expiry unchanged — the round still runs its /// full period). diff --git a/src/lib.rs b/src/lib.rs index 24903ce..a4739af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,6 +36,8 @@ mod snapshot_record; #[cfg(feature = "rocksdb")] mod snapshot_rocksdb; mod stores; +#[cfg(feature = "transport")] +mod transport; pub use applied::{BatchConfig, ExportRequest, WatchScope, watch_applied}; pub use artifact::{ARTIFACT_SCHEMA_VERSION, ArtifactFile, ExportManifest, MANIFEST_FILE}; @@ -50,3 +52,5 @@ pub use snapshot_fjall::{FjallConfig, FjallReader, FjallSnapshot}; #[cfg(feature = "rocksdb")] pub use snapshot_rocksdb::{RocksDbConfig, RocksDbReader, RocksDbSnapshot}; pub use stores::{Connection, ConnectionCapabilities, KvStore, StorageType, StoreConfig}; +#[cfg(feature = "transport")] +pub use transport::{ArtifactTransport, ObjectStoreTransport, run_export_round}; diff --git a/src/transport.rs b/src/transport.rs new file mode 100644 index 0000000..3af1a5a --- /dev/null +++ b/src/transport.rs @@ -0,0 +1,454 @@ +//! Artifact transport: ship export artifacts to object storage and fetch them +//! back for bootstrap. Feature `transport`. +//! +//! The wire format is a **plain tar** of the artifact directory +//! (`MANIFEST.json` + `data/…`) at `/`, with the manifest +//! duplicated as a sibling object `.manifest.json` so a node can peek at +//! an artifact's cursor/backend without downloading the payload. No +//! compression layer: fjall/RocksDB payload files are already lz4/zstd +//! compressed. The sibling manifest is uploaded **last**, so its presence +//! means the payload object is complete — the remote twin of the local +//! artifact's manifest-written-last discipline. +//! +//! Transport is **untrusted**: [`download`](ArtifactTransport::download) +//! cross-checks the tar's embedded manifest against the sibling object, and +//! the backend `import` re-verifies every payload file hash regardless. +//! +//! [`run_export_round`] composes the whole at-most-once round: +//! lease → export (through the [`watch_applied`](crate::watch_applied) +//! [`ExportRequest`] channel) → upload → publish completion → **delete the +//! local artifact** (artifacts hardlink fold files and pin storage if they +//! linger — transience is enforced here, not hoped for). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use object_store::path::Path as ObjPath; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload, WriteMultipart}; +use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot}; +use tracing::warn; + +use crate::applied::ExportRequest; +use crate::artifact::{ + ExportManifest, MANIFEST_FILE, check_dest_available, manifest_from_slice, +}; +use crate::export_lease::ExportLease; +use crate::kv::WatchCursor; +use crate::snapshot::SnapshotError; + +/// Buffered chunk size for uploads/downloads (also the multipart part size). +/// 8 MiB clears S3's 5 MiB minimum part size with headroom. +const CHUNK: usize = 8 << 20; + +/// Concurrent in-flight multipart parts (bounds upload memory to +/// `CHUNK × MAX_CONCURRENT_PARTS`). +const MAX_CONCURRENT_PARTS: usize = 8; + +/// Ship artifacts to durable storage and fetch them back. See [module](self) +/// docs for the wire format. +#[async_trait] +pub trait ArtifactTransport: Send + Sync { + /// Tar `artifact_dir` and upload it at `key`, then upload the manifest as + /// the sibling object `.manifest.json` (the completeness marker). + /// Re-uploading the same `key` overwrites — "latest" keys are + /// last-write-wins by design. + async fn upload(&self, key: &str, artifact_dir: &Path) -> Result<(), SnapshotError>; + + /// Fetch only the sibling manifest — peek at an artifact's cursor and + /// backend before committing to a payload download. + async fn manifest(&self, key: &str) -> Result; + + /// Download and unpack the artifact at `key` into `dest_dir` (which must + /// not exist or be an empty directory), returning its manifest. The + /// unpacked directory is a local artifact, ready for the backend's + /// `import` — which re-verifies every file hash; this method only + /// cross-checks the embedded manifest against the sibling object. + async fn download(&self, key: &str, dest_dir: &Path) -> Result; +} + +/// [`ArtifactTransport`] over any [`object_store::ObjectStore`] — S3, GCS, +/// Azure, or local filesystem — under a key prefix. +pub struct ObjectStoreTransport { + store: Arc, + prefix: ObjPath, +} + +impl ObjectStoreTransport { + /// Wrap an already-configured store. Keys are placed under `prefix`. + pub fn new(store: Arc, prefix: impl AsRef) -> Self { + Self { + store, + prefix: ObjPath::from(prefix.as_ref()), + } + } + + /// Build from a URL (`s3://bucket/prefix`, `file:///path`, …) plus + /// explicit builder options (e.g. `aws_endpoint`, `aws_access_key_id`, + /// `aws_virtual_hosted_style_request`). + /// + /// NOTE: [`object_store::parse_url_opts`] does **not** read process env + /// vars — every non-default setting (credentials, endpoint, path-style) + /// must be passed in `options`. To use env-based configuration, build the + /// store yourself (e.g. `AmazonS3Builder::from_env()`) and use + /// [`new`](Self::new). + pub fn from_url_opts(url: &str, options: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: Into, + { + let url = url::Url::parse(url) + .map_err(|e| SnapshotError::Backend(format!("invalid transport url: {e}")))?; + let (store, prefix) = object_store::parse_url_opts(&url, options).map_err(map_obj)?; + Ok(Self { + store: Arc::from(store), + prefix, + }) + } + + // `Path::from` parses `/` separators, so multi-segment keys + // (`edge-origins/us-east/latest`) land as real object hierarchy. + fn payload_path(&self, key: &str) -> ObjPath { + ObjPath::from(format!("{}/{key}", self.prefix)) + } + + fn manifest_path(&self, key: &str) -> ObjPath { + ObjPath::from(format!("{}/{key}.manifest.json", self.prefix)) + } +} + +#[async_trait] +impl ArtifactTransport for ObjectStoreTransport { + async fn upload(&self, key: &str, artifact_dir: &Path) -> Result<(), SnapshotError> { + // Read the manifest first — it doubles as the artifact-completeness + // check (export writes it last). + let manifest_bytes = tokio::fs::read(artifact_dir.join(MANIFEST_FILE)) + .await + .map_err(SnapshotError::Io)?; + // Validate before shipping: never upload an artifact we couldn't read back. + manifest_from_slice(&manifest_bytes)?; + + // Tar the artifact into a temp file on a blocking task. A temp file + // (rather than streaming the tar straight into the upload) keeps the + // blocking tar writer and the async multipart writer decoupled; the + // disk cost is one tar's worth, transient. + let src = artifact_dir.to_path_buf(); + let tar_file = tokio::task::spawn_blocking(move || -> Result { + let tmp = tempfile::NamedTempFile::new()?; + let mut builder = tar::Builder::new(std::io::BufWriter::new(tmp.reopen()?)); + builder.append_dir_all(".", &src)?; + builder.into_inner()?.into_inner().map_err(|e| SnapshotError::Io(e.into_error()))?; + Ok(tmp) + }) + .await + .map_err(|e| SnapshotError::Backend(format!("tar task panicked: {e}")))??; + + // Stream the tar up as multipart, memory-bounded. + let mut file = tokio::fs::File::open(tar_file.path()) + .await + .map_err(SnapshotError::Io)?; + let upload = self + .store + .put_multipart(&self.payload_path(key)) + .await + .map_err(map_obj)?; + let mut wm = WriteMultipart::new_with_chunk_size(upload, CHUNK); + let mut buf = vec![0u8; CHUNK]; + loop { + use tokio::io::AsyncReadExt; + let n = file.read(&mut buf).await.map_err(SnapshotError::Io)?; + if n == 0 { + break; + } + wm.wait_for_capacity(MAX_CONCURRENT_PARTS) + .await + .map_err(map_obj)?; + wm.write(&buf[..n]); + } + wm.finish().await.map_err(map_obj)?; + + // Manifest sibling LAST: its presence marks the payload complete. + self.store + .put(&self.manifest_path(key), PutPayload::from(manifest_bytes)) + .await + .map_err(map_obj)?; + Ok(()) + } + + async fn manifest(&self, key: &str) -> Result { + let bytes = self + .store + .get(&self.manifest_path(key)) + .await + .map_err(map_obj)? + .bytes() + .await + .map_err(map_obj)?; + manifest_from_slice(&bytes) + } + + async fn download(&self, key: &str, dest_dir: &Path) -> Result { + check_dest_available(dest_dir)?; + let parent = dest_dir + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .ok_or_else(|| { + SnapshotError::ArtifactInvalid(format!( + "destination {} has no parent directory", + dest_dir.display() + )) + })?; + + // Sibling manifest first — it is the completeness marker and the value + // we cross-check the tar against. + let sibling = self + .store + .get(&self.manifest_path(key)) + .await + .map_err(map_obj)? + .bytes() + .await + .map_err(map_obj)?; + let manifest = manifest_from_slice(&sibling)?; + + // Stream the tar to a temp file. + let tar_tmp = tempfile::NamedTempFile::new_in(parent)?; + let mut tar_writer = tokio::fs::File::create(tar_tmp.path()) + .await + .map_err(SnapshotError::Io)?; + let mut stream = self + .store + .get(&self.payload_path(key)) + .await + .map_err(map_obj)? + .into_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(map_obj)?; + tar_writer.write_all(&chunk).await.map_err(SnapshotError::Io)?; + } + tar_writer.flush().await.map_err(SnapshotError::Io)?; + drop(tar_writer); + + // Unpack into a stage beside the destination, cross-check the embedded + // manifest against the sibling, and atomically rename into place — all + // blocking work, offloaded. + let dest = dest_dir.to_path_buf(); + let parent = parent.to_path_buf(); + tokio::task::spawn_blocking(move || -> Result<(), SnapshotError> { + let stage = tempfile::Builder::new() + .prefix(".slipstream-download-") + .tempdir_in(&parent)?; + let file = std::fs::File::open(tar_tmp.path())?; + let mut archive = tar::Archive::new(std::io::BufReader::new(file)); + // tar's unpack refuses entries that escape the destination, on top + // of the manifest path validation import performs again. + archive.unpack(stage.path())?; + + let embedded = std::fs::read(stage.path().join(MANIFEST_FILE)).map_err(|_| { + SnapshotError::ArtifactInvalid( + "downloaded artifact tar has no embedded manifest".into(), + ) + })?; + if embedded != sibling.as_ref() { + return Err(SnapshotError::ArtifactInvalid( + "embedded manifest disagrees with the sibling manifest object".into(), + )); + } + + check_dest_available(&dest)?; + if dest.is_dir() { + std::fs::remove_dir(&dest)?; + } + let root = stage.keep(); + std::fs::rename(&root, &dest)?; + Ok(()) + }) + .await + .map_err(|e| SnapshotError::Backend(format!("untar task panicked: {e}")))??; + + Ok(manifest) + } +} + +fn map_obj(e: object_store::Error) -> SnapshotError { + match e { + object_store::Error::NotFound { path, .. } => { + SnapshotError::ArtifactInvalid(format!("remote artifact object not found: {path}")) + } + other => SnapshotError::Backend(format!("object store: {other}")), + } +} + +// --------------------------------------------------------------------------- +// The composed round +// --------------------------------------------------------------------------- + +/// Run one complete at-most-once export round: +/// +/// 1. [`ExportLease::try_acquire`] — `Ok(None)` means another node owns this +/// round; nothing else happens. +/// 2. An [`ExportRequest`] into the live [`watch_applied`](crate::watch_applied) +/// loop (pending batch flushed first; artifact cursor == applied cursor). +/// 3. [`ArtifactTransport::upload`]. +/// 4. [`LeaseGuard::complete`](crate::LeaseGuard::complete) — only after the +/// upload succeeded, so a published completion never lies. +/// 5. Delete the local artifact (transience: artifacts hardlink fold files and +/// pin storage if they linger). +/// +/// On any failure after winning the lease, the artifact is cleaned up and the +/// lease **abandoned** so the fleet can retry promptly instead of waiting out +/// the ttl. `scratch_dir` must exist and should be on the same filesystem as +/// the fold (the export stages beside it; hardlinks degrade to copies across +/// filesystems). +pub async fn run_export_round( + lease: &ExportLease, + ttl: Duration, + exports: &mpsc::Sender, + transport: &dyn ArtifactTransport, + key: &str, + scratch_dir: &Path, +) -> Result, SnapshotError> { + let Some(guard) = lease + .try_acquire(ttl) + .await + .map_err(|e| SnapshotError::Backend(format!("export lease: {e}")))? + else { + return Ok(None); + }; + + // The artifact lives inside a TempDir for the duration of the round — + // dropped on every path out of this function, success or failure. + let round_dir = match tempfile::Builder::new() + .prefix(".slipstream-export-round-") + .tempdir_in(scratch_dir) + { + Ok(d) => d, + Err(e) => { + guard.abandon().await; + return Err(SnapshotError::Io(e)); + } + }; + let artifact_dir = round_dir.path().join("artifact"); + + // Export through the watch loop. + let (reply_tx, reply_rx) = oneshot::channel(); + let request = ExportRequest { + dest_dir: artifact_dir.clone(), + reply: reply_tx, + }; + if exports.send(request).await.is_err() { + guard.abandon().await; + return Err(SnapshotError::Backend( + "watch loop is gone; export request not delivered".into(), + )); + } + let manifest = match reply_rx.await { + Ok(Ok(m)) => m, + Ok(Err(e)) => { + guard.abandon().await; + return Err(e); + } + Err(_) => { + guard.abandon().await; + return Err(SnapshotError::Backend( + "watch loop dropped the export reply".into(), + )); + } + }; + + // Upload; only then publish completion. + if let Err(e) = transport.upload(key, &artifact_dir).await { + guard.abandon().await; + return Err(e); + } + if let Err(e) = guard.complete(&manifest.cursor).await { + // The artifact IS uploaded — the round succeeded. Losing the + // completion record costs observability, not correctness. + warn!(key, error = %e, "export round uploaded but completion record failed"); + } + + drop(round_dir); // enforce artifact transience + Ok(Some(manifest)) +} + +// --------------------------------------------------------------------------- +// Per-backend remote-import conveniences +// --------------------------------------------------------------------------- + +/// Download `key` into a throwaway dir under `scratch_dir`, returning the +/// artifact path and the guard keeping it alive. +async fn download_to_scratch( + transport: &dyn ArtifactTransport, + key: &str, + scratch_dir: &Path, +) -> Result<(tempfile::TempDir, PathBuf), SnapshotError> { + let tmp = tempfile::Builder::new() + .prefix(".slipstream-bootstrap-") + .tempdir_in(scratch_dir)?; + let artifact = tmp.path().join("artifact"); + transport.download(key, &artifact).await?; + Ok((tmp, artifact)) +} + +impl crate::AppendLogSnapshot { + /// Fetch the artifact at `key` and import it as a new fold at `dest_path` + /// (download → full verification → open), resuming from the embedded + /// cursor. The downloaded artifact is deleted afterwards. + pub async fn import_remote( + transport: &dyn ArtifactTransport, + key: &str, + scratch_dir: &Path, + dest_path: &Path, + compact_threshold: u64, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (_guard, artifact) = download_to_scratch(transport, key, scratch_dir).await?; + let dest = dest_path.to_path_buf(); + tokio::task::spawn_blocking(move || Self::import(&artifact, &dest, compact_threshold)) + .await + .map_err(|e| SnapshotError::Backend(format!("import task panicked: {e}")))? + } +} + +#[cfg(feature = "fjall")] +impl crate::FjallSnapshot { + /// Fetch the artifact at `key` and import it as a new fold at `dest_dir` + /// (download → full verification → verify-open → rename), resuming from + /// the embedded cursor. The downloaded artifact is deleted afterwards. + pub async fn import_remote( + transport: &dyn ArtifactTransport, + key: &str, + scratch_dir: &Path, + dest_dir: &Path, + config: crate::FjallConfig, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (_guard, artifact) = download_to_scratch(transport, key, scratch_dir).await?; + let dest = dest_dir.to_path_buf(); + tokio::task::spawn_blocking(move || Self::import(&artifact, &dest, config)) + .await + .map_err(|e| SnapshotError::Backend(format!("import task panicked: {e}")))? + } +} + +#[cfg(feature = "rocksdb")] +impl crate::RocksDbSnapshot { + /// Fetch the artifact at `key` and import it as a new fold at `dest_dir` + /// (download → full verification → verify-open → rename), resuming from + /// the embedded cursor. The downloaded artifact is deleted afterwards. + pub async fn import_remote( + transport: &dyn ArtifactTransport, + key: &str, + scratch_dir: &Path, + dest_dir: &Path, + config: crate::RocksDbConfig, + ) -> Result<(WatchCursor, Self), SnapshotError> { + let (_guard, artifact) = download_to_scratch(transport, key, scratch_dir).await?; + let dest = dest_dir.to_path_buf(); + tokio::task::spawn_blocking(move || Self::import(&artifact, &dest, config)) + .await + .map_err(|e| SnapshotError::Backend(format!("import task panicked: {e}")))? + } +} diff --git a/tests/transport.rs b/tests/transport.rs new file mode 100644 index 0000000..d74c98e --- /dev/null +++ b/tests/transport.rs @@ -0,0 +1,409 @@ +//! Transport-layer tests (feature `transport`), Tier 3: the +//! [`ObjectStoreTransport`] wire format and [`run_export_round`] against +//! `object_store`'s local filesystem backend — no cloud, no servers, except a +//! throwaway `nats-server` where a real lease/watch loop is part of the claim. +#![cfg(feature = "transport")] + +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use slipstream::snapshot::{SnapshotError, SnapshotStore}; +use slipstream::{ + AppendLogSnapshot, ArtifactTransport, BatchConfig, Connection, ExportLease, ExportManifest, + KvEntry, KvUpdate, KvWriter, NatsConnection, NatsConnectionConfig, ObjectStoreTransport, + StoreConfig, VersionToken, WatchCursor, WatchScope, run_export_round, watch_applied, +}; +use tempfile::TempDir; +use tokio::sync::{mpsc, watch}; + +// --- Fixtures ---------------------------------------------------------------- + +fn put(key: &str, value: &[u8], rev: u64) -> KvUpdate { + KvUpdate::Put(KvEntry { + key: key.to_string(), + value: value.to_vec(), + version: VersionToken::from_u64(rev), + }) +} + +/// A local-filesystem transport rooted in a tempdir, plus the dir handle (the +/// "bucket" is inspectable for tamper tests). +fn local_transport() -> (ObjectStoreTransport, TempDir) { + let bucket = TempDir::new().unwrap(); + let store = object_store::local::LocalFileSystem::new_with_prefix(bucket.path()).unwrap(); + ( + ObjectStoreTransport::new(Arc::new(store), "slipstream-artifacts"), + bucket, + ) +} + +/// Export a 3-entry append-log fold and return `(artifact_dir, manifest, dir)`. +fn exported_artifact() -> (std::path::PathBuf, ExportManifest, TempDir) { + let dir = TempDir::new().unwrap(); + let (_r, mut s) = AppendLogSnapshot::open(&dir.path().join("fold.snap"), u64::MAX).unwrap(); + s.apply( + &[put("a", b"1", 1), put("b", b"2", 2), put("c", b"3", 3)], + &WatchCursor::from_u64(3), + ) + .unwrap(); + let artifact = dir.path().join("artifact"); + let manifest = s.export_to(&artifact).unwrap(); + (artifact, manifest, dir) +} + +// --- Wire-format tests --------------------------------------------------------- + +/// upload → manifest-peek → download → import: the full remote round-trip, with +/// the imported fold byte-identical to the source. +#[tokio::test(flavor = "multi_thread")] +async fn upload_manifest_download_import_round_trip() { + let (transport, _bucket) = local_transport(); + let (artifact, manifest, dir) = exported_artifact(); + + transport.upload("edge/us-east/latest", &artifact).await.unwrap(); + + // Manifest peek without a payload download. + let peeked = transport.manifest("edge/us-east/latest").await.unwrap(); + assert_eq!(peeked.cursor, manifest.cursor); + assert_eq!(peeked.backend, "append-log"); + + // Download to a fresh "node" and import. + let downloaded = dir.path().join("downloaded"); + let got = transport + .download("edge/us-east/latest", &downloaded) + .await + .unwrap(); + assert_eq!(got.cursor, manifest.cursor); + + let (cursor, imported) = + AppendLogSnapshot::import(&downloaded, &dir.path().join("imported.snap"), u64::MAX) + .unwrap(); + assert_eq!(cursor.as_u64(), Some(3)); + assert_eq!(imported.get("b").unwrap().unwrap().value, b"2"); +} + +/// `import_remote` composes download + import + cleanup in one call. +#[tokio::test(flavor = "multi_thread")] +async fn import_remote_bootstraps_a_fold() { + let (transport, _bucket) = local_transport(); + let (artifact, _m, dir) = exported_artifact(); + transport.upload("latest", &artifact).await.unwrap(); + + let scratch = dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + let (cursor, imported) = AppendLogSnapshot::import_remote( + &transport, + "latest", + &scratch, + &dir.path().join("bootstrapped.snap"), + u64::MAX, + ) + .await + .unwrap(); + assert_eq!(cursor.as_u64(), Some(3)); + assert_eq!(imported.get("a").unwrap().unwrap().value, b"1"); + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "downloaded artifact cleaned from scratch" + ); +} + +/// A sibling manifest that disagrees with the tar's embedded manifest is +/// rejected at download — the transport is untrusted. +#[tokio::test(flavor = "multi_thread")] +async fn download_rejects_manifest_disagreement() { + let (transport, bucket) = local_transport(); + let (artifact, _m, dir) = exported_artifact(); + transport.upload("latest", &artifact).await.unwrap(); + + // Doctor the sibling manifest object in the "bucket" (cursor 999). + let sibling = bucket + .path() + .join("slipstream-artifacts") + .join("latest.manifest.json"); + let raw = std::fs::read(&sibling).unwrap(); + let mut json: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + json["cursor_hex"] = "00000000000003e7".into(); + std::fs::write(&sibling, serde_json::to_vec(&json).unwrap()).unwrap(); + + let dest = dir.path().join("downloaded"); + match transport.download("latest", &dest).await { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("disagrees"), "{msg}"); + } + other => panic!("expected ArtifactInvalid, got {other:?}"), + } + assert!(!dest.exists(), "nothing lands at the destination"); +} + +/// A missing remote object is an ArtifactInvalid, not an opaque backend error. +#[tokio::test(flavor = "multi_thread")] +async fn missing_remote_artifact_is_artifact_invalid() { + let (transport, _bucket) = local_transport(); + match transport.manifest("never-uploaded").await { + Err(SnapshotError::ArtifactInvalid(msg)) => assert!(msg.contains("not found"), "{msg}"), + other => panic!("expected ArtifactInvalid, got {other:?}"), + } +} + +// --- run_export_round ------------------------------------------------------------ +// These need a real lease (NATS KV) and a live watch_applied loop. + +struct TestNats { + child: Child, + url: String, + _store_dir: TempDir, +} + +impl TestNats { + async fn start() -> TestNats { + let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); + let port = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let store_dir = tempfile::tempdir().unwrap(); + let child = Command::new(&bin) + .args([ + "--jetstream", + "--addr", + "127.0.0.1", + "--port", + &port.to_string(), + "--store_dir", + store_dir.path().to_str().unwrap(), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap_or_else(|e| panic!("failed to spawn `{bin}`: {e}. Run `mise install`.")); + let url = format!("nats://127.0.0.1:{port}"); + for _ in 0..100 { + if async_nats::connect(&url).await.is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + TestNats { + child, + url, + _store_dir: store_dir, + } + } +} + +impl Drop for TestNats { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Everything a live export round needs: a fold being driven by watch_applied +/// over a real NATS bucket, an export channel into it, and a lease store. +struct LiveRound { + _nats: TestNats, + _conn: NatsConnection, + writer: Arc, + lease_store: Arc, + exports: mpsc::Sender, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, + dir: TempDir, +} + +async fn live_round() -> LiveRound { + let nats = TestNats::start().await; + let conn = NatsConnection::new(NatsConnectionConfig { + url: nats.url.clone(), + creds: None, + creds_file: None, + }); + conn.connect().await.unwrap(); + let bucket = conn + .store_with_config(StoreConfig { + name: "routes".into(), + max_bytes: Some(8 * 1024 * 1024), + ..Default::default() + }) + .await + .unwrap(); + let lease_store = conn + .store_with_config(StoreConfig { + name: "leases".into(), + max_bytes: Some(1024 * 1024), + ..Default::default() + }) + .await + .unwrap(); + + let writer = bucket.writer().unwrap(); + let watcher = bucket.watcher().unwrap(); + + let dir = TempDir::new().unwrap(); + let (_r, fold) = AppendLogSnapshot::open(&dir.path().join("fold.snap"), u64::MAX).unwrap(); + let (ex_tx, ex_rx) = mpsc::channel(1); + let (sd_tx, sd_rx) = watch::channel(false); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + Some(fold), + Some(ex_rx), + BatchConfig::default(), + |u: &KvUpdate| match u { + KvUpdate::Put(e) => Some(e.key.clone()), + _ => None, + }, + |_batch: Vec| {}, + |_| {}, + sd_rx, + )); + + LiveRound { + _nats: nats, + _conn: conn, + writer, + lease_store, + exports: ex_tx, + shutdown: sd_tx, + task, + dir, + } +} + +/// Wait until the fold has applied at least one update (watch attached). +async fn settle_watch(round: &LiveRound) { + round.writer.put("seed.key", b"seed").await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; +} + +/// The composed happy path: lease won → export through the live loop → upload +/// → completion published → local artifact gone. And a concurrent second +/// caller loses the round cleanly. +#[tokio::test(flavor = "multi_thread")] +async fn run_export_round_uploads_once_and_cleans_up() { + let round = live_round().await; + settle_watch(&round).await; + + let (transport, _bucket) = local_transport(); + let scratch = round.dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + + let lease_a = ExportLease::new(round.lease_store.as_ref(), "round", "node-a").unwrap(); + let lease_b = ExportLease::new(round.lease_store.as_ref(), "round", "node-b").unwrap(); + + let (a, b) = tokio::join!( + run_export_round( + &lease_a, + Duration::from_secs(60), + &round.exports, + &transport, + "latest", + &scratch, + ), + run_export_round( + &lease_b, + Duration::from_secs(60), + &round.exports, + &transport, + "latest", + &scratch, + ), + ); + let outcomes = [a.unwrap(), b.unwrap()]; + let winners: Vec<_> = outcomes.iter().filter(|o| o.is_some()).collect(); + assert_eq!(winners.len(), 1, "exactly one round runs"); + let manifest = winners[0].as_ref().unwrap(); + assert!(!manifest.cursor.is_none(), "exported a live cursor"); + + // Uploaded and fetchable; completion published on the lease key. + let peeked = transport.manifest("latest").await.unwrap(); + assert_eq!(peeked.cursor, manifest.cursor); + let record = lease_a.current().await.unwrap().unwrap(); + assert!( + record.completed_cursor_hex.is_some(), + "completion is fleet-visible" + ); + + // Transience: nothing left in scratch. + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "local artifact deleted after upload" + ); + + round.shutdown.send(true).unwrap(); + round.task.await.unwrap().unwrap(); +} + +/// A transport that always fails its upload. +struct FailingTransport; + +#[async_trait] +impl ArtifactTransport for FailingTransport { + async fn upload(&self, _key: &str, _dir: &Path) -> Result<(), SnapshotError> { + Err(SnapshotError::Backend("injected upload failure".into())) + } + async fn manifest(&self, _key: &str) -> Result { + Err(SnapshotError::Backend("unused".into())) + } + async fn download(&self, _key: &str, _dest: &Path) -> Result { + Err(SnapshotError::Backend("unused".into())) + } +} + +/// Upload failure: the error propagates, no completion is published, the local +/// artifact is cleaned up, and — because the lease is abandoned — the next +/// round can be won immediately instead of waiting out the ttl. +#[tokio::test(flavor = "multi_thread")] +async fn run_export_round_upload_failure_abandons_lease() { + let round = live_round().await; + settle_watch(&round).await; + + let scratch = round.dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + let lease = ExportLease::new(round.lease_store.as_ref(), "round", "node-a").unwrap(); + + let err = run_export_round( + &lease, + Duration::from_secs(600), // long ttl: only abandon makes retry possible + &round.exports, + &FailingTransport, + "latest", + &scratch, + ) + .await + .expect_err("upload failure propagates"); + assert!(matches!(err, SnapshotError::Backend(_))); + + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "artifact cleaned up after the failed round" + ); + + // The lease was abandoned: a fresh round wins immediately with a working + // transport, long before the 600s ttl. + let (transport, _bucket) = local_transport(); + let retry = run_export_round( + &lease, + Duration::from_secs(60), + &round.exports, + &transport, + "latest", + &scratch, + ) + .await + .unwrap(); + assert!(retry.is_some(), "abandoned lease frees the next round"); + + round.shutdown.send(true).unwrap(); + round.task.await.unwrap().unwrap(); +} From eb71a7cdecf3251b1da6307a5ae3fa0d8bd0353f Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:38:57 -0700 Subject: [PATCH 09/12] test(bootstrap+s3): live export/import/resume with delta-only assertions; MinIO tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/bootstrap.rs (fjall + rocksdb): node A folds a real NATS bucket under churn, exports through the watch loop's request channel, node B imports the artifact and resumes from the embedded cursor. The load-bearing assertions are delta-only: B is DELIVERED exactly the post-export tail (counted), the first delivered revision is cursor+1 (no overlap, no gap), and a delete in the tail reaches B — convergence alone would mask a full replay. tests/transport_s3.rs: the same full loop through run_export_round against a real S3 API — a throwaway MinIO per test (mise-installed binary + mc for bucket creation, no Docker; same pattern as TestNats) — plus a ~25 MiB incompressible artifact exercising real multipart upload. tests/common/ holds the shared TestNats/TestMinio guards; mise.toml pins minio (asdf backend; ubi rejects MinIO's extension-less release assets) and mc (aqua). Co-Authored-By: Claude Fable 5 --- mise.toml | 6 + tests/bootstrap.rs | 330 +++++++++++++++++++++++++++++++++++++++ tests/common/mod.rs | 191 +++++++++++++++++++++++ tests/transport.rs | 84 ++++------ tests/transport_s3.rs | 347 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 902 insertions(+), 56 deletions(-) create mode 100644 tests/bootstrap.rs create mode 100644 tests/common/mod.rs create mode 100644 tests/transport_s3.rs diff --git a/mise.toml b/mise.toml index 9dbe040..8e183f9 100644 --- a/mise.toml +++ b/mise.toml @@ -10,6 +10,12 @@ rust = { version = "1.92", components = "clippy,rustfmt" } yamlfmt = "0.12.1" dprint = "0.50.2" "ubi:nats-io/nats-server" = "2.14.1" +# Real-S3 transport tests (tests/transport_s3.rs): single static binaries, no +# Docker — same pattern as nats-server. mc creates the test bucket. (ubi can't +# install minio: its releases ship extension-less raw binaries that ubi's +# asset filter rejects, hence the asdf-backend plugin.) +"asdf:mise-plugins/mise-minio" = "2025-09-07T16-13-09Z" +"aqua:minio/mc" = "RELEASE.2025-08-13T08-35-41Z" [tasks.format] description = "Auto-format all code using dprint (Go, TypeScript, YAML, JSON, Markdown)" diff --git a/tests/bootstrap.rs b/tests/bootstrap.rs new file mode 100644 index 0000000..e54bd3d --- /dev/null +++ b/tests/bootstrap.rs @@ -0,0 +1,330 @@ +//! Tier-2 bootstrap tests: export a fold from a LIVE `watch_applied` loop +//! under churn, import it as a second node, resume the watch from the +//! embedded cursor — and prove **delta-only resume**: the bootstrapped node +//! receives exactly the post-export tail, never a replay of the full history. +//! +//! Convergence alone would mask a full replay (the end state is identical +//! either way); the delivery COUNT is the assertion that carries the scaling +//! property — bootstrap cost = artifact + tail, not a rescan. +//! +//! Generic over the on-disk backends; instantiated for fjall and RocksDB. +#![cfg(any(feature = "fjall", feature = "rocksdb"))] + +mod common; + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use common::TestNats; +use slipstream::snapshot::{SnapshotError, SnapshotStore}; +use slipstream::{ + BatchConfig, Connection, ExportRequest, KvStore, KvUpdate, NatsConnection, + NatsConnectionConfig, StoreConfig, WatchCursor, WatchScope, watch_applied, +}; +use tempfile::TempDir; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio::time::timeout; + +async fn open_bucket(nats: &TestNats) -> (NatsConnection, Arc) { + let conn = NatsConnection::new(NatsConnectionConfig { + url: nats.url.clone(), + creds: None, + creds_file: None, + }); + conn.connect().await.expect("connect"); + let store = conn + .store_with_config(StoreConfig { + name: "routes".into(), + max_bytes: Some(8 * 1024 * 1024), + ..Default::default() + }) + .await + .expect("open bucket"); + (conn, store) +} + +/// Spawn a `watch_applied` over `bucket` folding into `fold`, with an export +/// channel and a watch on the applied cursor. +struct Node { + exports: mpsc::Sender, + applied: Arc, + delivered: Arc, + min_rev: Arc, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, +} + +fn spawn_node( + bucket: &Arc, + fold: S, + resume: Option, +) -> Node { + let watcher = bucket.watcher().expect("bucket watcher"); + let (ex_tx, ex_rx) = mpsc::channel(1); + let (sd_tx, sd_rx) = watch::channel(false); + let applied = Arc::new(AtomicU64::new(0)); + let delivered = Arc::new(AtomicU64::new(0)); + let min_rev = Arc::new(AtomicU64::new(u64::MAX)); + + let applied_w = Arc::clone(&applied); + let delivered_w = Arc::clone(&delivered); + let min_rev_w = Arc::clone(&min_rev); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + resume, + Some(fold), + Some(ex_rx), + BatchConfig::default(), + move |u: &KvUpdate| { + // Count every DELIVERED update and track the lowest revision — + // the delta-only assertions read these. + delivered_w.fetch_add(1, Ordering::SeqCst); + if let Some(rev) = u.version().as_u64() { + min_rev_w.fetch_min(rev, Ordering::SeqCst); + } + Some(()) + }, + move |_batch: Vec<()>| {}, + move |cur: WatchCursor| { + applied_w.store(cur.as_u64().unwrap_or(0), Ordering::SeqCst); + }, + sd_rx, + )); + + Node { + exports: ex_tx, + applied, + delivered, + min_rev, + shutdown: sd_tx, + task, + } +} + +async fn wait_applied(node: &Node, at_least: u64) { + timeout(Duration::from_secs(10), async { + loop { + if node.applied.load(Ordering::SeqCst) >= at_least { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "node never applied rev {at_least} (at {})", + node.applied.load(Ordering::SeqCst) + ) + }); +} + +/// The full bootstrap story for one backend: +/// +/// 1. Node A folds the bucket live; N updates land. +/// 2. Export through A's request channel → artifact cursor == applied cursor. +/// 3. M more updates land (churn after the export). +/// 4. Node B imports the artifact and resumes from the embedded cursor. +/// 5. **Delta-only**: B was delivered exactly M updates, the first at +/// cursor+1 — no overlap, no gap, no full replay. +/// 6. B's fold state equals the bucket (the truth). +async fn live_export_bootstrap_delta_only( + open: impl Fn(&Path) -> (WatchCursor, S), + import: fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>, +) where + S: SnapshotStore + Send + 'static, +{ + let nats = TestNats::start().await; + let (_conn, bucket) = open_bucket(&nats).await; + let writer = bucket.writer().expect("writer"); + let dir = TempDir::new().unwrap(); + + // Node A, live. + let (_r, fold_a) = open(&dir.path().join("node-a")); + let node_a = spawn_node(&bucket, fold_a, None); + + // Deterministic attach: write until A applies something (KV watches + // deliver new updates only; writes before the consumer attaches are + // missed, so we probe rather than sleep). + let attach_rev = timeout(Duration::from_secs(10), async { + loop { + let v = writer.put("route.seed", b"seed").await.expect("seed"); + tokio::time::sleep(Duration::from_millis(50)).await; + if node_a.applied.load(Ordering::SeqCst) > 0 { + return v.as_u64().expect("nats rev"); + } + } + }) + .await + .expect("node A watch never attached"); + + // Pre-export history: N real updates. + let n = 12u64; + let mut last_rev = attach_rev; + for i in 0..n { + last_rev = writer + .put(&format!("route.pre.{i}"), format!("pre-{i}").as_bytes()) + .await + .expect("put") + .as_u64() + .expect("rev"); + } + wait_applied(&node_a, last_rev).await; + + // Export from the live loop. + let artifact = dir.path().join("artifact"); + let (reply_tx, reply_rx) = oneshot::channel(); + node_a + .exports + .send(ExportRequest { + dest_dir: artifact.clone(), + reply: reply_tx, + }) + .await + .expect("send export request"); + let manifest = reply_rx.await.expect("reply").expect("export succeeds"); + let export_rev = manifest.cursor.as_u64().expect("cursor rev"); + assert!(export_rev >= last_rev, "artifact covers the pre-export history"); + + // Post-export churn: exactly M updates (including a delete — tombstones + // must ride the tail too). + let m = 7u64; + let mut final_rev = export_rev; + for i in 0..m - 1 { + final_rev = writer + .put(&format!("route.post.{i}"), format!("post-{i}").as_bytes()) + .await + .expect("put") + .as_u64() + .expect("rev"); + } + assert!(writer.delete("route.pre.0").await.expect("delete")); + final_rev += 1; // the delete's revision + + wait_applied(&node_a, final_rev).await; + + // Node B: import the artifact, resume from its cursor. + let dest_b = dir.path().join("node-b"); + let (cursor_b, fold_b) = import(&artifact, &dest_b).expect("import"); + assert_eq!( + cursor_b, manifest.cursor, + "imported cursor is the manifest cursor" + ); + let node_b = spawn_node(&bucket, fold_b, Some(cursor_b.clone())); + wait_applied(&node_b, final_rev).await; + + // THE delta-only assertions. + assert_eq!( + node_b.delivered.load(Ordering::SeqCst), + m, + "bootstrapped node was delivered exactly the post-export tail, not a replay" + ); + assert_eq!( + node_b.min_rev.load(Ordering::SeqCst), + export_rev + 1, + "the tail starts at cursor+1 — no overlap, no gap" + ); + + // Shut both down; B's fold must equal the bucket. + node_a.shutdown.send(true).unwrap(); + node_a.task.await.unwrap().unwrap(); + node_b.shutdown.send(true).unwrap(); + node_b.task.await.unwrap().unwrap(); + + let (final_cursor, fold_b) = open(&dest_b); + assert_eq!(final_cursor.as_u64(), Some(final_rev)); + let mut fold_state: Vec<(String, Vec)> = fold_b + .range("route.") + .expect("range") + .into_iter() + .map(|e| (e.key, e.value)) + .collect(); + fold_state.sort(); + let mut bucket_state: Vec<(String, Vec)> = bucket + .reader() + .scan("route.") + .await + .expect("scan") + .into_iter() + .map(|e| (e.key, e.value)) + .collect(); + bucket_state.sort(); + assert_eq!( + fold_state, bucket_state, + "bootstrapped fold equals the bucket" + ); + assert!( + !fold_state.iter().any(|(k, _)| k == "route.pre.0"), + "the tail's delete reached the bootstrapped fold" + ); +} + +#[cfg(feature = "fjall")] +mod fjall_bootstrap { + use super::*; + use slipstream::{FjallConfig, FjallSnapshot}; + + #[tokio::test(flavor = "multi_thread")] + async fn fjall_live_export_bootstrap_delta_only() { + live_export_bootstrap_delta_only( + |path| { + FjallSnapshot::open( + path, + FjallConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + .expect("open fjall") + }, + |artifact, dest| { + FjallSnapshot::import( + artifact, + dest, + FjallConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + }, + ) + .await; + } +} + +#[cfg(feature = "rocksdb")] +mod rocksdb_bootstrap { + use super::*; + use slipstream::{RocksDbConfig, RocksDbSnapshot}; + + #[tokio::test(flavor = "multi_thread")] + async fn rocksdb_live_export_bootstrap_delta_only() { + live_export_bootstrap_delta_only( + |path| { + RocksDbSnapshot::open( + path, + RocksDbConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + .expect("open rocksdb") + }, + |artifact, dest| { + RocksDbSnapshot::import( + artifact, + dest, + RocksDbConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + }, + ) + .await; + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..a81aafb --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,191 @@ +//! Shared integration-test infrastructure: throwaway `nats-server` and +//! `minio` instances, one per test, killed on drop. +//! +//! All binaries come from mise (`mise install`); env overrides +//! `NATS_SERVER_BIN` / `MINIO_BIN` / `MC_BIN` point at explicit paths when +//! running outside an activated mise shell. No Docker anywhere. +#![allow(dead_code)] // each test crate uses a subset of this harness + +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use tempfile::TempDir; + +pub fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("read local addr") + .port() +} + +// --- NATS ---------------------------------------------------------------------- + +/// A running `nats-server` with JetStream enabled. Killed on drop. +pub struct TestNats { + child: Child, + pub url: String, + _store_dir: TempDir, +} + +impl TestNats { + pub async fn start() -> TestNats { + let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); + let port = free_port(); + let store_dir = tempfile::tempdir().expect("create jetstream store dir"); + let child = Command::new(&bin) + .args([ + "--jetstream", + "--addr", + "127.0.0.1", + "--port", + &port.to_string(), + "--store_dir", + store_dir.path().to_str().expect("utf-8 store path"), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `{bin}`: {e}. Is nats-server installed? \ + Run `mise install` or set NATS_SERVER_BIN." + ) + }); + let url = format!("nats://127.0.0.1:{port}"); + for _ in 0..100 { + if async_nats::connect(&url).await.is_ok() { + return TestNats { + child, + url, + _store_dir: store_dir, + }; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("nats-server at {url} never became ready"); + } +} + +impl Drop for TestNats { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +// --- MinIO --------------------------------------------------------------------- + +pub const MINIO_USER: &str = "minioadmin"; +pub const MINIO_PASSWORD: &str = "minioadmin"; +pub const MINIO_BUCKET: &str = "test-bucket"; + +/// A running `minio` with [`MINIO_BUCKET`] pre-created via `mc`. Killed on +/// drop. The `mc mb` retry loop doubles as the readiness probe. +pub struct TestMinio { + child: Child, + pub endpoint: String, + _data_dir: TempDir, +} + +impl TestMinio { + pub async fn start() -> TestMinio { + let minio_bin = std::env::var("MINIO_BIN").unwrap_or_else(|_| "minio".to_string()); + let mc_bin = std::env::var("MC_BIN").unwrap_or_else(|_| "mc".to_string()); + let api_port = free_port(); + let console_port = free_port(); + let data_dir = tempfile::tempdir().expect("create minio data dir"); + + let child = Command::new(&minio_bin) + .args([ + "server", + data_dir.path().to_str().expect("utf-8 data path"), + "--address", + &format!("127.0.0.1:{api_port}"), + "--console-address", + &format!("127.0.0.1:{console_port}"), + ]) + .env("MINIO_ROOT_USER", MINIO_USER) + .env("MINIO_ROOT_PASSWORD", MINIO_PASSWORD) + .env("MINIO_BROWSER", "off") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `{minio_bin}`: {e}. Is minio installed? \ + Run `mise install` or set MINIO_BIN." + ) + }); + + let endpoint = format!("http://127.0.0.1:{api_port}"); + + // Create the test bucket with mc, retrying until the server is up. + // A throwaway --config-dir keeps ~/.mc untouched. + let mc_cfg = tempfile::tempdir().expect("create mc config dir"); + let cfg = mc_cfg.path().to_str().expect("utf-8 mc config path"); + let mut ready = false; + for _ in 0..100 { + let alias = Command::new(&mc_bin) + .args([ + "--config-dir", + cfg, + "alias", + "set", + "t", + &endpoint, + MINIO_USER, + MINIO_PASSWORD, + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap_or_else(|e| panic!("failed to spawn `{mc_bin}`: {e}. Run `mise install`.")); + if alias.success() { + let mb = Command::new(&mc_bin) + .args([ + "--config-dir", + cfg, + "mb", + "--ignore-existing", + &format!("t/{MINIO_BUCKET}"), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("run mc mb"); + if mb.success() { + ready = true; + break; + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + assert!(ready, "minio at {endpoint} never became ready (mc mb failed)"); + + TestMinio { + child, + endpoint, + _data_dir: data_dir, + } + } + + /// Builder options for `ObjectStoreTransport::from_url_opts` pointing at + /// this instance (path-style, http, root credentials). + pub fn s3_options(&self) -> Vec<(&'static str, String)> { + vec![ + ("aws_access_key_id", MINIO_USER.to_string()), + ("aws_secret_access_key", MINIO_PASSWORD.to_string()), + ("aws_endpoint", self.endpoint.clone()), + ("aws_allow_http", "true".to_string()), + ("aws_region", "us-east-1".to_string()), + ] + } +} + +impl Drop for TestMinio { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/tests/transport.rs b/tests/transport.rs index d74c98e..c301e4f 100644 --- a/tests/transport.rs +++ b/tests/transport.rs @@ -4,11 +4,14 @@ //! throwaway `nats-server` where a real lease/watch loop is part of the claim. #![cfg(feature = "transport")] +mod common; + use std::path::Path; -use std::process::{Child, Command, Stdio}; use std::sync::Arc; use std::time::Duration; +use common::TestNats; + use async_trait::async_trait; use slipstream::snapshot::{SnapshotError, SnapshotStore}; use slipstream::{ @@ -153,57 +156,6 @@ async fn missing_remote_artifact_is_artifact_invalid() { // --- run_export_round ------------------------------------------------------------ // These need a real lease (NATS KV) and a live watch_applied loop. -struct TestNats { - child: Child, - url: String, - _store_dir: TempDir, -} - -impl TestNats { - async fn start() -> TestNats { - let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); - let port = std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); - let store_dir = tempfile::tempdir().unwrap(); - let child = Command::new(&bin) - .args([ - "--jetstream", - "--addr", - "127.0.0.1", - "--port", - &port.to_string(), - "--store_dir", - store_dir.path().to_str().unwrap(), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap_or_else(|e| panic!("failed to spawn `{bin}`: {e}. Run `mise install`.")); - let url = format!("nats://127.0.0.1:{port}"); - for _ in 0..100 { - if async_nats::connect(&url).await.is_ok() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - TestNats { - child, - url, - _store_dir: store_dir, - } - } -} - -impl Drop for TestNats { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - /// Everything a live export round needs: a fold being driven by watch_applied /// over a real NATS bucket, an export channel into it, and a lease store. struct LiveRound { @@ -212,6 +164,7 @@ struct LiveRound { writer: Arc, lease_store: Arc, exports: mpsc::Sender, + applied: Arc, shutdown: watch::Sender, task: tokio::task::JoinHandle>, dir: TempDir, @@ -249,6 +202,8 @@ async fn live_round() -> LiveRound { let (_r, fold) = AppendLogSnapshot::open(&dir.path().join("fold.snap"), u64::MAX).unwrap(); let (ex_tx, ex_rx) = mpsc::channel(1); let (sd_tx, sd_rx) = watch::channel(false); + let applied = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let applied_w = Arc::clone(&applied); let task = tokio::spawn(watch_applied( watcher, @@ -262,7 +217,12 @@ async fn live_round() -> LiveRound { _ => None, }, |_batch: Vec| {}, - |_| {}, + move |cur: WatchCursor| { + applied_w.store( + cur.as_u64().unwrap_or(0), + std::sync::atomic::Ordering::SeqCst, + ); + }, sd_rx, )); @@ -272,16 +232,28 @@ async fn live_round() -> LiveRound { writer, lease_store, exports: ex_tx, + applied, shutdown: sd_tx, task, dir, } } -/// Wait until the fold has applied at least one update (watch attached). +/// Deterministically wait until the fold has applied an update: KV watches +/// deliver new updates only, so writes that land before the consumer attaches +/// are missed — probe with repeated puts until the applied cursor moves. async fn settle_watch(round: &LiveRound) { - round.writer.put("seed.key", b"seed").await.unwrap(); - tokio::time::sleep(Duration::from_millis(300)).await; + tokio::time::timeout(Duration::from_secs(10), async { + loop { + round.writer.put("seed.key", b"seed").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + if round.applied.load(std::sync::atomic::Ordering::SeqCst) > 0 { + return; + } + } + }) + .await + .expect("watch never attached"); } /// The composed happy path: lease won → export through the live loop → upload diff --git a/tests/transport_s3.rs b/tests/transport_s3.rs new file mode 100644 index 0000000..d735b60 --- /dev/null +++ b/tests/transport_s3.rs @@ -0,0 +1,347 @@ +//! Tier-4 tests: the full bootstrap loop against a REAL S3 API — a throwaway +//! MinIO per test (mise-installed binary, no Docker). Node A exports through a +//! live `watch_applied` via `run_export_round` (lease + upload + completion); +//! node B downloads, imports, and resumes — with the delta-only assertion. +//! Plus the multipart upload path (artifact >> part size). +#![cfg(all(feature = "transport", any(feature = "fjall", feature = "rocksdb")))] + +mod common; + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use common::{MINIO_BUCKET, TestMinio, TestNats}; +use slipstream::snapshot::{SnapshotError, SnapshotStore}; +use slipstream::{ + ArtifactTransport, BatchConfig, Connection, ExportLease, ExportRequest, KvStore, KvUpdate, + NatsConnection, NatsConnectionConfig, ObjectStoreTransport, StoreConfig, WatchCursor, + WatchScope, run_export_round, watch_applied, +}; +use tempfile::TempDir; +use tokio::sync::{mpsc, watch}; +use tokio::time::timeout; + +fn minio_transport(minio: &TestMinio) -> ObjectStoreTransport { + ObjectStoreTransport::from_url_opts( + &format!("s3://{MINIO_BUCKET}/slipstream"), + minio.s3_options(), + ) + .expect("build s3 transport against minio") +} + +async fn open_buckets(nats: &TestNats) -> (NatsConnection, Arc, Arc) { + let conn = NatsConnection::new(NatsConnectionConfig { + url: nats.url.clone(), + creds: None, + creds_file: None, + }); + conn.connect().await.expect("connect"); + let routes = conn + .store_with_config(StoreConfig { + name: "routes".into(), + max_bytes: Some(8 * 1024 * 1024), + ..Default::default() + }) + .await + .expect("routes bucket"); + let leases = conn + .store_with_config(StoreConfig { + name: "leases".into(), + max_bytes: Some(1024 * 1024), + ..Default::default() + }) + .await + .expect("leases bucket"); + (conn, routes, leases) +} + +struct Node { + exports: mpsc::Sender, + applied: Arc, + delivered: Arc, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, +} + +fn spawn_node( + bucket: &Arc, + fold: S, + resume: Option, +) -> Node { + let watcher = bucket.watcher().expect("watcher"); + let (ex_tx, ex_rx) = mpsc::channel(1); + let (sd_tx, sd_rx) = watch::channel(false); + let applied = Arc::new(AtomicU64::new(0)); + let delivered = Arc::new(AtomicU64::new(0)); + let applied_w = Arc::clone(&applied); + let delivered_w = Arc::clone(&delivered); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + resume, + Some(fold), + Some(ex_rx), + BatchConfig::default(), + move |_u: &KvUpdate| { + delivered_w.fetch_add(1, Ordering::SeqCst); + Some(()) + }, + move |_batch: Vec<()>| {}, + move |cur: WatchCursor| applied_w.store(cur.as_u64().unwrap_or(0), Ordering::SeqCst), + sd_rx, + )); + + Node { + exports: ex_tx, + applied, + delivered, + shutdown: sd_tx, + task, + } +} + +async fn wait_applied(node: &Node, at_least: u64) { + timeout(Duration::from_secs(15), async { + while node.applied.load(Ordering::SeqCst) < at_least { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "node never applied rev {at_least} (at {})", + node.applied.load(Ordering::SeqCst) + ) + }); +} + +/// Full loop, generic over the on-disk backend: live node A → run_export_round +/// (lease won, exported through the watch loop, uploaded to MinIO, completion +/// published, artifact deleted) → node B downloads + imports + resumes → +/// delta-only delivery → state equals the bucket. +async fn full_bootstrap_loop( + open: impl Fn(&Path) -> (WatchCursor, S), + import: fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>, +) where + S: SnapshotStore + Send + 'static, +{ + let (nats, minio) = tokio::join!(TestNats::start(), TestMinio::start()); + let (_conn, routes, leases) = open_buckets(&nats).await; + let writer = routes.writer().expect("writer"); + let transport = minio_transport(&minio); + let dir = TempDir::new().unwrap(); + let scratch = dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + + // Node A live; deterministic watch attach. + let (_r, fold_a) = open(&dir.path().join("node-a")); + let node_a = spawn_node(&routes, fold_a, None); + timeout(Duration::from_secs(10), async { + loop { + writer.put("route.seed", b"seed").await.expect("seed"); + tokio::time::sleep(Duration::from_millis(50)).await; + if node_a.applied.load(Ordering::SeqCst) > 0 { + return; + } + } + }) + .await + .expect("node A watch never attached"); + + // History, then the round. + let mut last_rev = 0; + for i in 0..10u64 { + last_rev = writer + .put(&format!("route.pre.{i}"), format!("pre-{i}").as_bytes()) + .await + .expect("put") + .as_u64() + .expect("rev"); + } + wait_applied(&node_a, last_rev).await; + + let lease = ExportLease::new(leases.as_ref(), "round.routes", "node-a").expect("lease"); + let manifest = run_export_round( + &lease, + Duration::from_secs(120), + &node_a.exports, + &transport, + "edge/us-east/latest", + &scratch, + ) + .await + .expect("round") + .expect("this node wins the only round"); + let export_rev = manifest.cursor.as_u64().expect("rev"); + assert!(export_rev >= last_rev); + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "artifact transience enforced" + ); + + // Post-export churn. + let m = 5u64; + let mut final_rev = export_rev; + for i in 0..m { + final_rev = writer + .put(&format!("route.post.{i}"), format!("post-{i}").as_bytes()) + .await + .expect("put") + .as_u64() + .expect("rev"); + } + wait_applied(&node_a, final_rev).await; + + // Node B: manifest peek, download, import, resume. + let peeked = transport.manifest("edge/us-east/latest").await.expect("peek"); + assert_eq!(peeked.cursor, manifest.cursor); + + let downloaded = dir.path().join("downloaded-artifact"); + transport + .download("edge/us-east/latest", &downloaded) + .await + .expect("download"); + let dest_b = dir.path().join("node-b"); + let (cursor_b, fold_b) = import(&downloaded, &dest_b).expect("import"); + assert_eq!(cursor_b, manifest.cursor); + + let node_b = spawn_node(&routes, fold_b, Some(cursor_b)); + wait_applied(&node_b, final_rev).await; + assert_eq!( + node_b.delivered.load(Ordering::SeqCst), + m, + "bootstrap cost = artifact download + tail replay, never a rescan" + ); + + // Converged with the truth. + node_a.shutdown.send(true).unwrap(); + node_a.task.await.unwrap().unwrap(); + node_b.shutdown.send(true).unwrap(); + node_b.task.await.unwrap().unwrap(); + + let (_c, fold_b) = open(&dest_b); + let mut fold_state: Vec<(String, Vec)> = fold_b + .range("route.") + .expect("range") + .into_iter() + .map(|e| (e.key, e.value)) + .collect(); + fold_state.sort(); + let mut bucket_state: Vec<(String, Vec)> = routes + .reader() + .scan("route.") + .await + .expect("scan") + .into_iter() + .map(|e| (e.key, e.value)) + .collect(); + bucket_state.sort(); + assert_eq!(fold_state, bucket_state, "bootstrapped fold equals the bucket"); +} + +#[cfg(feature = "fjall")] +#[tokio::test(flavor = "multi_thread")] +async fn minio_full_bootstrap_loop_fjall() { + use slipstream::{FjallConfig, FjallSnapshot}; + let cfg = FjallConfig { + sync: false, + cache_size_bytes: 64 << 20, + }; + full_bootstrap_loop( + move |path| FjallSnapshot::open(path, cfg).expect("open fjall"), + |artifact, dest| { + FjallSnapshot::import( + artifact, + dest, + FjallConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + }, + ) + .await; +} + +#[cfg(feature = "rocksdb")] +#[tokio::test(flavor = "multi_thread")] +async fn minio_full_bootstrap_loop_rocksdb() { + use slipstream::{RocksDbConfig, RocksDbSnapshot}; + let cfg = RocksDbConfig { + sync: false, + cache_size_bytes: 64 << 20, + }; + full_bootstrap_loop( + move |path| RocksDbSnapshot::open(path, cfg).expect("open rocksdb"), + |artifact, dest| { + RocksDbSnapshot::import( + artifact, + dest, + RocksDbConfig { + sync: false, + cache_size_bytes: 64 << 20, + }, + ) + }, + ) + .await; +} + +/// Multipart path: an artifact several times the 8 MiB part size streams up as +/// real multipart against the S3 API and round-trips intact. (Values are +/// random-ish so tar size ≈ data size; ~2,500 × 10 KiB ≈ 25 MiB ≈ 4 parts.) +#[tokio::test(flavor = "multi_thread")] +async fn minio_multipart_upload_round_trips() { + use slipstream::AppendLogSnapshot; + + let minio = TestMinio::start().await; + let transport = minio_transport(&minio); + let dir = TempDir::new().unwrap(); + + let (_r, mut fold) = AppendLogSnapshot::open(&dir.path().join("big.snap"), u64::MAX).unwrap(); + let mut batch = Vec::with_capacity(2500); + for i in 0..2500u64 { + // Pseudo-random bytes (LCG) so neither tar nor S3 sees compressible runs. + let mut v = vec![0u8; 10 * 1024]; + let mut x = i.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + for b in &mut v { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + *b = (x >> 33) as u8; + } + batch.push(KvUpdate::Put(slipstream::KvEntry { + key: format!("route.{i:06}"), + value: v, + version: slipstream::VersionToken::from_u64(i + 1), + })); + } + fold.apply(&batch, &WatchCursor::from_u64(2500)).unwrap(); + + let artifact = dir.path().join("artifact"); + let manifest = fold.export_to(&artifact).unwrap(); + let payload_bytes: u64 = manifest.files.iter().map(|f| f.size).sum(); + assert!( + payload_bytes > 20 * 1024 * 1024, + "artifact must exceed several part sizes (got {payload_bytes} bytes)" + ); + + transport.upload("big/latest", &artifact).await.expect("multipart upload"); + + let downloaded = dir.path().join("downloaded"); + transport + .download("big/latest", &downloaded) + .await + .expect("download"); + let (cursor, imported) = + AppendLogSnapshot::import(&downloaded, &dir.path().join("imported.snap"), u64::MAX) + .expect("import verifies every hash"); + assert_eq!(cursor.as_u64(), Some(2500)); + assert_eq!( + imported.range("route.").unwrap().len(), + 2500, + "all entries survive the multipart round trip" + ); +} From 1677ff5ad685129f886b5c689ab3a8a00fb1927a Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 10 Jun 2026 23:43:59 -0700 Subject: [PATCH 10/12] ci+docs: full feature matrix in CI; ARCHITECTURE export/import section; v0.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI now tests --features fjall,rocksdb,transport (previously default features only, silently skipping the fjall/rocksdb conformance suites on every PR) plus a default-features pass, and clippy covers --all-features. MinIO + mc land on the runner via the existing mise-action — no Docker. ARCHITECTURE.md gains the Export/Import section (artifact anatomy, the cursor-consistency invariant, per-backend mechanics, storage accounting, lease + transport), three Design Decisions entries, and Failure Modes rows. v0.4.0 breaking inventory for downstream bumps: - SnapshotStore gains required cursor() and export_to() - watch_applied gains the exports: Option> parameter (pass None) - SnapshotError gains ArtifactInvalid (breaks exhaustive matches) - inherent FjallSnapshot::cursor()/RocksDbSnapshot::cursor() removed in favor of the trait method (returns owned WatchCursor) Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 10 ++++++++-- ARCHITECTURE.md | 39 +++++++++++++++++++++++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/artifact.rs | 25 ++++++++++++++----------- src/export_lease.rs | 6 +++--- src/snapshot.rs | 6 +++++- src/snapshot_fjall.rs | 9 ++------- src/transport.rs | 32 +++++++++++++++++++------------- tests/bootstrap.rs | 9 +++++++-- tests/common/mod.rs | 38 ++++++++++++++++++++++---------------- tests/integration.rs | 16 ++++++++++++---- tests/snapshot_store.rs | 11 +++++++++-- tests/transport.rs | 5 ++++- tests/transport_s3.rs | 31 ++++++++++++++++++++++--------- 15 files changed, 168 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92a615d..da2d4a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,12 @@ jobs: - uses: actions/checkout@v5 - uses: jdx/mise-action@v2 - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --all-targets -- -D warnings - # Integration tests boot a real nats-server (installed above via mise). + - run: cargo clippy --all-targets --all-features -- -D warnings + # The full feature matrix, or the fjall/rocksdb conformance suites and + # the transport tier silently skip (default features exclude them all). + # Integration tests boot real nats-server + minio binaries (installed + # above via mise — no Docker). + - run: cargo test --lib --tests --features fjall,rocksdb,transport + # The default-features surface consumers without the optional backends + # actually compile against. - run: cargo test --lib --tests diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57efa51..068cbb5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -259,6 +259,28 @@ write_update() compact() [blocking: replay → dedup `checkpoint()` writes only a cursor record and calls `BufWriter::flush()` — a `write(2)` into the page cache. This survives a process crash but NOT a power loss. The only `fsync` is in `compact()`. The snapshot is a cache; a lost tail is rebuilt from a NATS scan + watch replay. +### Export / Import (replica bootstrap) + +When the watched bucket is a **bounded** log (size-capped, history evicted), a fold is no longer rebuildable from NATS alone — the folds become the only full replicas. Export/import makes them transferable, which is what lets a new node, a node with a lost/corrupt fold, or a node whose cursor aged out of the log bootstrap at all. + +**Artifact anatomy.** A directory: the backend's files under `data/`, plus `MANIFEST.json` carrying the artifact schema version, the backend identity and its on-disk format generation, per-file sizes + BLAKE3 digests, and — the load-bearing field — the **watch cursor the payload is exactly consistent with**. Import resumes the watch from that cursor and replays only the log tail. The manifest is written last and the whole stage is atomically renamed, so an artifact that exists is complete; a crash mid-export leaves only a hidden temp dir. + +**The cursor-consistency invariant.** `export_to(&mut self)` cannot run concurrently with `apply` (exclusive borrow), and inside `watch_applied` exports run between flushes via the `ExportRequest` channel (pending batch flushed first) — so the embedded cursor equals the applied cursor, exactly. Every backend re-proves it at export time by **reopening the copy** and checking cursor equality: because every `apply` commits the cursor in the same atomic batch as its data, a recovered cursor that matches the live one is a complete tail-loss detector. + +**Per-backend export mechanics.** + +- **append-log**: write a compacted log from the in-RAM fold (`compact_to_file`), verify by `load()`. +- **RocksDB**: the engine's native `Checkpoint` (memtable flush + SST hardlinks) — consistent by construction; verify-open anyway for the uniform guarantee. +- **fjall** (no checkpoint API): `persist(SyncAll)` (journal complete), best-effort quiesce (rotate memtables, drain flushes/compactions, bounded), copy the DB dir — hardlink immutable `tables/`/`blobs/`, byte-copy journal + metadata + `lock` — with a bounded retry against background GC, then verify-by-reopen. Correctness rests on the exclusive borrow + the verify gate, never on the quiesce. + +**Import** stages a verified copy beside the destination (every hash re-checked — the transport that delivered the artifact is untrusted), opens the staged copy and gates on manifest-cursor equality, then atomically renames. A bad artifact never becomes a fold; a crash mid-import leaves nothing at the destination. + +**Storage accounting.** Exports are hardlink-dominant (extra disk ≈ journal + small metadata, not 2×), but a lingering artifact pins files the source later compacts away — artifacts are **transient**: upload, then delete (`run_export_round` enforces this). Stage and destination must be on the fold's filesystem; the EXDEV fallback silently degrades hardlinks to full copies. + +**Fleet coordination.** `ExportLease` makes exactly one replica perform a given export round: create-only CAS to win, expiry embedded in the value (no server TTL machinery), CAS takeover of expired/corrupt leases, `complete()` publishes the exported cursor on the key (the fleet-visible last-export record), `abandon()` frees a failed round early. Clock skew at worst causes a duplicate export — safe; the lease is dedup, not a correctness gate. + +**Transport** (feature `transport`): plain tar of the artifact at `/` via any `object_store` backend, manifest duplicated as a sibling `.manifest.json` uploaded last (remote completeness marker + cheap cursor peek). `run_export_round` composes lease → export → upload → complete → delete-local; per-backend `import_remote` composes download → verify → import. + ## Connection Lifecycle ``` @@ -315,6 +337,18 @@ async-nats ≤0.46 has a race: the server can deliver the first batch of push-co Checkpoints are frequent (every N watch events). An fsync per checkpoint would add milliseconds of disk-sync latency to the hot watch path. Since the snapshot is a cache backed by NATS, a tail lost to power loss is rebuilt from a NATS replay — not a correctness failure. The only `fsync` is in `compact_to_file()`, where it guarantees the new compact file is durable before the atomic rename replaces the old one. +### Why raw backend-dir artifacts instead of a logical re-encode? + +Export copies the backend's own files rather than re-encoding entries into a portable format. The engine's native integrity machinery (fjall journal CRCs + version checksums, RocksDB MANIFEST validation) then verifies the artifact on open — the same code path that validates the fold after a crash — and hardlinks make export cost ~O(metadata) instead of O(data). The trade is per-backend artifact formats; the manifest's backend identity + format generation make that explicit, and the engines' own format markers travel inside the payload as defense in depth. + +### Why does export verify by reopening the copy? + +fjall has no checkpoint API, so its copy is assembled from parts; rather than trusting that assembly, every backend opens the staged copy and requires the recovered cursor to equal the live fold's. Cursor-in-every-apply-batch makes this a complete tail-loss detector, and it reuses the engine's recovery as the oracle instead of reimplementing consistency checks. The cost — one extra open per export — is noise for a periodic exporter. + +### Why is the export lease's expiry in the value instead of a server TTL? + +Per-message TTLs need a new-enough NATS server, a bucket flag, and a backend that supports them; an `expires_at` inside the value works on any `KvWriter` and keeps acquisition/takeover as two plain CAS operations. The cost is wall-clock comparison across nodes — acceptable because a premature steal merely produces a duplicate artifact (last-write-wins on the same key), never corruption: the lease is work-dedup, not a correctness gate. + ### Why write in sorted key order during compaction? `HashMap` iteration order is random per process. Sorting produces a deterministic byte layout for a given logical state, enabling byte-level snapshot comparison (integrity checksums, test assertions) and making file diffs readable. The O(n log n) sort is negligible relative to the I/O it precedes. @@ -364,6 +398,11 @@ Checkpoints are frequent (every N watch events). An fsync per checkpoint would a | Snapshot wrong format version | `SnapshotError::InvalidFormat`; delete file, full NATS replay | | `compact()` I/O error | Retry; if persistent, delete file and rebuild from NATS | | Synadia Cloud stream limit | Raw API path treats as non-fatal; verifies with `get_key_value` | +| Tampered/torn artifact | `ArtifactInvalid` at import (hash/cursor gate); nothing lands at the destination — fetch another artifact | +| Artifact backend/format mismatch | `ArtifactInvalid` before any open; payload format markers re-checked by the engine itself | +| Export under churn (fjall) | Copy retries on GC'd files (×3); verify-by-reopen catches anything torn | +| Export/upload fails mid-round | Lease abandoned, local artifact deleted; next trigger elects a new node | +| Artifact cursor older than the log | `CursorExpired` on resume → full watch fallback; checkpoint more often than log retention | ## Package Structure diff --git a/Cargo.lock b/Cargo.lock index 7b69167..204b0a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "beyond-slipstream" -version = "0.3.3" +version = "0.4.0" dependencies = [ "async-nats", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 864bb05..261beff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ rust-version = "1.92" [package] name = "beyond-slipstream" -version = "0.3.3" +version = "0.4.0" edition.workspace = true license.workspace = true rust-version.workspace = true diff --git a/src/artifact.rs b/src/artifact.rs index 6f8942c..c2e121a 100644 --- a/src/artifact.rs +++ b/src/artifact.rs @@ -136,7 +136,7 @@ pub(crate) fn hex_encode(bytes: &[u8]) -> String { } pub(crate) fn hex_decode(s: &str) -> Option> { - if s.len() % 2 != 0 { + if !s.len().is_multiple_of(2) { return None; } (0..s.len()) @@ -421,12 +421,14 @@ fn stage_dir_in(parent: &Path) -> Result { } fn dest_parent(dest: &Path) -> Result<&Path, SnapshotError> { - dest.parent().filter(|p| !p.as_os_str().is_empty()).ok_or_else(|| { - invalid(format!( - "destination {} has no parent directory", - dest.display() - )) - }) + dest.parent() + .filter(|p| !p.as_os_str().is_empty()) + .ok_or_else(|| { + invalid(format!( + "destination {} has no parent directory", + dest.display() + )) + }) } // --------------------------------------------------------------------------- @@ -571,9 +573,7 @@ pub(crate) fn verify_and_stage_import( .to_str() .ok_or_else(|| invalid(format!("non-UTF-8 payload path: {}", rel.display())))?; if !declared.contains(rel) { - return Err(invalid(format!( - "payload contains undeclared file: {rel}" - ))); + return Err(invalid(format!("payload contains undeclared file: {rel}"))); } } @@ -722,7 +722,10 @@ mod tests { #[test] fn rejects_wrong_schema_version() { let dir = TempDir::new().unwrap(); - write_raw_manifest(dir.path(), &wire_json("", "[]", ARTIFACT_SCHEMA_VERSION + 1)); + write_raw_manifest( + dir.path(), + &wire_json("", "[]", ARTIFACT_SCHEMA_VERSION + 1), + ); match read_manifest(dir.path()) { Err(SnapshotError::ArtifactInvalid(msg)) => { assert!(msg.contains("schema_version"), "{msg}"); diff --git a/src/export_lease.rs b/src/export_lease.rs index 65631dd..2f5d6b9 100644 --- a/src/export_lease.rs +++ b/src/export_lease.rs @@ -64,7 +64,7 @@ pub struct LeaseRecord { } /// Coordinates "at most one export per round" across every replica of a fold. -/// See the [module docs](self). +/// See the module docs for the CAS + embedded-expiry mechanism. pub struct ExportLease { reader: Arc, writer: Arc, @@ -122,8 +122,8 @@ impl ExportLease { completed_cursor_hex: None, completed_at_unix: None, }; - let bytes = serde_json::to_vec(&record) - .map_err(|e| KvError::SerializationError(e.to_string()))?; + let bytes = + serde_json::to_vec(&record).map_err(|e| KvError::SerializationError(e.to_string()))?; // Fast path: the round is open — create-only, one winner. match self.writer.create(&self.key, &bytes).await { diff --git a/src/snapshot.rs b/src/snapshot.rs index 332499a..de82d59 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -508,7 +508,11 @@ impl AppendLogSnapshot { })?; // This backend's artifact is exactly one payload file. - let expected = format!("{}/{}", crate::artifact::PAYLOAD_DIR, Self::ARTIFACT_PAYLOAD); + let expected = format!( + "{}/{}", + crate::artifact::PAYLOAD_DIR, + Self::ARTIFACT_PAYLOAD + ); if manifest.files.len() != 1 || manifest.files[0].path != expected { return Err(SnapshotError::ArtifactInvalid(format!( "append-log artifact must contain exactly {expected:?}" diff --git a/src/snapshot_fjall.rs b/src/snapshot_fjall.rs index 10b1f98..b91e597 100644 --- a/src/snapshot_fjall.rs +++ b/src/snapshot_fjall.rs @@ -534,9 +534,7 @@ impl SnapshotStore for FjallSnapshot { let stage = ExportStage::new(dest_dir)?; let payload = stage.payload(); - self.db - .persist(PersistMode::SyncAll) - .map_err(map_fjall)?; + self.db.persist(PersistMode::SyncAll).map_err(map_fjall)?; self.quiesce(); let mut attempt = 0; @@ -549,10 +547,7 @@ impl SnapshotStore for FjallSnapshot { { // A straggler flush/compaction GC'd a file under the copy. // Clear and re-copy from the now-quieter tree. - tracing::warn!( - attempt, - "fjall export copy raced background GC; retrying" - ); + tracing::warn!(attempt, "fjall export copy raced background GC; retrying"); std::fs::remove_dir_all(&payload)?; } Err(e) => return Err(e), diff --git a/src/transport.rs b/src/transport.rs index 3af1a5a..f4d7a3e 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -33,9 +33,7 @@ use tokio::sync::{mpsc, oneshot}; use tracing::warn; use crate::applied::ExportRequest; -use crate::artifact::{ - ExportManifest, MANIFEST_FILE, check_dest_available, manifest_from_slice, -}; +use crate::artifact::{ExportManifest, MANIFEST_FILE, check_dest_available, manifest_from_slice}; use crate::export_lease::ExportLease; use crate::kv::WatchCursor; use crate::snapshot::SnapshotError; @@ -48,8 +46,8 @@ const CHUNK: usize = 8 << 20; /// `CHUNK × MAX_CONCURRENT_PARTS`). const MAX_CONCURRENT_PARTS: usize = 8; -/// Ship artifacts to durable storage and fetch them back. See [module](self) -/// docs for the wire format. +/// Ship artifacts to durable storage and fetch them back. See the module docs +/// for the wire format. #[async_trait] pub trait ArtifactTransport: Send + Sync { /// Tar `artifact_dir` and upload it at `key`, then upload the manifest as @@ -137,13 +135,18 @@ impl ArtifactTransport for ObjectStoreTransport { // blocking tar writer and the async multipart writer decoupled; the // disk cost is one tar's worth, transient. let src = artifact_dir.to_path_buf(); - let tar_file = tokio::task::spawn_blocking(move || -> Result { - let tmp = tempfile::NamedTempFile::new()?; - let mut builder = tar::Builder::new(std::io::BufWriter::new(tmp.reopen()?)); - builder.append_dir_all(".", &src)?; - builder.into_inner()?.into_inner().map_err(|e| SnapshotError::Io(e.into_error()))?; - Ok(tmp) - }) + let tar_file = tokio::task::spawn_blocking( + move || -> Result { + let tmp = tempfile::NamedTempFile::new()?; + let mut builder = tar::Builder::new(std::io::BufWriter::new(tmp.reopen()?)); + builder.append_dir_all(".", &src)?; + builder + .into_inner()? + .into_inner() + .map_err(|e| SnapshotError::Io(e.into_error()))?; + Ok(tmp) + }, + ) .await .map_err(|e| SnapshotError::Backend(format!("tar task panicked: {e}")))??; @@ -228,7 +231,10 @@ impl ArtifactTransport for ObjectStoreTransport { .into_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk.map_err(map_obj)?; - tar_writer.write_all(&chunk).await.map_err(SnapshotError::Io)?; + tar_writer + .write_all(&chunk) + .await + .map_err(SnapshotError::Io)?; } tar_writer.flush().await.map_err(SnapshotError::Io)?; drop(tar_writer); diff --git a/tests/bootstrap.rs b/tests/bootstrap.rs index e54bd3d..5f58137 100644 --- a/tests/bootstrap.rs +++ b/tests/bootstrap.rs @@ -132,9 +132,11 @@ async fn wait_applied(node: &Node, at_least: u64) { /// 5. **Delta-only**: B was delivered exactly M updates, the first at /// cursor+1 — no overlap, no gap, no full replay. /// 6. B's fold state equals the bucket (the truth). +type ImportFn = fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>; + async fn live_export_bootstrap_delta_only( open: impl Fn(&Path) -> (WatchCursor, S), - import: fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>, + import: ImportFn, ) where S: SnapshotStore + Send + 'static, { @@ -188,7 +190,10 @@ async fn live_export_bootstrap_delta_only( .expect("send export request"); let manifest = reply_rx.await.expect("reply").expect("export succeeds"); let export_rev = manifest.cursor.as_u64().expect("cursor rev"); - assert!(export_rev >= last_rev, "artifact covers the pre-export history"); + assert!( + export_rev >= last_rev, + "artifact covers the pre-export history" + ); // Post-export churn: exactly M updates (including a delete — tombstones // must ride the tail too). diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a81aafb..bdf1cf6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -52,18 +52,20 @@ impl TestNats { Run `mise install` or set NATS_SERVER_BIN." ) }); - let url = format!("nats://127.0.0.1:{port}"); + // Build the guard FIRST: if readiness polling panics, Drop reaps the + // child instead of leaking a zombie. + let guard = TestNats { + child, + url: format!("nats://127.0.0.1:{port}"), + _store_dir: store_dir, + }; for _ in 0..100 { - if async_nats::connect(&url).await.is_ok() { - return TestNats { - child, - url, - _store_dir: store_dir, - }; + if async_nats::connect(&guard.url).await.is_ok() { + return guard; } tokio::time::sleep(Duration::from_millis(100)).await; } - panic!("nats-server at {url} never became ready"); + panic!("nats-server at {} never became ready", guard.url); } } @@ -118,7 +120,13 @@ impl TestMinio { ) }); - let endpoint = format!("http://127.0.0.1:{api_port}"); + // Build the guard FIRST so a panicking readiness loop reaps the child. + let guard = TestMinio { + child, + endpoint: format!("http://127.0.0.1:{api_port}"), + _data_dir: data_dir, + }; + let endpoint = guard.endpoint.clone(); // Create the test bucket with mc, retrying until the server is up. // A throwaway --config-dir keeps ~/.mc untouched. @@ -161,13 +169,11 @@ impl TestMinio { } tokio::time::sleep(Duration::from_millis(200)).await; } - assert!(ready, "minio at {endpoint} never became ready (mc mb failed)"); - - TestMinio { - child, - endpoint, - _data_dir: data_dir, - } + assert!( + ready, + "minio at {endpoint} never became ready (mc mb failed)" + ); + guard } /// Builder options for `ObjectStoreTransport::from_url_opts` pointing at diff --git a/tests/integration.rs b/tests/integration.rs index 1cb832d..0656ca2 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1505,8 +1505,12 @@ async fn export_lease_one_winner_among_concurrent_acquirers() { let mut tasks = Vec::new(); for i in 0..8 { - let lease = slipstream::ExportLease::new(store.as_ref(), "export.edge.us-east", format!("node-{i}")) - .expect("lease"); + let lease = slipstream::ExportLease::new( + store.as_ref(), + "export.edge.us-east", + format!("node-{i}"), + ) + .expect("lease"); tasks.push(tokio::spawn(async move { lease .try_acquire(Duration::from_secs(60)) @@ -1580,7 +1584,10 @@ async fn export_lease_expiry_allows_takeover() { assert_eq!(guard.record().holder_id, "node-b"); let current = b.current().await.expect("read").expect("record exists"); - assert_eq!(current.holder_id, "node-b", "takeover is visible fleet-wide"); + assert_eq!( + current.holder_id, "node-b", + "takeover is visible fleet-wide" + ); } /// `complete` publishes the exported cursor on the lease key — the @@ -1590,7 +1597,8 @@ async fn export_lease_complete_publishes_outcome() { let nats = TestNats::start().await; let (_conn, store) = nats.store("lease-complete").await; - let lease = slipstream::ExportLease::new(store.as_ref(), "export.round", "node-a").expect("lease"); + let lease = + slipstream::ExportLease::new(store.as_ref(), "export.round", "node-a").expect("lease"); let guard = lease .try_acquire(Duration::from_secs(60)) .await diff --git a/tests/snapshot_store.rs b/tests/snapshot_store.rs index 76a4569..2cb7ad7 100644 --- a/tests/snapshot_store.rs +++ b/tests/snapshot_store.rs @@ -413,7 +413,11 @@ fn check_export_import_round_trip( let dest = dir.path().join("imported"); let (cursor, s) = import(&artifact, &dest).expect("import"); - assert_eq!(cursor.as_u64(), Some(7), "imported cursor is the manifest's"); + assert_eq!( + cursor.as_u64(), + Some(7), + "imported cursor is the manifest's" + ); assert_eq!(cursor, manifest.cursor); assert_eq!(dump(&s), expected_state(), "imported state is identical"); } @@ -706,7 +710,10 @@ fn append_log_empty_value_round_trip() { check_empty_value_round_trip(open_append_log); } -fn import_append_log(artifact: &Path, dest: &Path) -> Result<(WatchCursor, AppendLogSnapshot), SnapshotError> { +fn import_append_log( + artifact: &Path, + dest: &Path, +) -> Result<(WatchCursor, AppendLogSnapshot), SnapshotError> { AppendLogSnapshot::import(artifact, dest, u64::MAX) } diff --git a/tests/transport.rs b/tests/transport.rs index c301e4f..b413b7f 100644 --- a/tests/transport.rs +++ b/tests/transport.rs @@ -66,7 +66,10 @@ async fn upload_manifest_download_import_round_trip() { let (transport, _bucket) = local_transport(); let (artifact, manifest, dir) = exported_artifact(); - transport.upload("edge/us-east/latest", &artifact).await.unwrap(); + transport + .upload("edge/us-east/latest", &artifact) + .await + .unwrap(); // Manifest peek without a payload download. let peeked = transport.manifest("edge/us-east/latest").await.unwrap(); diff --git a/tests/transport_s3.rs b/tests/transport_s3.rs index d735b60..80feb35 100644 --- a/tests/transport_s3.rs +++ b/tests/transport_s3.rs @@ -122,10 +122,10 @@ async fn wait_applied(node: &Node, at_least: u64) { /// (lease won, exported through the watch loop, uploaded to MinIO, completion /// published, artifact deleted) → node B downloads + imports + resumes → /// delta-only delivery → state equals the bucket. -async fn full_bootstrap_loop( - open: impl Fn(&Path) -> (WatchCursor, S), - import: fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>, -) where +type ImportFn = fn(&Path, &Path) -> Result<(WatchCursor, S), SnapshotError>; + +async fn full_bootstrap_loop(open: impl Fn(&Path) -> (WatchCursor, S), import: ImportFn) +where S: SnapshotStore + Send + 'static, { let (nats, minio) = tokio::join!(TestNats::start(), TestMinio::start()); @@ -197,7 +197,10 @@ async fn full_bootstrap_loop( wait_applied(&node_a, final_rev).await; // Node B: manifest peek, download, import, resume. - let peeked = transport.manifest("edge/us-east/latest").await.expect("peek"); + let peeked = transport + .manifest("edge/us-east/latest") + .await + .expect("peek"); assert_eq!(peeked.cursor, manifest.cursor); let downloaded = dir.path().join("downloaded-artifact"); @@ -240,7 +243,10 @@ async fn full_bootstrap_loop( .map(|e| (e.key, e.value)) .collect(); bucket_state.sort(); - assert_eq!(fold_state, bucket_state, "bootstrapped fold equals the bucket"); + assert_eq!( + fold_state, bucket_state, + "bootstrapped fold equals the bucket" + ); } #[cfg(feature = "fjall")] @@ -307,9 +313,13 @@ async fn minio_multipart_upload_round_trips() { for i in 0..2500u64 { // Pseudo-random bytes (LCG) so neither tar nor S3 sees compressible runs. let mut v = vec![0u8; 10 * 1024]; - let mut x = i.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let mut x = i + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); for b in &mut v { - x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); *b = (x >> 33) as u8; } batch.push(KvUpdate::Put(slipstream::KvEntry { @@ -328,7 +338,10 @@ async fn minio_multipart_upload_round_trips() { "artifact must exceed several part sizes (got {payload_bytes} bytes)" ); - transport.upload("big/latest", &artifact).await.expect("multipart upload"); + transport + .upload("big/latest", &artifact) + .await + .expect("multipart upload"); let downloaded = dir.path().join("downloaded"); transport From 4a3155b24885a7cf70c97fba5621f08d16649156 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Thu, 11 Jun 2026 09:57:12 -0700 Subject: [PATCH 11/12] docs(snapshot): settled-tree benchmark numbers + BACKENDS.md selection guide Replace artifact-era latency claims with measured settled-tree data from the 500M-route bench (percentiles, not means): cold gets p50 542us/292us (fjall/rocksdb), absent-key ~400ns via restored bottom-level filters, settle() costs (fjall ~19min full rewrite + 2x disk transient; rocksdb ~40s drain). BACKENDS.md collects the full cross-scale dataset and the backend-selection heuristic in one place. Co-Authored-By: Claude Fable 5 --- BACKENDS.md | 127 ++++++++++++++++++++++++++++++++++++++++ src/snapshot_fjall.rs | 24 +++++--- src/snapshot_rocksdb.rs | 32 ++++++---- 3 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 BACKENDS.md diff --git a/BACKENDS.md b/BACKENDS.md new file mode 100644 index 0000000..b109fcc --- /dev/null +++ b/BACKENDS.md @@ -0,0 +1,127 @@ +# Snapshot Backends + +Three [`SnapshotStore`] backends ship with slipstream. Pick one based on fold size +and read pattern. + +## Quick reference + +| | `AppendLogSnapshot` | `FjallSnapshot` | `RocksDbSnapshot` | +|---|---|---|---| +| Feature flag | (default) | `fjall` | `rocksdb` | +| Build deps | none | none | C++ toolchain, libclang | +| Fold size | fits in RAM | any | any | +| Cold get p50 (500M routes) | n/a | 542 µs | 292 µs | +| Cold get p999 (500M routes) | n/a | 3.7 ms | 898 µs | +| Hydrate + settle (500M) | n/a | 40 min | 43 min | +| Disk per entry (500M) | n/a | 226 B | 245 B | +| `settle()` cost (500M) | n/a | ~19 min, 2x disk | ~40 s | + +## Choosing + +**`AppendLogSnapshot`**: fold fits in RAM. No configuration, no dependencies. + +**`FjallSnapshot`**: fold is too large for RAM; build environment is pure Rust; +write throughput matters more than cold-read tail latency. + +**`RocksDbSnapshot`**: fold is too large for RAM; cold-read tail latency or settle +time matters; operational tooling (`ldb`, `sst_dump`) is useful. + +Total time-to-serving-ready at 500M routes is a wash (2602 s fjall, 2593 s +rocksdb). The divergence is tail latency and settle cost. + +## Benchmark data + +All numbers: 500M routes, ~60 B keys, ~200 B incompressible values, 1 GiB block +cache, NVMe ext4, settled trees. Source: `benches/snapshot_backends.rs`. + +### Hydration (`apply` path, 1024-update batches) + +| | fjall | rocksdb | +|---|---|---| +| 50M routes | 47 s (1.06 M/s) | 121 s (0.41 M/s) | +| 100M routes | 110 s (0.91 M/s) | 344 s (0.29 M/s) | +| 250M routes | 480 s (0.52 M/s) | 1165 s (0.21 M/s) | +| 500M routes | 1475 s (0.34 M/s) | 2552 s (0.20 M/s) | + +fjall throughput decays with scale as compaction debt accumulates during +hydration; rocksdb drains that debt concurrently. + +### `settle()` after 500M hydration + +| | fjall | rocksdb | +|---|---|---| +| Duration | 1127 s | 41 s | +| Peak disk | 203 GiB (2x: old + new generations) | 105 GiB (shrinks: zstd reaches bottom) | +| Mechanism | Full tree rewrite (major compaction) | Drain queued compactions | + +**Call `settle()` before serving.** Both engines accumulate compaction debt during +bulk hydration. Without it, cold point-gets are 8-10x slower (measured: rocksdb +~0.9 ms mean unsettled vs 248 µs mean settled at 500M). + +### Cold point-gets (settled, uniform random, 10k probes) + +| | fjall | rocksdb | +|---|---|---| +| p50 | 542 µs | 292 µs | +| p90 | 757 µs | 485 µs | +| p99 | 1.9 ms | 686 µs | +| p999 | 3.7 ms | 898 µs | +| max | 39.9 ms | 2.5 ms | + +### Absent-key lookups (settled, filters reject in RAM) + +| | fjall | rocksdb | +|---|---|---| +| p50 | 421 ns | 321 ns | + +Both engines build bottom-level filters. `expect_point_read_hits` (fjall) and +`optimize_filters_for_hits` (rocksdb) are disabled in both backends: without +bottom-level filters, absent-key lookups become guaranteed disk probes, and +cold-get latency on unsettled trees spikes to double digits of milliseconds. + +### Prefix scans (hot prefix, 1000 entries) + +| | fjall | rocksdb | +|---|---|---| +| 50M routes | 189 µs | 122 µs | +| 500M routes | 176 µs | 129 µs | + +Scan latency is flat with scale for both engines. This measures a single +service's routes, resident in cache. + +### `RocksDbReader::multi_get` vs get-loop (100 cold keys) + +| | settled | unsettled | +|---|---|---| +| get-loop | 23.7 ms | 103 ms | +| multi_get | 19.5 ms | 18.5 ms | + +`multi_get` overlaps cold block reads the loop pays sequentially. Its edge +grows with compaction debt and cache-miss rate. Against a hot working set, +the loop is faster (marshaling overhead, nothing to coalesce). + +## Usage + +```rust +use slipstream::{RocksDbConfig, RocksDbSnapshot, SnapshotStore}; + +// open or resume +let (cursor, mut store) = RocksDbSnapshot::open(path, RocksDbConfig::default())?; + +// after bulk hydration, before serving +store.settle()?; + +// concurrent read handle (safe to clone across threads) +let reader = store.reader(); +``` + +```rust +use slipstream::{FjallConfig, FjallSnapshot, SnapshotStore}; + +let (cursor, mut store) = FjallSnapshot::open(path, FjallConfig::default())?; +store.settle()?; // ~19 min at 500M routes; budget 2x disk headroom +let reader = store.reader(); +``` + +Both backends implement [`SnapshotStore`], so the `watch_applied` integration +is identical regardless of which you choose. diff --git a/src/snapshot_fjall.rs b/src/snapshot_fjall.rs index b91e597..98403db 100644 --- a/src/snapshot_fjall.rs +++ b/src/snapshot_fjall.rs @@ -48,13 +48,14 @@ //! and it makes every absent-key lookup a guaranteed disk probe. Everything //! else is deliberately left at fjall's defaults. //! -//! Measured at 500M routes (NVMe, `benches/snapshot_backends.rs`): tuned -//! hydration runs 0.42 M entries/s — 2.2× the RocksDB backend — at -//! 226 B/entry on disk. The trade is cold uniform point-gets once the fold -//! dwarfs RAM: multi-millisecond here vs sub-millisecond on the RocksDB -//! backend, whose partitioned, cache-pinned metadata bounds a cold get to -//! fewer disk reads. Write-heavy or hot-set-served folds favor fjall; -//! uniform cold-read folds favor RocksDB. +//! Measured at 500M routes (NVMe, `benches/snapshot_backends.rs`, settled +//! tree): hydration 0.34 M entries/s — 1.7× the RocksDB backend — at +//! 226 B/entry on disk. Cold uniform point-gets: p50 542 µs / p99 1.9 ms / +//! p999 3.7 ms (RocksDB: 292 µs / 686 µs / 898 µs — fjall's median is ~1.9× +//! and its tail ~4× behind). Absent-key lookups reject in-RAM via filters at +//! ~420 ns. Reaching "settled" is the catch: see [`settle`](FjallSnapshot::settle). +//! Write-heavy and hot-set-served folds favor fjall; tail-latency-sensitive +//! cold-read folds favor RocksDB. use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -231,9 +232,16 @@ impl FjallSnapshot { /// /// fjall's background compaction is write-driven: after a bulk hydration /// stops, residual overlapping runs can persist indefinitely and inflate - /// cold-read latency (every unrejected run costs an extra disk probe). + /// cold-read latency (every unrejected run costs an extra disk probe — + /// measured ~10 ms cold gets unsettled vs 542 µs p50 settled at 500M). /// Call this after hydrating and before latency-sensitive serving begins; /// steady-state folding does not need it. + /// + /// This is a full tree rewrite — budget for it: ~19 minutes and a + /// transient ~2× disk footprint at 500M routes (105 GiB store), measured + /// while old and new generations coexist. The RocksDB backend's + /// [`settle`](crate::RocksDbSnapshot::settle) merely drains already-queued + /// compactions (~40 s at the same scale). pub fn settle(&self) -> Result<(), SnapshotError> { self.data.major_compact().map_err(map_fjall) } diff --git a/src/snapshot_rocksdb.rs b/src/snapshot_rocksdb.rs index 9b8f9ac..f8c74de 100644 --- a/src/snapshot_rocksdb.rs +++ b/src/snapshot_rocksdb.rs @@ -178,13 +178,14 @@ pub struct RocksDbConfig { /// `resident working set + ~200 MB metadata`; `0` falls back to RocksDB's /// 32 MiB default. /// - /// Measured with `benches/snapshot_backends.rs` (NVMe ext4, criterion - /// median, default 1 GiB cache): point-gets against a cache-resident - /// working set run ~2 µs; uniform random gets over a 500M-route fold - /// (114 GiB on disk, 245 B/entry, vs 27 GiB RAM — most reads hit disk) - /// run ~0.9 ms mean, per-service prefix scans ~127 ns/entry, hydration - /// 0.19 M entries/s. The cache buys hot-set residency — the µs-vs-ms gap - /// above — so size it to the working set. + /// Measured with `benches/snapshot_backends.rs` (NVMe ext4, default + /// 1 GiB cache, 500M-route fold: ~105 GiB settled on disk vs 27 GiB RAM, + /// so uniform reads mostly hit disk): hot-set point-gets ~2 µs; cold + /// uniform point-gets p50 292 µs / p99 686 µs / p999 898 µs; absent-key + /// lookups ~320 ns (in-RAM filter rejection); per-service prefix scans + /// ~129 ns/entry; hydration 0.20 M entries/s. The cache buys hot-set + /// residency — the µs-vs-hundreds-of-µs gap — so size it to the working + /// set. pub cache_size_bytes: u64, } @@ -374,9 +375,12 @@ impl RocksDbSnapshot { /// drained. /// /// A bulk hydration leaves the tree with pending compactions that inflate - /// cold-read latency until they drain. Call this after hydrating and - /// before latency-sensitive serving begins; steady-state folding does not - /// need it. + /// cold-read latency until they drain (measured: ~8× on cold point-gets + /// at 500M routes). Call this after hydrating and before + /// latency-sensitive serving begins; steady-state folding does not need + /// it. Cheap relative to the fjall backend's full-rewrite + /// [`settle`](crate::FjallSnapshot::settle): ~40 s at 500M routes, + /// because the background threads drained most debt during hydration. pub fn settle(&self) -> Result<(), SnapshotError> { let mut opts = WaitForCompactOptions::default(); opts.set_flush(true); @@ -466,9 +470,11 @@ impl RocksDbReader { /// and index lookups per SST. /// /// Measured (`benches/snapshot_backends.rs`, 100-key uniform random - /// batches against a 500M-route fold, most reads hitting NVMe): 18.5 ms - /// per batch vs 103 ms for a loop of [`get`](Self::get)s — 5.5× — because - /// `MultiGet` overlaps the cold block reads the loop pays sequentially. + /// batches against a 500M-route fold, most reads hitting NVMe): on a + /// settled tree, 19.5 ms per batch vs 23.7 ms for a loop of + /// [`get`](Self::get)s (~1.2×); on a tree mid-compaction-debt the gap was + /// 5.5× (18.5 ms vs 103 ms) — `MultiGet` overlaps the cold block reads + /// the loop pays sequentially, so its edge grows with reads-per-get. /// Against a cache-resident working set the loop is marginally *faster* /// (per-batch marshaling, nothing left to coalesce). Use this when /// batches are likely to miss; use the loop when they're hot. From 12d22026a61ddaa621a49ecd3617acd2d39a610a Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Thu, 11 Jun 2026 12:35:09 -0700 Subject: [PATCH 12/12] audit fixes --- ARCHITECTURE.md | 34 +- Cargo.lock | 3 +- Cargo.toml | 3 +- README.md | 21 +- benches/applied.rs | 1 + src/applied.rs | 671 +++++++++++++++++++++++++++++++++++++--- src/artifact.rs | 98 +++++- src/export_lease.rs | 54 +++- src/kv.rs | 44 ++- src/lib.rs | 1 + src/nats.rs | 290 +++++++++++++++-- src/snapshot.rs | 4 +- src/snapshot_rocksdb.rs | 14 +- src/transport.rs | 148 ++++++--- tests/bootstrap.rs | 1 + tests/integration.rs | 107 +++++-- tests/snapshot_store.rs | 46 +++ tests/transport.rs | 222 +++++++++++++ tests/transport_s3.rs | 1 + 19 files changed, 1587 insertions(+), 176 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 068cbb5..b0f5adb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -15,8 +15,10 @@ Disk ──► load(path) ──► replay_log() ──► HashMap / \ Yes No │ │ - watch_all(tx) + stale_keys() delta stream - └───────┬─────────┘ + stale-key resync (synthetic delta stream + deletes) + watch_all(tx) │ + (state-sync re-list) │ + └───────┬─────────────┘ │ KvUpdate → cache.apply() + snap.write_update() │ @@ -54,6 +56,8 @@ watch_all_from(cursor, tx) delta stream caller falls back to watch_all(tx) ``` +Every non-`_from` watch is a **state-sync** stream (NATS `DeliverPolicy::LastPerSubject`): the current value of every matching key is delivered first — the re-list — then live updates. A no-cursor consumer therefore converges on full bucket state from the watch alone, with no separate scan and no scan-to-watch race window. The `_from` variants skip the re-list and deliver only the delta. + ## Concepts & Terminology | Term | Definition | NOT | @@ -73,7 +77,8 @@ watch_all_from(cursor, tx) | `AppendLogSnapshot` | Default `SnapshotStore`: append-only log + in-RAM fold (pure-Rust) | Not for folds larger than RAM | | `FjallSnapshot` | On-disk `SnapshotStore` (fjall LSM, `feature = "fjall"`) for large folds | Not in the pure-Rust core; opt-in feature | | `RocksDbSnapshot` | On-disk `SnapshotStore` (RocksDB, `feature = "rocksdb"`) for large folds | Not pure-Rust; opt-in feature with a C++ build dep | -| `watch_applied` | Combinator: batch → apply → *then* advance cursor / fold into `SnapshotStore` | Not a raw watch; the cursor follows `apply`, not receipt | +| `watch_applied` | Combinator: batch → apply → *then* advance cursor / fold into `SnapshotStore`; resyncs stale keys on cursor expiry | Not a raw watch; the cursor follows `apply`, not receipt | +| `WatchScope` | What `watch_applied` watches: `All`, `Prefix`, or `Prefixes` (multi-filter union) | Not N consumers; `Prefixes` costs one consumer | | `ConnectionCapabilities` | Feature flags for runtime branching (CAS, streaming watch, …) | Not enforced; purely advisory | ## Layer Architecture @@ -113,7 +118,7 @@ watch_all_from(cursor, tx) The cursor is the NATS stream sequence number at the last checkpoint. On restart, pass it to `watch_all_from()` to subscribe at `cursor+1` — only the delta arrives, not the full history. -When the cursor expires (NATS retention window evicted those records), `CursorExpired` is returned. The caller falls back to `watch_all()` and should call `Snapshot::stale_keys()` to emit synthetic `Delete` events for keys that disappeared during the gap: +When the cursor expires (NATS retention window evicted those records), `CursorExpired` is returned. The fallback `watch_all()` re-list re-delivers the current value of every live key, but it cannot cover keys **deleted during the gap whose delete markers were also evicted** — those need synthetic `Delete` events diffed from prior state. `watch_applied` does this automatically when given a reader (see below). A raw-API caller hand-rolls the same diff with `Snapshot::stale_keys()`: ```rust match watcher.watch_all_from(&snap.cursor, tx).await { @@ -147,7 +152,9 @@ This is the lesson of Saltzer, Reed & Clark, *End-to-End Arguments in System Des **Snapshot consistency.** Raw `KvUpdate`s stream to the snapshot log as they arrive, but the *checkpoint* cursor is the post-apply cursor. A crash after a raw record is written but before its `apply`/checkpoint leaves the log holding data *ahead* of its cursor — which is safe: the cursor never names a revision whose `apply` had not returned, so resume re-delivers and re-applies that tail rather than skipping it. Compaction runs off the hot path via `spawn_blocking`, as everywhere else in the snapshot subsystem. -**Flush triggers.** A batch flushes when any of these fires: the `window` elapses, `batch.len()` reaches `config.max`, a shutdown is signalled, or the channel closes with a pending batch (the remainder is flushed before returning). On `CursorExpired` from the resume path the combinator logs and falls back to the full-scope watch (`watch_all` / `watch_prefix`); v1 replays the full re-list as a stream of puts (a deeper "resync" signal that diffs against prior state is a documented TODO). +**Flush triggers.** A batch flushes when any of these fires: the `window` elapses, `batch.len()` reaches `config.max`, a shutdown is signalled, or the channel closes with a pending batch (the remainder is flushed before returning). + +**Cursor-expired resync.** On `CursorExpired` from the resume path the combinator falls back to the full-scope watch (`watch_all` / `watch_prefix` / `watch_prefixes`), whose state-sync re-list re-delivers every live key as puts. The re-list cannot cover keys deleted during the gap whose markers were evicted with the cursor, so — when the combinator is given a `KvReader` and a store — it closes that hole first: the watch task lists the bucket's live keys, hands them to the main loop, and waits for an ack; the main loop flushes, diffs the fold's in-scope keys against the listing, and runs a synthetic `KvUpdate::Delete` (unknown version — never advances the cursor) through `parse`/`apply`/store for each key that vanished; only then does the fallback watch start. That ack ordering is the invariant: a synthetic delete always precedes the re-list put for the same key, so delete-then-recreate during the gap converges. Without a reader the fallback is re-list-only and logs the possible stale keys. This is the layer the tunnel router (swap route table) and edge origin watcher (rebuild hashrings) both collapse onto: `parse` extracts the domain registration, `apply` swaps the live state, `on_applied` persists the cursor. @@ -307,6 +314,14 @@ The double-check pattern in `connect()` guards a concurrent connect race: a seco The raw `KvWatcher` + `WatchCursor` + `SnapshotWriter` pieces let callers hand-roll the batch/apply/advance loop — and every known caller advanced the cursor on *receipt* rather than after *apply*, silently skipping un-applied updates on crash+resume. That is a footgun in the library's core guarantee, not a caller bug to be fixed N times. Encoding cursor-after-apply once, behind a combinator that callers can't get wrong, is cheaper and safer than documentation. `apply` stays the only domain logic; the cursor/snapshot/`on_applied` bookkeeping is the library's. See [Applied-Cursor Watch](#applied-cursor-watch-watch_applied). +### Why do watches deliver current state first (state-sync semantics)? + +async-nats's bare `watch`/`watch_all`/`watch_many` ride `DeliverPolicy::New` — live updates only. A consumer built on that needs a separate `scan()` to seed, and any write landing between the scan and the watch attach is in neither — silently lost until the next reseed (the seed-then-watch race, demonstrated in `watch_prefix_relist_covers_seed_then_watch_gap`). Mapping every non-`_from` watch to `_with_history` (`LastPerSubject`) makes the watch itself deliver the seed, in revision order, with no race window: one primitive, correct by construction. The cost is a full re-list on every no-cursor watch start — which is what a no-cursor start *means*; consumers that have state resume with a `_from` variant and skip it. + +### Why does the cursor-expired resync list keys instead of re-scanning values? + +The fallback watch's re-list already carries every live key's value, so the resync only needs to learn which keys *no longer exist* — `reader.keys()` (headers only, no value bytes) is sufficient and cheap. The synthetic deletes carry an unknown version and never advance the cursor: the fold's persisted cursor stays at its (expired) position until real re-list revisions move it, so a crash mid-resync just re-runs the same idempotent diff on the next start. Ordering, not versioning, provides correctness: the resync acks before the fallback watch is established, so a synthetic delete can never land after the re-list put that resurrects the same key. + ### Why KvError: Clone instead of Box? A failed connect future may be observed by multiple concurrent callers waiting on a shared result. `Clone` lets the error fan out to N waiters without `Arc`. The cost: `std::io::Error` and `async-nats` error types are not `Clone`, so their structured cause chain is flattened into a pre-rendered `String` at the boundary. The trade-off is explicit: no `#[source]` chain, but the message carries context instead. @@ -364,8 +379,11 @@ Per-message TTLs need a new-enough NATS server, a bucket flag, and a backend tha | `delete_with_version()` | `kv.update(key, [], rev)` — CAS write of empty value as tombstone | | `KvUpdate::Purge` | `KV-Operation: PURGE` — all history removed; treated same as Delete in snapshot | | `scan()` / `keys()` | Ephemeral push consumer with `DeliverPolicy::LastPerSubject` | -| `watch_prefix()` | `kv.watch("{prefix}>")` — server-side subject-filter wildcard | +| `watch_all()` | `kv.watch_with_history(">")` — `LastPerSubject`: state-sync re-list, then live | +| `watch_prefix()` | `kv.watch_with_history("{prefix}>")` — server-side subject filter, same re-list | +| `watch_prefixes()` | `kv.watch_many_with_history([..])` — ONE multi-filter consumer (NATS 2.10) | | `watch_all_from(cursor)` | `kv.watch_all_from_revision(cursor+1)` — server-side delta delivery | +| `watch_prefixes_from()` | Hand-built ordered push consumer: `filter_subjects` + `ByStartSequence(cursor+1)` (async-nats has no `watch_many_from_revision`) | ## Trust Model @@ -388,7 +406,7 @@ Per-message TTLs need a new-enough NATS server, a bucket flag, and a backend tha | Failure | Recovery | | ------------------------------- | -------------------------------------------------------------- | -| `CursorExpired` | Fall back to `watch_all()`; use `stale_keys()` for deletes | +| `CursorExpired` | `watch_applied` resyncs (synthetic deletes) + falls back to the state-sync re-list; raw callers use `stale_keys()` | | `WatchError` | Re-subscribe; watch stream dropped (NATS restart, reconnect) | | `Timeout` on any op | CLOSE_WAIT connection; call `shutdown()` + `connect()` | | `RevisionMismatch` on CAS | Re-read with `entry()`, resolve conflict, retry | @@ -415,7 +433,7 @@ Per-message TTLs need a new-enough NATS server, a bucket flag, and a backend tha | `src/snapshot_fjall.rs` | `FjallSnapshot`: on-disk `SnapshotStore` backed by fjall (`feature = "fjall"`) | | `src/snapshot_rocksdb.rs` | `RocksDbSnapshot`: on-disk `SnapshotStore` backed by RocksDB (`feature = "rocksdb"`) | | `src/snapshot_record.rs` | Shared `[ver_len][version][value]` value-record codec for the LSM backends | -| `src/applied.rs` | `watch_applied` cursor-after-apply combinator, generic over `SnapshotStore`: `WatchScope`, `BatchConfig` | +| `src/applied.rs` | `watch_applied` cursor-after-apply combinator, generic over `SnapshotStore`: `WatchScope`, `BatchConfig`, cursor-expired stale-key resync | | `src/lib.rs` | Re-exports all public types; no logic | | `benches/` | Criterion benchmarks for snapshot write/checkpoint/load throughput and batch throughput | | `tests/` | Integration tests (require live NATS) | diff --git a/Cargo.lock b/Cargo.lock index 204b0a8..b269c91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,8 +124,9 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "beyond-slipstream" -version = "0.4.0" +version = "0.5.0" dependencies = [ + "aho-corasick", "async-nats", "async-trait", "base64", diff --git a/Cargo.toml b/Cargo.toml index 261beff..115bf6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ rust-version = "1.92" [package] name = "beyond-slipstream" -version = "0.4.0" +version = "0.5.0" edition.workspace = true license.workspace = true rust-version.workspace = true @@ -19,6 +19,7 @@ name = "slipstream" path = "src/lib.rs" [dependencies] +aho-corasick = "1" async-nats = "0.46" async-trait = "0.1" base64 = "0.22" diff --git a/README.md b/README.md index 667761c..375e8e2 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,10 @@ use slipstream::{KvUpdate, KvWatcher}; let watcher = store.watcher().expect("store supports streaming"); let (tx, mut rx) = tokio::sync::mpsc::channel(128); +// Watches are state-sync streams: the current value of every matching key is +// delivered first (as puts), then live updates. No separate scan needed — and +// no scan-to-watch race window. +// // watch_all blocks until the stream ends — run it in a separate task tokio::spawn(async move { watcher.watch_all(tx).await.unwrap(); @@ -154,7 +158,9 @@ match watcher.watch_all_from(&cursor, tx.clone()).await { } ``` -`watch_prefix_from()` works the same way for prefix-filtered streams. +`watch_prefix_from()` works the same way for prefix-filtered streams, and +`watch_prefixes_from()` resumes the union of several prefixes on one +multi-filter consumer. ## Snapshot @@ -254,7 +260,10 @@ let (resume, store) = AppendLogSnapshot::load(Path::new("/var/lib/svc/state.snap // let (resume, store) = RocksDbSnapshot::open(dir, RocksDbConfig { sync: false, ..Default::default() })?; let final_cursor = watch_applied( - watcher, WatchScope::All, Some(resume), Some(store), BatchConfig::default(), + watcher, WatchScope::All, Some(resume), + Some(reader), // arms the cursor-expired stale-key resync; None to skip + Some(store), None, // store; export-request channel + BatchConfig::default(), parse, apply, on_applied, shutdown, ).await?; ``` @@ -270,9 +279,12 @@ use slipstream::{watch_applied, AppendLogSnapshot, BatchConfig, KvUpdate, WatchC let final_cursor = watch_applied( watcher, - WatchScope::All, // or WatchScope::Prefix("node.".into()) + WatchScope::All, // or Prefix("node.".into()) / Prefixes(vec![...]) Some(resume), // Option — resume here, or None + Some(reader), // Option> — arms the + // cursor-expired stale-key resync, or None Some(store), // any SnapshotStore (e.g. AppendLogSnapshot), or None + None, // Option> — live exports BatchConfig::default(), // 10ms window, 100 updates per batch |update: &KvUpdate| parse(update), // KvUpdate -> Option; None just drops it |batch: Vec| cache.apply_batch(batch), // your only domain logic @@ -286,7 +298,7 @@ A batch closes when `window` elapses or it hits `max` updates, whichever comes f Persist the cursor on receipt instead and a crash between receive and apply loses data: the cursor reads "caught up to rev N" while rev N sits in an unapplied buffer, and the next resume starts past it. `watch_applied` checkpoints at the applied cursor, so a persisted cursor always means every update up to it has been applied. - `parse` returning `None` (corrupt bytes, irrelevant key) still advances the cursor — nothing to apply means nothing to skip. -- On `CursorExpired`, it falls back to a full watch automatically. +- On `CursorExpired`, it falls back to a full watch automatically. With a `reader` wired, it first diffs the fold against the bucket's live keys and applies synthetic deletes for keys that vanished during the gap (their delete markers were evicted with the cursor) — the one case the fallback re-list can't cover. - It returns the final applied cursor on shutdown or stream close. `apply` runs inline. If it panics, the panic aborts the watch. @@ -301,6 +313,7 @@ Persist the cursor on receipt instead and a crash between receive and apply lose | `delete()` | Writes empty value (soft delete). Always returns `Ok(true)` | | `KvUpdate::Purge` | Hard delete: all history removed from stream | | `scan()` | `DeliverPolicy::LastPerSubject`: one entry per key, one consumer | +| `watch_*()` | `DeliverPolicy::LastPerSubject`: current state, then live updates | | `watch_prefix()` | Native NATS subject filter (`{prefix}>` wildcard) | ## Feature detection diff --git a/benches/applied.rs b/benches/applied.rs index 34651e1..06c1bb6 100644 --- a/benches/applied.rs +++ b/benches/applied.rs @@ -89,6 +89,7 @@ fn bench_batch_throughput(c: &mut Criterion) { watcher as Arc, WatchScope::All, None, + None, // reader: cursor-expired resync not exercised here None::, None, BatchConfig { diff --git a/src/applied.rs b/src/applied.rs index ff0271e..8d8c2b1 100644 --- a/src/applied.rs +++ b/src/applied.rs @@ -59,7 +59,7 @@ use tokio::sync::{oneshot, watch}; use tracing::warn; use crate::artifact::ExportManifest; -use crate::kv::{KvError, KvUpdate, KvWatcher, WatchCursor}; +use crate::kv::{KvError, KvReader, KvUpdate, KvWatcher, WatchCursor}; use crate::snapshot::{SnapshotError, SnapshotStore}; /// A request, sent into a running [`watch_applied`] loop, to export the fold it @@ -83,18 +83,50 @@ pub struct ExportRequest { pub reply: oneshot::Sender>, } -/// What to watch: every key, or every key under a prefix. +/// What to watch: every key, every key under a prefix, or the union of several +/// prefixes. /// /// Mirrors the [`KvWatcher`] surface — `All` maps to `watch_all` / -/// `watch_all_from`, `Prefix` to `watch_prefix` / `watch_prefix_from`. +/// `watch_all_from`, `Prefix` to `watch_prefix` / `watch_prefix_from`, +/// `Prefixes` to `watch_prefixes` / `watch_prefixes_from` (one multi-filter +/// consumer for the whole union, never one consumer per prefix). #[derive(Debug, Clone)] pub enum WatchScope { /// Watch all keys in the bucket. All, /// Watch only keys beginning with this prefix. Prefix(String), + /// Watch keys beginning with ANY of these prefixes, on a single consumer. + Prefixes(Vec), } +impl WatchScope { + /// The scope as a list of key prefixes (`All` = the empty prefix), for + /// callers that enumerate scope contents (live listings, fold ranges). + fn prefixes(&self) -> Vec { + match self { + WatchScope::All => vec![String::new()], + WatchScope::Prefix(p) => vec![p.clone()], + WatchScope::Prefixes(ps) => ps.clone(), + } + } +} + +/// Internal: a cursor-expired resync handoff from the watch task to the main +/// loop. Carries the bucket's live key listing for the watch scope; the main +/// loop diffs it against the fold and applies synthetic deletes for keys that +/// vanished during the gap, then acks so the watch task can start the fallback +/// watch — the ack ordering guarantees every synthetic delete is applied before +/// the first re-list put arrives. +struct ResyncRequest { + live_keys: Vec, + ack: oneshot::Sender<()>, +} + +/// Internal: what the watch task needs to initiate a resync — the reader that +/// lists live keys and the channel into the main loop that owns the fold. +type ResyncHandle = (Arc, mpsc::Sender); + /// Batching policy for [`watch_applied`]. /// /// A flush fires when **either** bound is hit, whichever comes first: `window` @@ -138,9 +170,25 @@ impl Default for BatchConfig { /// whose `apply` had not returned. The store fold is atomic (data + cursor), so a /// crash leaves the store consistent and resume re-folds only the tail. /// +/// # Cursor expiry and stale-key resync +/// /// On [`KvError::CursorExpired`] from the `*_from` resume path, this logs and -/// falls back to a full-scope watch (`watch_all` / `watch_prefix`). Callers see -/// the full re-list as a stream of puts, exactly as the hand-rolled loops did. +/// falls back to a full-scope watch (`watch_all` / `watch_prefix` / +/// `watch_prefixes`), whose state-sync re-list re-delivers the current value of +/// every in-scope key as puts. The re-list alone cannot cover keys that were +/// **deleted during the gap** and whose delete markers the backend has since +/// evicted — they simply don't appear, leaving the fold (and the caller's +/// domain state) holding them forever. +/// +/// When both `reader` and `store` are provided, the expiry path closes that +/// hole: before the fallback watch starts, the bucket's live keys are listed +/// via `reader`, diffed against the fold's in-scope keys, and a synthetic +/// [`KvUpdate::Delete`] (with an unknown [`VersionToken`](crate::VersionToken)) is run through +/// `parse`/`apply`/`store` for each key that vanished. The synthetic deletes +/// are strictly ordered before the first re-list put, so a key deleted and +/// re-created during the gap converges correctly. Without a `reader` (or +/// without a `store` to diff against) the fallback is re-list-only and a +/// warning marks the possible stale keys. /// /// See `ARCHITECTURE.md` ("Applied-Cursor Watch") for the invariant and its /// rationale. @@ -155,12 +203,18 @@ impl Default for BatchConfig { #[allow(clippy::too_many_arguments)] // The flush macro resets `batch_high`/`batch_deadline` for the next loop // iteration. At the two flush sites that return immediately afterward (shutdown, -// channel-close) those resets are dead stores — correct, but flagged. +// channel-close) those resets are dead stores — correct, but flagged. The allow +// must sit on the function: a statement-scoped `#[allow]` inside the macro body +// trips the experimental attributes-on-expressions gate (E0658) on stable. #[allow(unused_assignments)] pub async fn watch_applied( watcher: Arc, scope: WatchScope, resume: Option, + // `Some(reader)` arms the cursor-expired stale-key resync (see the function + // docs); `None` keeps the re-list-only fallback. Only consulted on expiry — + // the hot path never touches it. + reader: Option>, mut store: Option, // `Some(rx)` arms an export-request arm in the select loop: each // [`ExportRequest`] is handled between flushes (pending batch flushed @@ -193,12 +247,31 @@ where None => WatchCursor::none(), }; + // The scope's prefixes, for the resync diff against the fold. Cloned out + // before `scope` moves into the watch task. + let scope_prefixes = scope.prefixes(); + + // Cursor-expired resync channel, armed only when there is a reader to list + // live keys AND a store to diff them against. The watch task sends the live + // listing here and waits for the ack before starting the fallback watch, so + // synthetic deletes always precede the re-list. + let (resync_pair, mut resyncs): (Option, Option>) = + match reader { + Some(reader) if store.is_some() => { + let (rs_tx, rs_rx) = mpsc::channel(1); + (Some((reader, rs_tx)), Some(rs_rx)) + } + _ => (None, None), + }; + // Spawn the watch task. It owns the cursor-expired fallback so the main loop // only ever sees a clean ordered stream of updates on `rx`. let (tx, mut rx) = mpsc::channel::(256); let handle = { let watcher = Arc::clone(&watcher); - tokio::spawn(async move { run_watch(watcher.as_ref(), &scope, resume, tx).await }) + tokio::spawn( + async move { run_watch(watcher.as_ref(), &scope, resume, resync_pair, tx).await }, + ) }; // Batch state. @@ -209,7 +282,9 @@ where // cursor to it after a single atomic `apply` is correct: having seen the max // means we've seen everything below it, and a rejected entry is still // "nothing to apply", hence covered. Reset to `none()` after every flush. - let batch_cap = config.max.clamp(1, 64); + // Pre-size to the flush bound so no batch ever re-climbs the reallocation + // ladder; `max(1)` only guards a nonsensical `max = 0` config. + let batch_cap = config.max.max(1); let mut batch: Vec = Vec::with_capacity(batch_cap); // Raw received updates for the durable `store`, in revision order. Only // populated when a `store` is present; the store folds the *raw* updates @@ -233,7 +308,9 @@ where macro_rules! flush { () => {{ // Nothing received since the last flush → nothing to do at all. - if !batch.is_empty() || !batch_high.is_none() { + // (`raw_batch` can be non-empty with no cursor advance only via the + // resync path's synthetic deletes, which carry no revision.) + if !batch.is_empty() || !raw_batch.is_empty() || !batch_high.is_none() { if !batch.is_empty() { // INVARIANT: apply() runs and RETURNS before any cursor // advance below. Move the batch out so a panicking apply @@ -244,34 +321,44 @@ where // ladder (4→8→…→cap). apply(std::mem::replace(&mut batch, Vec::with_capacity(batch_cap))); } - if !batch_high.is_none() { + let advanced = !batch_high.is_none(); + if advanced { applied = batch_high.clone(); - if let Some(mut st) = store.take() { - let raw = std::mem::take(&mut raw_batch); - let cur = applied.clone(); - // Hand the store back unconditionally on a clean return so - // a *failed* apply (Ok(Err)) keeps the watch running; only - // a *panicked* task (Err) loses the store and is fatal. - match tokio::task::spawn_blocking(move || { - let res = st.apply(&raw, &cur); - (st, res) - }) - .await - { - Ok((st, Ok(()))) => store = Some(st), - Ok((st, Err(e))) => { - warn!(error = %e, "snapshot store apply failed; continuing"); - store = Some(st); - } - Err(e) => { - warn!(error = %e, "snapshot store task panicked; aborting watch"); - handle.abort(); - return Err(KvError::WatchError(format!( - "snapshot store task panicked: {e}" - ))); - } + } + if !raw_batch.is_empty() + && let Some(mut st) = store.take() + { + let raw = std::mem::take(&mut raw_batch); + // Fold at the post-advance cursor. A synthetic-deletes-only + // batch leaves the cursor where it was (the deletes are a + // state correction, not log entries), which is safe: an + // unchanged — possibly expired — cursor only ever re-runs + // this same resync on the next restart. + let cur = applied.clone(); + // Hand the store back unconditionally on a clean return so + // a *failed* apply (Ok(Err)) keeps the watch running; only + // a *panicked* task (Err) loses the store and is fatal. + match tokio::task::spawn_blocking(move || { + let res = st.apply(&raw, &cur); + (st, res) + }) + .await + { + Ok((st, Ok(()))) => store = Some(st), + Ok((st, Err(e))) => { + warn!(error = %e, "snapshot store apply failed; continuing"); + store = Some(st); + } + Err(e) => { + warn!(error = %e, "snapshot store task panicked; aborting watch"); + handle.abort(); + return Err(KvError::WatchError(format!( + "snapshot store task panicked: {e}" + ))); } } + } + if advanced { on_applied(applied.clone()); batch_high = WatchCursor::none(); } @@ -319,6 +406,70 @@ where flush!(); } + // Cursor-expired resync. Placed before `rx.recv()` (biased) so the + // synthetic deletes are folded before any update the fallback watch + // delivers — though the ack protocol already guarantees the fallback + // hasn't started while this arm runs. Diff the fold's in-scope keys + // against the bucket's live listing; anything the fold holds that + // the bucket no longer does vanished during the gap (its delete + // marker evicted with the cursor), so synthesize the delete the + // re-list can't deliver. + req = async { resyncs.as_mut().expect("arm guarded by is_some").recv().await }, + if resyncs.is_some() => { + match req { + Some(ResyncRequest { live_keys, ack }) => { + // Flush first so the diff runs against a fold that + // reflects everything received so far. + flush!(); + let live: std::collections::HashSet<&str> = + live_keys.iter().map(String::as_str).collect(); + let mut stale: Vec = Vec::new(); + if let Some(st) = &store { + for prefix in &scope_prefixes { + match st.range(prefix) { + Ok(entries) => stale.extend( + entries + .into_iter() + .filter(|e| !live.contains(e.key.as_str())) + .map(|e| e.key), + ), + Err(e) => { + warn!(error = %e, prefix = %prefix, + "resync fold range failed; stale keys under this prefix may persist"); + } + } + } + } + // Overlapping prefixes can list a key twice. + stale.sort_unstable(); + stale.dedup(); + if !stale.is_empty() { + warn!(stale = stale.len(), "cursor-expired resync: deleting keys that vanished during the gap"); + } + for key in stale { + // Synthetic: carries no revision (unknown version) + // and so never advances the cursor. + let u = KvUpdate::Delete { + key, + version: crate::kv::VersionToken::unknown(), + }; + if store.is_some() { + raw_batch.push(u.clone()); + } + if let Some(parsed) = parse(&u) { + batch.push(parsed); + } + } + flush!(); + // Ack AFTER the deletes are applied: the watch task is + // holding the fallback watch until it hears back, which + // is what orders deletes before the re-list. + let _ = ack.send(()); + } + None => resyncs = None, + } + } + // Export request. Placed after shutdown/window (they stay prompt) // and before `rx.recv()` so a firehose of updates cannot starve an // export indefinitely. The pending batch is flushed first, so the @@ -430,11 +581,13 @@ where } /// Run the underlying watch for `scope`, resuming from `resume` when it carries -/// a position, with the [`KvError::CursorExpired`] → full-watch fallback. +/// a position, with the [`KvError::CursorExpired`] → resync + full-watch +/// fallback. async fn run_watch( watcher: &dyn KvWatcher, scope: &WatchScope, resume: Option, + resync: Option, tx: mpsc::Sender, ) -> Result<(), KvError> { // Resume only when the cursor carries a real position; an absent or `none()` @@ -448,13 +601,10 @@ async fn run_watch( if let Some(cursor) = resume_cursor { match watcher.watch_all_from(&cursor, tx.clone()).await { Err(KvError::CursorExpired) => { - // TODO(v2): signal a "resync" to the caller so it can - // diff the full re-list against prior state and emit - // synthetic deletes for keys that vanished during the - // gap (see Snapshot::stale_keys). For v1 the full - // re-list is replayed as a stream of puts, matching the - // hand-rolled loops this combinator replaces. - warn!("watch cursor expired, falling back to full watch_all"); + warn!( + "watch cursor expired; resyncing, then falling back to full watch_all" + ); + resync_stale_keys(scope, &resync).await; watcher.watch_all(tx).await } other => other, @@ -467,8 +617,10 @@ async fn run_watch( if let Some(cursor) = resume_cursor { match watcher.watch_prefix_from(prefix, &cursor, tx.clone()).await { Err(KvError::CursorExpired) => { - // TODO(v2): see the watch_all arm above. - warn!("watch cursor expired, falling back to full watch_prefix"); + warn!( + "watch cursor expired; resyncing, then falling back to full watch_prefix" + ); + resync_stale_keys(scope, &resync).await; watcher.watch_prefix(prefix, tx).await } other => other, @@ -477,6 +629,69 @@ async fn run_watch( watcher.watch_prefix(prefix, tx).await } } + WatchScope::Prefixes(prefixes) => { + let refs: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + if let Some(cursor) = resume_cursor { + match watcher + .watch_prefixes_from(&refs, &cursor, tx.clone()) + .await + { + Err(KvError::CursorExpired) => { + warn!( + "watch cursor expired; resyncing, then falling back to full watch_prefixes" + ); + resync_stale_keys(scope, &resync).await; + watcher.watch_prefixes(&refs, tx).await + } + other => other, + } + } else { + watcher.watch_prefixes(&refs, tx).await + } + } + } +} + +/// Cursor-expired stale-key resync, run BEFORE the fallback watch is +/// established: list the scope's live keys, hand them to the main loop (which +/// diffs them against the fold and applies synthetic deletes), and wait for the +/// ack. That ordering — deletes applied, then fallback watch armed — is what +/// makes a delete-then-recreate during the gap converge: the synthetic delete +/// always lands before the re-list put. +/// +/// Best-effort: with no reader/store wired (`resync` is `None`), or a failed +/// listing, this degrades to the warn-and-relist-only fallback — keys deleted +/// during the gap stay in the fold until their next update. +async fn resync_stale_keys(scope: &WatchScope, resync: &Option) { + let Some((reader, resync_tx)) = resync else { + warn!( + "no reader wired for cursor-expired resync; keys deleted during the gap may persist in the fold" + ); + return; + }; + let mut live_keys = Vec::new(); + for prefix in scope.prefixes() { + match reader.keys(&prefix).await { + Ok(keys) => live_keys.extend(keys), + Err(e) => { + warn!(error = %e, prefix = %prefix, + "resync live-key listing failed; keys deleted during the gap may persist in the fold"); + return; + } + } + } + let (ack_tx, ack_rx) = oneshot::channel(); + if resync_tx + .send(ResyncRequest { + live_keys, + ack: ack_tx, + }) + .await + .is_ok() + { + // A dropped ack (main loop shutting down) just means the fallback watch + // is about to die with it; nothing to recover. + let _ = ack_rx.await; } } @@ -585,6 +800,47 @@ mod tests { self.deliver(&self.from, tx).await; Ok(()) } + + // Same mirroring for the multi-prefix resume arm. + async fn watch_prefixes_from( + &self, + _prefixes: &[&str], + _cursor: &WatchCursor, + tx: Sender, + ) -> Result<(), KvError> { + if self.from_expires { + return Err(KvError::CursorExpired); + } + self.deliver(&self.from, tx).await; + Ok(()) + } + } + + /// A reader whose `keys()` serves a scripted live listing — the only call + /// the cursor-expired resync makes. Filters by prefix like a real backend + /// so prefix-scoped resyncs are exercised faithfully. + struct MockReader { + live: Vec, + } + + #[async_trait] + impl KvReader for MockReader { + async fn get(&self, _key: &str) -> Result, KvError> { + unreachable!("resync only lists keys") + } + + async fn keys(&self, prefix: &str) -> Result, KvError> { + Ok(self + .live + .iter() + .filter(|k| k.starts_with(prefix)) + .cloned() + .collect()) + } + + async fn scan(&self, _prefix: &str) -> Result, KvError> { + unreachable!("resync only lists keys") + } } /// A watcher whose entry points all fail. Used to prove the watch task's @@ -638,6 +894,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -673,6 +930,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -716,6 +974,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig { @@ -756,6 +1015,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig { @@ -807,6 +1067,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig { @@ -854,6 +1115,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -898,6 +1160,7 @@ mod tests { watcher, WatchScope::All, Some(WatchCursor::from_u64(5)), // resume position that "expired" + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -917,6 +1180,203 @@ mod tests { ); } + /// Cursor-expired resync: with a reader + store wired, a key the fold holds + /// that the live listing no longer does gets a synthetic delete — applied + /// strictly BEFORE the fallback re-list — and the persisted fold converges + /// to the live state. The synthetic delete (unknown version) must not move + /// the cursor; the re-list put must. + #[tokio::test] + async fn cursor_expired_resync_deletes_stale_keys() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("resync.snap"); + let (_r, mut store) = AppendLogSnapshot::open(&path, u64::MAX).unwrap(); + // The fold from the previous run: node.a and node.b at cursor 2. + store + .apply( + &[put("node.a", b"1", 1), put("node.b", b"2", 2)], + &WatchCursor::from_u64(2), + ) + .unwrap(); + + // During the gap node.b was deleted (marker since evicted) and node.a + // updated; the resume cursor (2) has expired. The fallback re-list + // therefore carries only the surviving key. + let mock = MockWatcher { + full: Mutex::new(Some(vec![put("node.a", b"1b", 10)])), + from: Mutex::new(Some(vec![])), + from_expires: true, + hold: false, + }; + let reader = MockReader { + live: vec!["node.a".to_string()], + }; + + // Record everything `parse` sees, in order, deletes included. + let seen = Arc::new(Mutex::new(Vec::<(String, bool)>::new())); + let s = Arc::clone(&seen); + let (_sd_tx, sd_rx) = watch::channel(false); + + let cursor = watch_applied( + Arc::new(mock), + WatchScope::All, + Some(WatchCursor::from_u64(2)), + Some(Arc::new(reader) as Arc), + Some(store), + None, + BatchConfig::default(), + move |u: &KvUpdate| { + s.lock() + .unwrap() + .push((u.key().to_string(), matches!(u, KvUpdate::Delete { .. }))); + Some(()) + }, + |_batch: Vec<()>| {}, + |_| {}, + sd_rx, + ) + .await + .unwrap(); + + // The re-list put advanced the cursor; the synthetic delete did not. + assert_eq!(cursor.as_u64(), Some(10)); + // The synthetic delete strictly precedes the re-list put. + assert_eq!( + *seen.lock().unwrap(), + vec![("node.b".to_string(), true), ("node.a".to_string(), false)], + "synthetic delete must be applied before the fallback re-list" + ); + + // The persisted fold converged: stale key gone, live key updated. + let snap = crate::snapshot::load(&path).unwrap().unwrap(); + assert_eq!(snap.cursor.as_u64(), Some(10)); + assert_eq!(snap.entries.len(), 1); + assert_eq!(snap.entries["node.a"].value, b"1b"); + } + + /// A prefix-scoped resync diffs only in-scope keys: an out-of-scope key the + /// fold holds survives, the in-scope stale key is deleted, and a flush + /// containing only synthetic deletes leaves the cursor untouched. + #[tokio::test] + async fn cursor_expired_resync_respects_scope() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("resync-scope.snap"); + let (_r, mut store) = AppendLogSnapshot::open(&path, u64::MAX).unwrap(); + store + .apply( + &[put("node.b", b"2", 1), put("other.z", b"9", 2)], + &WatchCursor::from_u64(2), + ) + .unwrap(); + + // Expired resume; the bucket no longer has ANY node.* keys; the + // fallback re-list is empty. + let mock = MockWatcher { + full: Mutex::new(Some(vec![])), + from: Mutex::new(Some(vec![])), + from_expires: true, + hold: false, + }; + let reader = MockReader { live: vec![] }; + let (_sd_tx, sd_rx) = watch::channel(false); + + let cursor = watch_applied( + Arc::new(mock), + WatchScope::Prefix("node.".to_string()), + Some(WatchCursor::from_u64(2)), + Some(Arc::new(reader) as Arc), + Some(store), + None, + BatchConfig::default(), + |_u: &KvUpdate| Some(()), + |_batch: Vec<()>| {}, + |_| {}, + sd_rx, + ) + .await + .unwrap(); + + // Deletes-only flush: cursor stays at the resume position. + assert_eq!(cursor.as_u64(), Some(2)); + + let snap = crate::snapshot::load(&path).unwrap().unwrap(); + assert_eq!(snap.cursor.as_u64(), Some(2)); + assert!( + !snap.entries.contains_key("node.b"), + "in-scope stale key must be resync-deleted" + ); + assert_eq!( + snap.entries["other.z"].value, b"9", + "out-of-scope key must survive a prefix-scoped resync" + ); + } + + /// `WatchScope::Prefixes` dispatches to `watch_prefixes` (no resume) and to + /// `watch_prefixes_from` with the expiry → full-watch fallback (resume). + #[tokio::test] + async fn prefixes_scope_dispatches_full_watch() { + let updates = vec![put("a.x", b"1", 1), put("b.y", b"2", 2)]; + let watcher = Arc::new(MockWatcher::new(updates, false)); + let applied_batches = Arc::new(Mutex::new(Vec::>::new())); + let ab = Arc::clone(&applied_batches); + let (_sd_tx, sd_rx) = watch::channel(false); + + let cursor = watch_applied( + watcher, + WatchScope::Prefixes(vec!["a.".to_string(), "b.".to_string()]), + None, + None, // reader (no resync in this test) + None::, + None, + BatchConfig::default(), + parse_put, + move |batch: Vec>| ab.lock().unwrap().extend(batch), + move |_| {}, + sd_rx, + ) + .await + .unwrap(); + + assert_eq!(cursor.as_u64(), Some(2)); + assert_eq!( + *applied_batches.lock().unwrap(), + vec![b"1".to_vec(), b"2".to_vec()] + ); + } + + /// `WatchScope::Prefixes` resume whose cursor has expired falls back to the + /// full multi-prefix watch and applies its updates. + #[tokio::test] + async fn prefixes_scope_expired_resume_falls_back() { + let mock = MockWatcher { + full: Mutex::new(Some(vec![put("a.x", b"1", 7)])), + from: Mutex::new(Some(vec![])), + from_expires: true, + hold: false, + }; + let applied_batches = Arc::new(Mutex::new(Vec::>::new())); + let ab = Arc::clone(&applied_batches); + let (_sd_tx, sd_rx) = watch::channel(false); + + let cursor = watch_applied( + Arc::new(mock), + WatchScope::Prefixes(vec!["a.".to_string()]), + Some(WatchCursor::from_u64(3)), + None, // reader (no resync in this test) + None::, + None, + BatchConfig::default(), + parse_put, + move |batch: Vec>| ab.lock().unwrap().extend(batch), + move |_| {}, + sd_rx, + ) + .await + .unwrap(); + + assert_eq!(cursor.as_u64(), Some(7)); + assert_eq!(*applied_batches.lock().unwrap(), vec![b"1".to_vec()]); + } + /// End-to-end with a real snapshot file: after the run, the persisted /// snapshot's cursor equals the applied cursor and its entries match the /// applied state — proving the checkpoint is written at the post-apply @@ -935,6 +1395,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) Some(store), None, BatchConfig::default(), @@ -983,6 +1444,7 @@ mod tests { watcher, WatchScope::All, Some(WatchCursor::from_u64(9)), // resume past rev 9 — not expired + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1022,6 +1484,7 @@ mod tests { watcher, WatchScope::Prefix("node.".to_string()), None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1040,6 +1503,53 @@ mod tests { ); } + /// `WatchScope::Prefix` happy-path resume: a non-expired cursor takes the + /// `watch_prefix_from` path and only the delta is applied — the prefix + /// twin of `resume_from_cursor_delivers_only_delta`. + #[tokio::test] + async fn prefix_resume_from_cursor_delivers_only_delta() { + let mock = MockWatcher { + // `full` would be delivered only if the resume path were (wrongly) + // bypassed; a distinguishing value makes that visible. + full: Mutex::new(Some(vec![put("node.x", b"FULL", 1)])), + from: Mutex::new(Some(vec![put("node.c", b"3", 10), put("node.d", b"4", 11)])), + from_expires: false, + hold: false, + }; + let watcher = Arc::new(mock); + + let applied_batches = Arc::new(Mutex::new(Vec::>::new())); + let ab = Arc::clone(&applied_batches); + let (_sd_tx, sd_rx) = watch::channel(false); + + let cursor = watch_applied( + watcher, + WatchScope::Prefix("node.".to_string()), + Some(WatchCursor::from_u64(9)), // resume past rev 9 — not expired + None, // reader (no resync in this test) + None::, + None, + BatchConfig::default(), + parse_put, + move |batch: Vec>| ab.lock().unwrap().extend(batch), + move |_| {}, + sd_rx, + ) + .await + .unwrap(); + + assert_eq!( + cursor.as_u64(), + Some(11), + "cursor advances to the delta max" + ); + assert_eq!( + *applied_batches.lock().unwrap(), + vec![b"3".to_vec(), b"4".to_vec()], + "only the post-cursor delta is applied via watch_prefix_from" + ); + } + /// `WatchScope::Prefix` resume whose cursor has expired falls back to the /// full `watch_prefix` and still applies the delivered updates — the prefix /// twin of `cursor_expired_falls_back_to_full_watch`. @@ -1061,6 +1571,7 @@ mod tests { watcher, WatchScope::Prefix("node.".to_string()), Some(WatchCursor::from_u64(5)), // resume position that "expired" + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1091,6 +1602,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1131,6 +1643,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1178,6 +1691,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, None, BatchConfig::default(), @@ -1228,6 +1742,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) Some(store), Some(ex_rx), BatchConfig { @@ -1270,6 +1785,70 @@ mod tests { task.await.unwrap().unwrap(); } + /// An [`ExportRequest`] that arrives with NOTHING pending (the window + /// already flushed everything) still produces a valid artifact whose + /// cursor is the applied cursor. The flush-before-export step must be a + /// clean no-op, not an error or a cursor regression. + #[tokio::test(start_paused = true)] + async fn export_with_empty_pending_batch_succeeds() { + let dir = tempfile::TempDir::new().unwrap(); + let store_path = dir.path().join("fold.snap"); + let artifact = dir.path().join("artifact"); + let (_r, store) = AppendLogSnapshot::open(&store_path, u64::MAX).unwrap(); + + let updates = vec![put("a", b"1", 1), put("b", b"2", 2)]; + let watcher = Arc::new(MockWatcher::new(updates, true)); // hold open + let (sd_tx, sd_rx) = watch::channel(false); + let (ex_tx, ex_rx) = mpsc::channel(1); + + let task = tokio::spawn(watch_applied( + watcher, + WatchScope::All, + None, + None, // reader (no resync in this test) + Some(store), + Some(ex_rx), + BatchConfig::default(), // 10 ms window + parse_put, + move |_batch: Vec>| {}, + move |_| {}, + sd_rx, + )); + + // Let the window flush both updates, so the export request finds an + // EMPTY pending batch. + tokio::time::sleep(Duration::from_millis(50)).await; + + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + ex_tx + .send(ExportRequest { + dest_dir: artifact.clone(), + reply: reply_tx, + }) + .await + .unwrap(); + let manifest = reply_rx + .await + .unwrap() + .expect("export succeeds with nothing pending"); + assert_eq!( + manifest.cursor.as_u64(), + Some(2), + "artifact cursor is the applied cursor, unchanged by the no-op flush" + ); + + // The artifact is importable and holds the already-flushed entries. + let (cursor, imported) = + AppendLogSnapshot::import(&artifact, &dir.path().join("imported.snap"), u64::MAX) + .unwrap(); + assert_eq!(cursor.as_u64(), Some(2)); + assert_eq!(imported.get("a").unwrap().unwrap().value, b"1"); + assert_eq!(imported.get("b").unwrap().unwrap().value, b"2"); + + sd_tx.send(true).unwrap(); + task.await.unwrap().unwrap(); + } + /// An export request against a store-less watch replies with an error and /// the watch keeps running. #[tokio::test(start_paused = true)] @@ -1282,6 +1861,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, Some(ex_rx), BatchConfig::default(), @@ -1335,6 +1915,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) Some(store), Some(ex_rx), BatchConfig::default(), @@ -1380,6 +1961,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) None::, Some(ex_rx), BatchConfig::default(), @@ -1429,6 +2011,7 @@ mod tests { watcher, WatchScope::All, None, + None, // reader (no resync in this test) Some(store), None, BatchConfig { diff --git a/src/artifact.rs b/src/artifact.rs index c2e121a..3df2677 100644 --- a/src/artifact.rs +++ b/src/artifact.rs @@ -99,7 +99,14 @@ pub struct ArtifactFile { // surface can hold a real WatchCursor while the JSON stays a stable hex string. // --------------------------------------------------------------------------- +// `deny_unknown_fields` on both: the manifest is the trust boundary for a +// remote-supplied artifact, and an unrecognized field is far more likely a +// corrupted/hostile manifest or a schema mismatch than benign noise — reject +// loudly rather than skip silently. Forward compatibility is governed by +// `schema_version`, which is checked first in `manifest_from_slice`; a future +// schema that adds fields must bump it. #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct ManifestWire { schema_version: u32, backend: String, @@ -112,6 +119,7 @@ struct ManifestWire { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct FileWire { path: String, size: u64, @@ -279,21 +287,21 @@ fn validate_payload_path(p: &str) -> Result<(), SnapshotError> { // Payload hashing // --------------------------------------------------------------------------- -/// Streaming BLAKE3 of one file, returning `(size, hex_digest)`. -fn hash_file(path: &Path) -> Result<(u64, String), SnapshotError> { +/// Streaming BLAKE3 of one file, returning `(size, hex_digest)` plus the open +/// handle so the caller can `sync_all` without a second `open(2)`. +fn hash_file(path: &Path, buf: &mut [u8]) -> Result<(File, u64, String), SnapshotError> { let mut file = File::open(path)?; let mut hasher = blake3::Hasher::new(); - let mut buf = vec![0u8; HASH_BUF]; let mut size = 0u64; loop { - let n = file.read(&mut buf)?; + let n = file.read(buf)?; if n == 0 { break; } size += n as u64; hasher.update(&buf[..n]); } - Ok((size, hasher.finalize().to_hex().to_string())) + Ok((file, size, hasher.finalize().to_hex().to_string())) } /// Every regular file under `root/data/`, relative `/`-separated paths, sorted. @@ -329,12 +337,13 @@ fn list_payload_files(root: &Path) -> Result, SnapshotError> { /// (files + directories), returning the manifest file list in sorted order. pub(crate) fn hash_payload(root: &Path) -> Result, SnapshotError> { let mut files = Vec::new(); + let mut buf = vec![0u8; HASH_BUF]; for abs in list_payload_files(root)? { - let (size, blake3) = hash_file(&abs)?; + let (file, size, blake3) = hash_file(&abs, &mut buf)?; // Durability before the rename: the artifact's completeness contract is // "exists ⇒ verifiable", which only holds if the hashed bytes are the // on-disk bytes. - File::open(&abs)?.sync_all()?; + file.sync_all()?; let rel = abs .strip_prefix(root) .map_err(|_| SnapshotError::Backend("payload path escaped artifact root".into()))?; @@ -400,7 +409,14 @@ pub(crate) fn check_dest_available(dest: &Path) -> Result<(), SnapshotError> { /// Remove `dest` if it is an (already-verified-empty) directory so a rename can /// land on its path, then rename `from` onto it and fsync the parent. -fn rename_into_place(from: &Path, dest: &Path) -> Result<(), SnapshotError> { +/// +/// The is_dir → remove_dir → rename sequence has a TOCTOU window: a concurrent +/// writer can recreate `dest` between the remove and the rename. That race +/// fails closed — `remove_dir` errors on a non-empty dir and `rename` errors +/// when `dest` reappears as a file or non-empty dir — so the worst case is an +/// error return, never a silent overwrite. Don't "fix" this with +/// remove-then-retry; refusing the round is the intended behavior. +pub(crate) fn rename_into_place(from: &Path, dest: &Path) -> Result<(), SnapshotError> { if dest.is_dir() { fs::remove_dir(dest)?; } @@ -585,7 +601,9 @@ pub(crate) fn verify_and_stage_import( }; // Copy-while-hashing: one read pass per file serves both the copy and the - // verification. + // verification. One buffer for the whole loop — a fresh 1 MiB zero-init per + // file would be O(files) wasted work on multi-hundred-file artifacts. + let mut buf = vec![0u8; HASH_BUF]; for f in &manifest.files { let src_path = artifact_dir.join(&f.path); let dst_path = stage.dir.path().join(&f.path); @@ -601,7 +619,6 @@ pub(crate) fn verify_and_stage_import( })?; let mut dst = File::create(&dst_path)?; let mut hasher = blake3::Hasher::new(); - let mut buf = vec![0u8; HASH_BUF]; let mut size = 0u64; loop { let n = src.read(&mut buf)?; @@ -758,6 +775,67 @@ mod tests { } } + #[test] + fn rejects_malformed_manifest_json() { + // Truly unparseable bytes (not merely wrong field values) must surface + // as ArtifactInvalid, never an Io/Backend error or a panic. + let dir = TempDir::new().unwrap(); + write_raw_manifest(dir.path(), "not json at all {{{"); + match read_manifest(dir.path()) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("malformed"), "{msg}"); + } + other => panic!("expected ArtifactInvalid, got {other:?}"), + } + } + + /// A symlink in the payload is refused at export: it would hash as its + /// target's bytes but restore as a link (or escape the payload entirely), + /// so `hash_payload` must reject it before a manifest is ever written. + #[cfg(unix)] + #[test] + fn hash_payload_rejects_symlink() { + let dir = TempDir::new().unwrap(); + let payload = dir.path().join(PAYLOAD_DIR); + fs::create_dir(&payload).unwrap(); + fs::write(payload.join("real"), b"data").unwrap(); + let target = dir.path().join("outside"); + fs::write(&target, b"outside the payload").unwrap(); + std::os::unix::fs::symlink(&target, payload.join("link")).unwrap(); + + match hash_payload(dir.path()) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("non-regular"), "{msg}"); + } + other => panic!("expected ArtifactInvalid, got {other:?}"), + } + } + + /// The TOCTOU window documented on `rename_into_place`: the destination + /// appears (non-empty) between `ExportStage::new` and `seal_and_finalize`. + /// The race must fail closed — an error return, never a silent overwrite. + #[test] + fn export_stage_fails_closed_when_dest_appears_before_seal() { + let dir = TempDir::new().unwrap(); + let dest = dir.path().join("artifact"); + let stage = ExportStage::new(&dest).unwrap(); + fs::create_dir(stage.payload()).unwrap(); + fs::write(stage.payload().join("fold.snap"), b"data").unwrap(); + + // A concurrent writer lands a non-empty directory at the destination. + fs::create_dir(&dest).unwrap(); + fs::write(dest.join("stray"), b"x").unwrap(); + + let err = stage + .seal_and_finalize("append-log", "2", &WatchCursor::from_u64(1)) + .unwrap_err(); + assert!(matches!(err, SnapshotError::ArtifactInvalid(_))); + assert!( + dest.join("stray").exists(), + "occupied destination is untouched" + ); + } + #[test] fn hex_round_trips() { for bytes in [&[][..], &[0u8][..], &[0xde, 0xad, 0xbe, 0xef][..]] { diff --git a/src/export_lease.rs b/src/export_lease.rs index 2f5d6b9..5ee2e40 100644 --- a/src/export_lease.rs +++ b/src/export_lease.rs @@ -82,6 +82,10 @@ pub struct LeaseGuard { key: String, record: LeaseRecord, version: VersionToken, + /// Set by [`complete`](Self::complete) / [`abandon`](Self::abandon). A + /// guard dropped without either (early `?`, cancelled future) leaks the + /// round — the fleet waits out the ttl — so [`Drop`] logs it. + resolved: bool, } impl ExportLease { @@ -168,10 +172,19 @@ impl ExportLease { } /// Read the current lease record, if any — the fleet-visible "last export" - /// state. `None` when no round has ever run (or the key was tombstoned). + /// state. `None` when no round has ever run (or the key was tombstoned); + /// [`KvError::SerializationError`] when the key holds unparseable bytes — + /// distinct from `None` so an operator can see "present but corrupt" (a + /// state [`try_acquire`](Self::try_acquire) will repair by takeover) rather + /// than a false "never ran". pub async fn current(&self) -> Result, KvError> { match self.reader.get(&self.key).await? { - Some(entry) => Ok(serde_json::from_slice(&entry.value).ok()), + Some(entry) => serde_json::from_slice(&entry.value).map(Some).map_err(|e| { + KvError::SerializationError(format!( + "lease key {:?} holds an unparseable value: {e}", + self.key + )) + }), None => Ok(None), } } @@ -182,6 +195,7 @@ impl ExportLease { key: self.key.clone(), record, version, + resolved: false, } } } @@ -200,7 +214,8 @@ impl LeaseGuard { /// Best-effort: a CAS conflict (someone already took over) or write error /// is logged, not surfaced — worst case the round waits out its ttl, which /// is the no-abandon behavior anyway. - pub async fn abandon(self) { + pub async fn abandon(mut self) { + self.resolved = true; match self .writer .delete_with_version(&self.key, &self.version) @@ -228,6 +243,7 @@ impl LeaseGuard { /// taken over (this round overran its ttl) and is logged, not surfaced — /// the artifact is already safe wherever the caller put it. pub async fn complete(mut self, cursor: &WatchCursor) -> Result<(), KvError> { + self.resolved = true; self.record.completed_cursor_hex = Some(hex_encode(cursor.version().as_bytes())); self.record.completed_at_unix = Some(unix_now()); let bytes = serde_json::to_vec(&self.record) @@ -247,9 +263,33 @@ impl LeaseGuard { } } +impl Drop for LeaseGuard { + fn drop(&mut self) { + if !self.resolved { + warn!( + key = %self.key, + holder = %self.record.holder_id, + "LeaseGuard dropped without complete() or abandon(); the fleet waits out the lease ttl" + ); + } + } +} + fn unix_now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) + match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(d) => d.as_secs(), + Err(_) => { + // A pre-epoch clock means every lease this node writes is already + // expired (`expires_at = 0 + ttl` is in the past), so any node can + // steal it — duplicate exports every round, which the lease design + // tolerates (dedup, not correctness). That direction is deliberate: + // the alternative sentinel (`u64::MAX`) would mint a never-expiring + // lease that wedges the fleet until manual cleanup. But it must not + // be silent — duplicate artifacts with no log line is undebuggable. + warn!( + "system clock predates the Unix epoch; lease expiry math degraded (expect duplicate export rounds until the clock is fixed)" + ); + 0 + } + } } diff --git a/src/kv.rs b/src/kv.rs index bf7369a..b943d5d 100644 --- a/src/kv.rs +++ b/src/kv.rs @@ -131,13 +131,13 @@ impl VersionToken { /// Create from FDB versionstamp (10 bytes). /// - /// `pub(crate)` until a FoundationDB backend ships and the round-trip is + /// `cfg(test)` until a FoundationDB backend ships and the round-trip is /// tested end-to-end: a 10-byte token has no `as_u64()`, so handing one to /// the NATS backend's CAS path yields an unactionable `OperationFailed`. - /// Exposed within the crate for the snapshot length-prefixed-version tests. - // Only the test suite constructs one today; retained as the seam the FDB - // backend will use, so `allow(dead_code)` rather than deletion. - #[allow(dead_code)] + /// Today it exists only for the snapshot length-prefixed-version tests; an + /// FDB backend should lift the gate (and the visibility) rather than add a + /// second constructor. + #[cfg(test)] pub(crate) fn from_fdb_versionstamp(vs: &[u8; 10]) -> Self { Self { len: 10, buf: *vs } } @@ -252,13 +252,20 @@ pub trait KvReader: Send + Sync { } /// Watch capability - optional, not all stores support real-time updates. +/// +/// The non-`_from` watches are **state-sync** streams: they first deliver the +/// current value of every matching key (the "re-list", as a stream of puts plus +/// any surviving delete markers), then live updates. A consumer starting with +/// no cursor therefore converges on the full bucket state without a separate +/// scan — and without the scan-to-watch race a separate scan would open. The +/// `_from` variants skip the re-list and deliver only the delta past the cursor. #[async_trait] pub trait KvWatcher: Send + Sync { - /// Watch all keys for changes. Sends updates through the channel. - /// Returns when the watch ends or an error occurs. + /// Watch all keys: current state first, then live changes. Sends updates + /// through the channel. Returns when the watch ends or an error occurs. async fn watch_all(&self, tx: Sender) -> Result<(), KvError>; - /// Watch keys matching a prefix. + /// Watch keys matching a prefix: current state first, then live changes. async fn watch_prefix(&self, prefix: &str, tx: Sender) -> Result<(), KvError>; /// Watch keys matching ANY of `prefixes`, delivered through one channel. @@ -298,6 +305,27 @@ pub trait KvWatcher: Send + Sync { let _ = cursor; self.watch_prefix(prefix, tx).await } + + /// Resume watching the union of `prefixes` from a previously saved cursor. + /// + /// Same single-consumer contract as [`watch_prefixes`](Self::watch_prefixes), + /// same delta semantics as the other `_from` variants: only updates past the + /// cursor are delivered, or [`KvError::CursorExpired`] if the backend has + /// compacted past it. + /// + /// Default implementation ignores the cursor and delegates to + /// `watch_prefixes()` — correct (the state-sync re-list is a superset of any + /// delta) but a full replay; backends that can seek a multi-filter stream + /// should override it. + async fn watch_prefixes_from( + &self, + prefixes: &[&str], + cursor: &WatchCursor, + tx: Sender, + ) -> Result<(), KvError> { + let _ = cursor; + self.watch_prefixes(prefixes, tx).await + } } /// Write operations - optional, edge proxy is primarily read-only. diff --git a/src/lib.rs b/src/lib.rs index a4739af..48d8be1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ //! ``` #![deny(unsafe_code)] +#![deny(unused_must_use)] mod applied; mod artifact; diff --git a/src/nats.rs b/src/nats.rs index aab50ff..5c446cd 100644 --- a/src/nats.rs +++ b/src/nats.rs @@ -124,6 +124,17 @@ pub async fn nats_connect( opts.connect(dial_url).await } +/// Render an untrusted server payload for logging: borrowed as-is when valid +/// UTF-8, lowercase hex otherwise. `from_utf8_lossy` would mash every invalid +/// byte into U+FFFD — exactly the bytes an incident needs to see — so the +/// fallback preserves them losslessly instead. +fn payload_for_log(payload: &[u8]) -> std::borrow::Cow<'_, str> { + match std::str::from_utf8(payload) { + Ok(s) => std::borrow::Cow::Borrowed(s), + Err(_) => std::borrow::Cow::Owned(format!("0x{}", crate::artifact::hex_encode(payload))), + } +} + /// Configuration for NATS connection. /// /// `Debug` is hand-written, not derived: `creds` holds decoded credential @@ -205,8 +216,7 @@ pub(crate) async fn create_kv_bucket_raw( .await .map_err(|e| KvError::ConnectionFailed(format!("failed to send create request: {}", e)))?; - let response_str = String::from_utf8_lossy(&response.payload); - debug!(bucket, response = %response_str, "raw JetStream response"); + debug!(bucket, response = %payload_for_log(&response.payload), "raw JetStream response"); match classify_raw_create_response(&response.payload) { RawCreateOutcome::AlreadyExists => { @@ -259,7 +269,7 @@ fn classify_raw_create_response(payload: &[u8]) -> RawCreateOutcome { // Do not remove that re-verify without making this path return `Failed`. let Ok(json) = serde_json::from_slice::(payload) else { warn!( - response = %String::from_utf8_lossy(payload), + response = %payload_for_log(payload), "unparseable STREAM.CREATE response; assuming created (caller re-verifies via get_key_value)" ); return RawCreateOutcome::Created; @@ -657,6 +667,9 @@ impl KvStore for NatsKvStore { fn watcher(&self) -> Option> { Some(Arc::new(NatsKvWatcher { kv: self.kv.clone(), + client: self.client.clone(), + js: self.js.clone(), + bucket: self.name.clone(), })) } @@ -1004,34 +1017,89 @@ async fn stream_watch( /// If these messages ever change, `cursor_expired_matches_known_nats_error_strings` /// is the canary that fails loudly on the next dependency bump. fn is_cursor_expired_error(err: &str) -> bool { - // Case-insensitive substring match without allocating a lowercased copy of - // the (cold-path) error string. The needles are already lowercase. - const NEEDLES: [&str; 4] = [ - "start sequence", - "first sequence", - "sequence not found", - "too old", - ]; - let haystack = err.as_bytes(); - NEEDLES.iter().any(|needle| { - let n = needle.as_bytes(); - haystack.len() >= n.len() && haystack.windows(n.len()).any(|w| w.eq_ignore_ascii_case(n)) - }) + use std::sync::OnceLock; + // One Aho-Corasick automaton over all needles: a single pass over the error + // string regardless of how many needles accumulate as NATS versions reword + // their messages, vs. one `windows()` scan per needle. Case-insensitivity is + // baked into the automaton, so no lowercased copy is allocated either. + static MATCHER: OnceLock = OnceLock::new(); + MATCHER + .get_or_init(|| { + aho_corasick::AhoCorasick::builder() + .ascii_case_insensitive(true) + .build([ + "start sequence", + "first sequence", + "sequence not found", + "too old", + ]) + .expect("static needle set always compiles") + }) + .is_match(err) } struct NatsKvWatcher { kv: Store, + // `watch_prefixes_from` has no async-nats equivalent (there is no + // `watch_many_from_revision`), so it hand-builds the multi-filter ordered + // consumer itself — which needs the raw client (inbox allocation), the + // JetStream context (stream lookup), and the bucket name (subject filters), + // same as the reader's scan path. + client: async_nats::Client, + js: async_nats::jetstream::Context, + bucket: String, +} + +/// Decode a raw KV stream message (as delivered by a hand-built ordered push +/// consumer) into a [`KvUpdate`] — the same mapping `async-nats`'s `kv::Watch` +/// performs internally for the `watch_*` paths: key from the subject (stripping +/// the `$KV.{bucket}.` prefix), operation from the `KV-Operation` header +/// (absent = Put), revision from the stream sequence in the ACK reply subject. +/// +/// Returns `None` for a subject outside the bucket's keyspace, which a +/// subject-filtered consumer should never deliver — skipped rather than +/// surfaced, matching `kv::Watch`'s behavior. +fn kv_message_to_update(msg: &async_nats::Message, kv_prefix: &str) -> Option { + let key = msg.subject.strip_prefix(kv_prefix)?.to_string(); + let revision = msg + .reply + .as_deref() + .and_then(stream_sequence_from_ack) + .unwrap_or(0); + let version = VersionToken::from_u64(revision); + let operation = msg + .headers + .as_ref() + .and_then(|h| h.get("KV-Operation")) + .map(|v| v.as_str()); + Some(match operation { + Some("DEL") => KvUpdate::Delete { key, version }, + Some("PURGE") => KvUpdate::Purge { key, version }, + // No header (or an explicit "PUT") is a put — the common case carries + // no KV-Operation header at all. + _ => KvUpdate::Put(KvEntry { + key, + value: msg.payload.to_vec(), + version, + }), + }) } #[async_trait] impl KvWatcher for NatsKvWatcher { async fn watch_all(&self, tx: Sender) -> Result<(), KvError> { + // `watch_with_history` (DeliverPolicy::LastPerSubject), NOT `watch_all` + // (DeliverPolicy::New): the trait contract is state-sync — current value + // of every key first, then live updates. async-nats's `watch_all` only + // delivers messages published AFTER the consumer exists, which would + // leave a no-cursor consumer empty until keys happen to change. + // // Bound the watch *setup* with `timed()` for the same reason every KV op // is bounded: a half-dead (CLOSE_WAIT) NATS connection parks this await // forever instead of failing. The streaming drain in `stream_watch` is // intentionally unbounded (a watch is long-lived), but establishing it // must not be able to hang a reconnecting caller. - let watcher = timed(self.kv.watch_all()) + let watcher = timed(self.kv.watch_with_history(">")) .await? .map_err(|e| KvError::WatchError(e.to_string()))?; stream_watch(watcher, &tx).await @@ -1040,8 +1108,9 @@ impl KvWatcher for NatsKvWatcher { async fn watch_prefix(&self, prefix: &str, tx: Sender) -> Result<(), KvError> { // Use native NATS subject-based filtering. KV key "node.abc" maps to // subject "$KV.BUCKET.node.abc", and ">" is the multi-level wildcard. + // `_with_history` for the same state-sync contract as `watch_all`. let nats_key = format!("{prefix}>"); - let watcher = timed(self.kv.watch(&nats_key)) + let watcher = timed(self.kv.watch_with_history(&nats_key)) .await? .map_err(|e| KvError::WatchError(e.to_string()))?; stream_watch(watcher, &tx).await @@ -1055,13 +1124,14 @@ impl KvWatcher for NatsKvWatcher { return Ok(()); } // ONE multi-filter consumer for every prefix (NATS 2.10 `filter_subjects`) - // rather than one consumer per prefix. `watch_many` builds a single ordered - // push consumer with `filter_subjects = [{p}> ...]` and yields the same - // `Entry` stream as `watch`, so `stream_watch` is reused verbatim. This is - // the per-stream-consumer-count fix: a node scoped to N prefixes costs 1 - // consumer, not N. + // rather than one consumer per prefix. `watch_many_with_history` builds a + // single ordered push consumer with `filter_subjects = [{p}> ...]` and + // yields the same `Entry` stream as `watch`, so `stream_watch` is reused + // verbatim. This is the per-stream-consumer-count fix: a node scoped to N + // prefixes costs 1 consumer, not N. `_with_history` for the same + // state-sync contract as `watch_all`. let keys: Vec = prefixes.iter().map(|p| format!("{p}>")).collect(); - let watcher = timed(self.kv.watch_many(keys)) + let watcher = timed(self.kv.watch_many_with_history(keys)) .await? .map_err(|e| KvError::WatchError(e.to_string()))?; stream_watch(watcher, &tx).await @@ -1120,6 +1190,100 @@ impl KvWatcher for NatsKvWatcher { info!(revision, prefix, "resumed prefix watch from cursor"); stream_watch(watcher, &tx).await } + + async fn watch_prefixes_from( + &self, + prefixes: &[&str], + cursor: &WatchCursor, + tx: Sender, + ) -> Result<(), KvError> { + use async_nats::jetstream::consumer::{DeliverPolicy, ReplayPolicy, push}; + + if prefixes.is_empty() { + // Same guard as watch_prefixes: an empty filter set must not become + // an unfiltered whole-bucket consumer. + return Ok(()); + } + let revision = match cursor.as_u64() { + Some(rev) if rev > 0 => rev, + _ => return self.watch_prefixes(prefixes, tx).await, + }; + + // async-nats has `watch_many` (multi-filter) and `watch_from_revision` + // (seek) but no combination of the two, so build the multi-filter + // ordered push consumer ourselves — the exact consumer + // `watch_many_with_deliver_policy` would build, with + // `ByStartSequence(cursor+1)` for the delta seek. The ordered-consumer + // machinery (gap detection, auto-recreate from the last delivered + // sequence) comes with `OrderedConfig` for free. + let bucket = self.bucket.as_str(); + let kv_prefix = format!("$KV.{bucket}."); + let filter_subjects: Vec = prefixes + .iter() + .map(|p| format!("{kv_prefix}{p}>")) + .collect(); + + let stream = timed(self.js.get_stream(format!("KV_{bucket}"))) + .await? + .map_err(|e| KvError::WatchError(format!("get KV stream: {e}")))?; + + let consumer = match timed(stream.create_consumer(push::OrderedConfig { + deliver_subject: self.client.new_inbox(), + description: Some("kv multi-prefix resume consumer".to_string()), + filter_subjects, + replay_policy: ReplayPolicy::Instant, + deliver_policy: DeliverPolicy::ByStartSequence { + start_sequence: revision + 1, + }, + ..Default::default() + })) + .await? + { + Ok(c) => c, + Err(e) => { + // Same expiry classification as watch_all_from: a start sequence + // the stream has compacted past surfaces as a consumer-create + // error whose message names the sequence problem. + let err_str = e.to_string(); + if is_cursor_expired_error(&err_str) { + warn!(revision, ?prefixes, error = %err_str, "cursor expired for multi-prefix watch, caller should fall back"); + return Err(KvError::CursorExpired); + } + return Err(KvError::WatchError(err_str)); + } + }; + + let mut messages = timed(consumer.messages()) + .await? + .map_err(|e| KvError::WatchError(e.to_string()))?; + + info!( + revision, + ?prefixes, + "resumed multi-prefix watch from cursor" + ); + while let Some(msg) = messages.next().await { + match msg { + Ok(msg) => { + // A subject-filtered consumer only delivers in-keyspace + // subjects; `None` here would be a server bug, skipped to + // match kv::Watch's tolerance. + let Some(update) = kv_message_to_update(&msg, &kv_prefix) else { + continue; + }; + if tx.send(update).await.is_err() { + debug!("watch receiver closed"); + break; + } + } + Err(e) => { + error!(error = %e, "NATS KV multi-prefix watch error"); + return Err(KvError::WatchError(e.to_string())); + } + } + } + Ok(()) + } } struct NatsKvWriterImpl { @@ -1206,7 +1370,11 @@ impl std::fmt::Debug for NatsConnection { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("NatsConnection") .field("url", &self.config.url) - .field("healthy", &self.healthy.load(Ordering::Relaxed)) + // `Acquire` to match every other read of `healthy` — a `Relaxed` + // outlier here reads like a deliberate exception during an atomics + // audit, and the fmt path is far too cold for the ordering to cost + // anything. + .field("healthy", &self.healthy.load(Ordering::Acquire)) .finish() } } @@ -1342,6 +1510,78 @@ mod tests { assert!(!is_cursor_expired_error("stream not found")); } + fn raw_kv_msg( + subject: &str, + reply: Option<&str>, + payload: &[u8], + op: Option<&str>, + ) -> async_nats::Message { + let headers = op.map(|op| { + let mut h = async_nats::HeaderMap::new(); + h.insert("KV-Operation", op); + h + }); + async_nats::Message { + subject: subject.to_string().into(), + reply: reply.map(|r| r.to_string().into()), + payload: payload.to_vec().into(), + headers, + status: None, + description: None, + length: 0, + } + } + + const ACK_42: &str = "$JS.ACK.KV_certs.cons.1.42.7.1700000000000000000.0"; + + #[test] + fn kv_message_decodes_put_without_operation_header() { + // The common case: a put carries no KV-Operation header at all. + let msg = raw_kv_msg("$KV.certs.node.a", Some(ACK_42), b"v1", None); + match kv_message_to_update(&msg, "$KV.certs.").expect("in keyspace") { + KvUpdate::Put(e) => { + assert_eq!(e.key, "node.a"); + assert_eq!(e.value, b"v1"); + assert_eq!(e.version.as_u64(), Some(42)); + } + other => panic!("expected Put, got {other:?}"), + } + } + + #[test] + fn kv_message_decodes_delete_and_purge_markers() { + let msg = raw_kv_msg("$KV.certs.node.a", Some(ACK_42), b"", Some("DEL")); + assert!(matches!( + kv_message_to_update(&msg, "$KV.certs.").expect("in keyspace"), + KvUpdate::Delete { ref key, ref version } if key == "node.a" && version.as_u64() == Some(42) + )); + + let msg = raw_kv_msg("$KV.certs.node.a", Some(ACK_42), b"", Some("PURGE")); + assert!(matches!( + kv_message_to_update(&msg, "$KV.certs.").expect("in keyspace"), + KvUpdate::Purge { ref key, .. } if key == "node.a" + )); + } + + #[test] + fn kv_message_outside_keyspace_is_skipped() { + // A subject-filtered consumer should never deliver this; the decode + // skips rather than mis-keys it. + let msg = raw_kv_msg("$KV.other.node.a", Some(ACK_42), b"v", None); + assert!(kv_message_to_update(&msg, "$KV.certs.").is_none()); + } + + #[test] + fn kv_message_without_reply_gets_revision_zero() { + // No ACK reply subject → revision unparseable → 0, the same "unknown + // version" convention scan() uses. + let msg = raw_kv_msg("$KV.certs.node.a", None, b"v", None); + match kv_message_to_update(&msg, "$KV.certs.").expect("in keyspace") { + KvUpdate::Put(e) => assert_eq!(e.version.as_u64(), Some(0)), + other => panic!("expected Put, got {other:?}"), + } + } + #[test] fn raw_create_already_exists_when_10058_in_code_field() { // Some Synadia Cloud deployments echo 10058 in `code` rather than diff --git a/src/snapshot.rs b/src/snapshot.rs index de82d59..bd43f8c 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -363,7 +363,9 @@ pub trait SnapshotStore: Sized + Send { /// Buffers the whole match set into a `Vec`. Convenient for the bounded /// prefixes an in-RAM consumer scans, but a broad prefix against an on-disk /// fold (the 1B-route case an on-disk backend exists for) materializes every - /// match at once — use [`for_each_in_range`](Self::for_each_in_range) there. + /// match at once — and an empty `prefix` is an unbounded full scan that + /// buffers the *entire* fold. Use + /// [`for_each_in_range`](Self::for_each_in_range) for either. fn range(&self, prefix: &str) -> Result, SnapshotError>; /// Stream every live entry whose key starts with `prefix`, in ascending key diff --git a/src/snapshot_rocksdb.rs b/src/snapshot_rocksdb.rs index f8c74de..e161303 100644 --- a/src/snapshot_rocksdb.rs +++ b/src/snapshot_rocksdb.rs @@ -398,7 +398,11 @@ impl RocksDbSnapshot { /// `dest_dir`; a bad artifact never becomes a fold. A crash mid-import /// leaves nothing at `dest_dir`; a crash after the final rename leaves a /// fully valid fold (a retried import then refuses the existing - /// destination — just [`open`](Self::open) it). + /// destination — just [`open`](Self::open) it). The same recovery applies + /// if `import` returns an error *after* the rename (the final open of + /// `dest_dir` failing, e.g. on a transient resource limit): the verified + /// fold is already in place, so call [`open`](Self::open) directly rather + /// than retrying `import`. /// /// Unlike the fjall/append-log imports, the manifest's `backend_version` is /// **not** gated: it records the rust-rocksdb binding version for @@ -539,9 +543,11 @@ impl SnapshotStore for RocksDbSnapshot { // outraces its data. let mut wb = WriteBatch::default(); // One scratch buffer reused across the whole batch. `put_cf` copies the - // bytes into the batch's internal representation before returning, so the - // buffer is free to be refilled for the next entry. That turns N per-`Put` - // assembly allocations into one amortized allocation. + // bytes into the batch's internal representation before returning, and + // `encode_value_into` clears `buf` before refilling it (its documented + // contract — stale bytes from the previous entry can never leak into the + // next value). That turns N per-`Put` assembly allocations into one + // amortized allocation. let mut scratch = Vec::new(); for update in batch { match update { diff --git a/src/transport.rs b/src/transport.rs index f4d7a3e..179933d 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -33,7 +33,9 @@ use tokio::sync::{mpsc, oneshot}; use tracing::warn; use crate::applied::ExportRequest; -use crate::artifact::{ExportManifest, MANIFEST_FILE, check_dest_available, manifest_from_slice}; +use crate::artifact::{ + ExportManifest, MANIFEST_FILE, check_dest_available, manifest_from_slice, rename_into_place, +}; use crate::export_lease::ExportLease; use crate::kv::WatchCursor; use crate::snapshot::SnapshotError; @@ -46,6 +48,40 @@ const CHUNK: usize = 8 << 20; /// `CHUNK × MAX_CONCURRENT_PARTS`). const MAX_CONCURRENT_PARTS: usize = 8; +/// Cap on a sibling-manifest object's size. The transport is untrusted, and a +/// manifest read buffers the whole object — without a cap, a hostile or +/// corrupted object at the manifest key is an OOM vector. Real manifests are +/// a few KB per payload file; 1 MiB is orders of magnitude of headroom. +const MAX_MANIFEST_BYTES: usize = 1 << 20; + +/// Per-await timeout on every object-store operation (each request, chunk, or +/// part — not the whole transfer, which is legitimately unbounded for large +/// artifacts). A half-dead TCP connection otherwise parks the `await` forever; +/// same hazard and same 30 s bound as the NATS layer's `timed()`. +const OP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Bound one object-store await by `limit`. +async fn timed_by( + what: &str, + limit: Duration, + fut: impl std::future::Future, +) -> Result { + tokio::time::timeout(limit, fut).await.map_err(|_| { + SnapshotError::Backend(format!( + "object store: {what} timed out after {}s", + limit.as_secs() + )) + }) +} + +/// Bound one object-store await by [`OP_TIMEOUT`]. +async fn timed( + what: &str, + fut: impl std::future::Future, +) -> Result { + timed_by(what, OP_TIMEOUT, fut).await +} + /// Ship artifacts to durable storage and fetch them back. See the module docs /// for the wire format. #[async_trait] @@ -117,6 +153,25 @@ impl ObjectStoreTransport { fn manifest_path(&self, key: &str) -> ObjPath { ObjPath::from(format!("{}/{key}.manifest.json", self.prefix)) } + + /// Fetch the sibling manifest object, enforcing [`MAX_MANIFEST_BYTES`]. + async fn fetch_manifest_bytes(&self, key: &str) -> Result, SnapshotError> { + let mut stream = timed("manifest get", self.store.get(&self.manifest_path(key))) + .await? + .map_err(map_obj)? + .into_stream(); + let mut buf = Vec::new(); + while let Some(chunk) = timed("manifest read", stream.next()).await? { + let chunk = chunk.map_err(map_obj)?; + if buf.len() + chunk.len() > MAX_MANIFEST_BYTES { + return Err(SnapshotError::ArtifactInvalid(format!( + "remote manifest for {key:?} exceeds {MAX_MANIFEST_BYTES} bytes" + ))); + } + buf.extend_from_slice(&chunk); + } + Ok(buf) + } } #[async_trait] @@ -133,11 +188,23 @@ impl ArtifactTransport for ObjectStoreTransport { // Tar the artifact into a temp file on a blocking task. A temp file // (rather than streaming the tar straight into the upload) keeps the // blocking tar writer and the async multipart writer decoupled; the - // disk cost is one tar's worth, transient. + // disk cost is one tar's worth, transient. Staged BESIDE the artifact, + // not in the system temp dir — /tmp is often a size-bounded tmpfs that + // a multi-GB artifact tar would exhaust. let src = artifact_dir.to_path_buf(); + let tar_parent = artifact_dir + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .ok_or_else(|| { + SnapshotError::ArtifactInvalid(format!( + "artifact {} has no parent directory to stage the tar in", + artifact_dir.display() + )) + })? + .to_path_buf(); let tar_file = tokio::task::spawn_blocking( move || -> Result { - let tmp = tempfile::NamedTempFile::new()?; + let tmp = tempfile::NamedTempFile::new_in(&tar_parent)?; let mut builder = tar::Builder::new(std::io::BufWriter::new(tmp.reopen()?)); builder.append_dir_all(".", &src)?; builder @@ -154,11 +221,12 @@ impl ArtifactTransport for ObjectStoreTransport { let mut file = tokio::fs::File::open(tar_file.path()) .await .map_err(SnapshotError::Io)?; - let upload = self - .store - .put_multipart(&self.payload_path(key)) - .await - .map_err(map_obj)?; + let upload = timed( + "multipart create", + self.store.put_multipart(&self.payload_path(key)), + ) + .await? + .map_err(map_obj)?; let mut wm = WriteMultipart::new_with_chunk_size(upload, CHUNK); let mut buf = vec![0u8; CHUNK]; loop { @@ -167,30 +235,35 @@ impl ArtifactTransport for ObjectStoreTransport { if n == 0 { break; } - wm.wait_for_capacity(MAX_CONCURRENT_PARTS) - .await + timed("part upload", wm.wait_for_capacity(MAX_CONCURRENT_PARTS)) + .await? .map_err(map_obj)?; wm.write(&buf[..n]); } - wm.finish().await.map_err(map_obj)?; + // finish() drains every in-flight part (up to MAX_CONCURRENT_PARTS × + // CHUNK bytes) plus the completion request, so it gets a proportionally + // larger stall bound than a single-request await. + timed_by( + "multipart finish", + OP_TIMEOUT * MAX_CONCURRENT_PARTS as u32, + wm.finish(), + ) + .await? + .map_err(map_obj)?; // Manifest sibling LAST: its presence marks the payload complete. - self.store - .put(&self.manifest_path(key), PutPayload::from(manifest_bytes)) - .await - .map_err(map_obj)?; + timed( + "manifest put", + self.store + .put(&self.manifest_path(key), PutPayload::from(manifest_bytes)), + ) + .await? + .map_err(map_obj)?; Ok(()) } async fn manifest(&self, key: &str) -> Result { - let bytes = self - .store - .get(&self.manifest_path(key)) - .await - .map_err(map_obj)? - .bytes() - .await - .map_err(map_obj)?; + let bytes = self.fetch_manifest_bytes(key).await?; manifest_from_slice(&bytes) } @@ -208,14 +281,7 @@ impl ArtifactTransport for ObjectStoreTransport { // Sibling manifest first — it is the completeness marker and the value // we cross-check the tar against. - let sibling = self - .store - .get(&self.manifest_path(key)) - .await - .map_err(map_obj)? - .bytes() - .await - .map_err(map_obj)?; + let sibling = self.fetch_manifest_bytes(key).await?; let manifest = manifest_from_slice(&sibling)?; // Stream the tar to a temp file. @@ -223,13 +289,11 @@ impl ArtifactTransport for ObjectStoreTransport { let mut tar_writer = tokio::fs::File::create(tar_tmp.path()) .await .map_err(SnapshotError::Io)?; - let mut stream = self - .store - .get(&self.payload_path(key)) - .await + let mut stream = timed("payload get", self.store.get(&self.payload_path(key))) + .await? .map_err(map_obj)? .into_stream(); - while let Some(chunk) = stream.next().await { + while let Some(chunk) = timed("payload read", stream.next()).await? { let chunk = chunk.map_err(map_obj)?; tar_writer .write_all(&chunk) @@ -250,6 +314,11 @@ impl ArtifactTransport for ObjectStoreTransport { .tempdir_in(&parent)?; let file = std::fs::File::open(tar_tmp.path())?; let mut archive = tar::Archive::new(std::io::BufReader::new(file)); + // The tar came from an untrusted transport: never adopt its mode + // bits (a crafted archive could plant world-writable or setuid + // entries) or mtimes — content is what import verifies. + archive.set_preserve_permissions(false); + archive.set_preserve_mtime(false); // tar's unpack refuses entries that escape the destination, on top // of the manifest path validation import performs again. archive.unpack(stage.path())?; @@ -259,18 +328,15 @@ impl ArtifactTransport for ObjectStoreTransport { "downloaded artifact tar has no embedded manifest".into(), ) })?; - if embedded != sibling.as_ref() { + if embedded != sibling { return Err(SnapshotError::ArtifactInvalid( "embedded manifest disagrees with the sibling manifest object".into(), )); } check_dest_available(&dest)?; - if dest.is_dir() { - std::fs::remove_dir(&dest)?; - } let root = stage.keep(); - std::fs::rename(&root, &dest)?; + rename_into_place(&root, &dest)?; Ok(()) }) .await diff --git a/tests/bootstrap.rs b/tests/bootstrap.rs index 5f58137..8c5e5e3 100644 --- a/tests/bootstrap.rs +++ b/tests/bootstrap.rs @@ -76,6 +76,7 @@ fn spawn_node( watcher, WatchScope::All, resume, + None, // reader: cursor-expired resync not exercised here Some(fold), Some(ex_rx), BatchConfig::default(), diff --git a/tests/integration.rs b/tests/integration.rs index 0656ca2..d9d31bd 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -123,11 +123,12 @@ async fn wait_until_ready(url: &str) { /// Deterministically wait until a freshly spawned watch is live. /// -/// NATS KV watches deliver updates only (no initial-state replay), and the -/// ephemeral consumer takes a moment to attach — so any write issued before the -/// subscription is established is silently missed. We close that race by writing -/// `sentinel` (which must fall within the watch's filter) on a retry loop until -/// the watch echoes it back, then drain any duplicate echoes. +/// Watches deliver the current state first (the re-list), then live updates — +/// but the consumer still takes a moment to attach, and a test that writes +/// before attachment can't tell where its write landed in that order. We make +/// the timeline deterministic by writing `sentinel` (which must fall within the +/// watch's filter) on a retry loop until the watch echoes it back, then drain +/// any buffered re-list entries and duplicate echoes. async fn establish_watch(writer: &dyn KvWriter, rx: &mut mpsc::Receiver, sentinel: &str) { loop { writer.put(sentinel, b"ready").await.expect("put sentinel"); @@ -621,6 +622,61 @@ async fn watch_prefixes_unions_on_a_single_consumer() { ); } +/// `watch_prefixes_from` — the hand-built multi-filter ordered consumer — must +/// deliver exactly the post-cursor writes within the prefix union: delta-only +/// (no re-list), union-scoped (server-side filtering), in revision order with +/// correct versions parsed from the ACK subject. +#[tokio::test] +async fn watch_prefixes_from_replays_only_the_delta() { + let nats = TestNats::start().await; + let (_conn, store) = nats.store("watchmany-from").await; + let writer = store.writer().expect("writer"); + let watcher = store.watcher().expect("watcher"); + + // Baseline state in both watched prefixes, plus the cursor. + writer.put("vpcA.a", b"1").await.expect("put A baseline"); + let cursor_rev = writer.put("vpcB.b", b"2").await.expect("put B baseline"); + let cursor = WatchCursor::from_u64(cursor_rev.as_u64().expect("u64 rev")); + + // Post-cursor: two in-union writes, one outside the union. + writer.put("vpcC.x", b"skip").await.expect("put C"); // filtered + writer.put("vpcA.a2", b"3").await.expect("put A delta"); + writer.put("vpcB.b2", b"4").await.expect("put B delta"); + + let (tx, mut rx) = mpsc::channel(64); + tokio::spawn(async move { + let _ = watcher + .watch_prefixes_from(&["vpcA.", "vpcB."], &cursor, tx) + .await; + }); + + // Exactly the two post-cursor in-union writes, in revision order, with + // their real revisions — and neither the baseline (no re-list on a delta + // resume) nor the out-of-union write. + let updates = collect_updates(&mut rx, 2).await; + let keys: Vec<&str> = updates.iter().map(|u| u.key()).collect(); + assert_eq!( + keys, + vec!["vpcA.a2", "vpcB.b2"], + "delta must be exactly the post-cursor in-union writes" + ); + for u in &updates { + let rev = u.version().as_u64().expect("u64 revision"); + assert!( + rev > cursor_rev.as_u64().unwrap(), + "delivered revision {rev} must be past the cursor" + ); + } + + // The out-of-union key must never arrive (server-side filtered). + assert!( + timeout(Duration::from_millis(500), rx.recv()) + .await + .is_err(), + "a non-watched prefix leaked through watch_prefixes_from" + ); +} + #[tokio::test] async fn watch_all_from_replays_only_the_delta() { let nats = TestNats::start().await; @@ -676,16 +732,17 @@ async fn watch_prefix_from_replays_only_the_delta() { assert!(matches!(&updates[0], KvUpdate::Put(e) if e.key == "node.b")); } -/// Demonstrates the seed-then-watch race a snapshot consumer must avoid, and that resuming from the -/// snapshot revision closes it. +/// The seed-then-watch race, closed by state-sync watches. /// /// A consumer that seeds via `scan()` and then subscribes via `watch_prefix()` has a window between -/// the two calls. `watch_prefix` uses NATS `DeliverPolicy::New` (no initial-state replay), so a -/// write that lands in that window is in neither the snapshot nor the watch stream — it is silently -/// lost until the next full reseed. `watch_prefix_from(snapshot_revision)` instead replays -/// everything after the snapshot, so the gap write is delivered. +/// the two calls. `watch_prefix` used to ride NATS `DeliverPolicy::New` (live updates only), so a +/// write landing in that window was in neither the snapshot nor the watch stream — silently lost +/// until the next full reseed. `watch_prefix` now delivers a state-sync re-list (the current value +/// of every matching key, in revision order) before live updates, so the gap write is re-delivered +/// by the re-list itself. `watch_prefix_from(snapshot_revision)` remains the delta-only variant: +/// everything after the snapshot, nothing before it. #[tokio::test] -async fn seed_then_watch_prefix_loses_writes_in_the_gap() { +async fn watch_prefix_relist_covers_seed_then_watch_gap() { let nats = TestNats::start().await; let (_conn, store) = nats.store("seed-race").await; let writer = store.writer().expect("writer"); @@ -706,30 +763,35 @@ async fn seed_then_watch_prefix_loses_writes_in_the_gap() { // 2) A write lands in the race window: after the scan, before any watch attaches. writer.put("blackhole.2", b"fraud").await.expect("put 2"); - // --- The bug: watch_prefix (DeliverPolicy::New) --- + // --- watch_prefix: the re-list carries BOTH keys, gap write included --- { let (tx, mut rx) = mpsc::channel(64); let w = watcher.clone(); tokio::spawn(async move { let _ = w.watch_prefix("blackhole.", tx).await; }); - // Handshake until the watch is provably attached and live (drains the sentinel echoes). - establish_watch(writer.as_ref(), &mut rx, "blackhole.sentinel").await; - // The watch is live now, so a *fresh* write is delivered... + // No attach handshake needed: the re-list is delivered regardless of + // subscription timing, in revision order. + let got = collect_updates(&mut rx, 2).await; + let keys: Vec<&str> = got.iter().map(|u| u.key()).collect(); + assert_eq!( + keys, + vec!["blackhole.1", "blackhole.2"], + "watch_prefix's re-list must deliver current state including the gap write" + ); + + // ...and live updates still flow after the re-list. writer.put("blackhole.3", b"spend").await.expect("put 3"); let got = collect_updates(&mut rx, 1).await; - // ...but the first thing we see is blackhole.3, not blackhole.2: the gap write was dropped. - // (blackhole.2 has a lower revision; had the watch carried it, it would arrive first.) assert!( matches!(&got[0], KvUpdate::Put(e) if e.key == "blackhole.3"), - "watch_prefix delivered a post-subscribe write but silently dropped the gap write \ - blackhole.2; got {:?}", + "live updates must follow the re-list; got {:?}", got[0].key() ); } - // --- The fix: watch_prefix_from(snapshot revision) --- + // --- watch_prefix_from(snapshot revision): delta-only, no re-list --- { let (tx, mut rx) = mpsc::channel(64); let cursor = WatchCursor::from_u64(baseline_rev); @@ -738,7 +800,7 @@ async fn seed_then_watch_prefix_loses_writes_in_the_gap() { let _ = w.watch_prefix_from("blackhole.", &cursor, tx).await; }); // Resuming from the snapshot revision replays everything after it; the first such entry is - // exactly the gap write that watch_prefix lost. + // exactly the gap write — and NOT blackhole.1, which the snapshot already has. let got = collect_updates(&mut rx, 1).await; assert!( matches!(&got[0], KvUpdate::Put(e) if e.key == "blackhole.2"), @@ -1227,6 +1289,7 @@ fn spawn_applied( watcher, WatchScope::All, baseline, + None, // reader: cursor-expired resync not exercised here snapshot, None, BatchConfig::default(), diff --git a/tests/snapshot_store.rs b/tests/snapshot_store.rs index 2cb7ad7..01d911a 100644 --- a/tests/snapshot_store.rs +++ b/tests/snapshot_store.rs @@ -526,6 +526,34 @@ fn check_import_rejects_undeclared_extra_file( assert!(!dest.exists()); } +/// A symlink planted in the payload is rejected — it would hash as its +/// target's bytes but restore as a link (or escape the staging area entirely), +/// the tar-archive twin of zip-slip. The manifest path validation can't catch +/// it (the path looks normal); the non-regular-file check must. +#[cfg(unix)] +fn check_import_rejects_symlink_in_payload( + open: impl Fn(&Path) -> (WatchCursor, S), + import: ImportFn, +) { + let (artifact, _manifest, dir) = exported_stream_artifact(&open); + let target = dir.path().join("outside"); + std::fs::write(&target, b"outside the artifact").unwrap(); + std::os::unix::fs::symlink(&target, artifact.join("data").join("escape")).unwrap(); + + let dest = dir.path().join("imported"); + match import(&artifact, &dest) { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!( + msg.contains("non-regular"), + "rejection names the link: {msg}" + ); + } + Err(other) => panic!("expected ArtifactInvalid, got {other:?}"), + Ok(_) => panic!("artifact with a payload symlink must not import"), + } + assert!(!dest.exists(), "nothing lands at the destination"); +} + /// Rewrite one top-level manifest field (checksums untouched) and re-import. fn with_doctored_manifest(artifact: &Path, field: &str, value: serde_json::Value) { let raw = std::fs::read(artifact.join(MANIFEST_FILE)).unwrap(); @@ -742,6 +770,12 @@ fn append_log_import_rejects_undeclared_extra_file() { check_import_rejects_undeclared_extra_file(open_append_log, import_append_log); } +#[cfg(unix)] +#[test] +fn append_log_import_rejects_symlink_in_payload() { + check_import_rejects_symlink_in_payload(open_append_log, import_append_log); +} + #[test] fn append_log_import_rejects_wrong_backend() { check_import_rejects_wrong_backend(open_append_log, import_append_log); @@ -897,6 +931,12 @@ mod fjall_backend { check_import_rejects_undeclared_extra_file(open_no_sync, import_fjall); } + #[cfg(unix)] + #[test] + fn fjall_import_rejects_symlink_in_payload() { + check_import_rejects_symlink_in_payload(open_no_sync, import_fjall); + } + #[test] fn fjall_import_rejects_wrong_backend() { check_import_rejects_wrong_backend(open_no_sync, import_fjall); @@ -1109,6 +1149,12 @@ mod rocksdb_backend { check_import_rejects_undeclared_extra_file(open_no_sync, import_rocksdb); } + #[cfg(unix)] + #[test] + fn rocksdb_import_rejects_symlink_in_payload() { + check_import_rejects_symlink_in_payload(open_no_sync, import_rocksdb); + } + #[test] fn rocksdb_import_rejects_wrong_backend() { check_import_rejects_wrong_backend(open_no_sync, import_rocksdb); diff --git a/tests/transport.rs b/tests/transport.rs index b413b7f..3d44c2b 100644 --- a/tests/transport.rs +++ b/tests/transport.rs @@ -156,6 +156,59 @@ async fn missing_remote_artifact_is_artifact_invalid() { } } +/// A sibling-manifest object larger than the 1 MiB cap is rejected before +/// being buffered whole — the OOM guard against a hostile or corrupted object +/// at the manifest key. +#[tokio::test(flavor = "multi_thread")] +async fn oversized_remote_manifest_is_rejected() { + let (transport, bucket) = local_transport(); + // Plant the oversized object directly in the "bucket" at the sibling key. + let sibling_dir = bucket.path().join("slipstream-artifacts"); + std::fs::create_dir_all(&sibling_dir).unwrap(); + std::fs::write( + sibling_dir.join("oversized.manifest.json"), + vec![b'x'; (1 << 20) + 1], + ) + .unwrap(); + + match transport.manifest("oversized").await { + Err(SnapshotError::ArtifactInvalid(msg)) => assert!(msg.contains("exceeds"), "{msg}"), + other => panic!("expected ArtifactInvalid, got {other:?}"), + } +} + +/// Both URL-parse failure modes surface as transport errors, not panics: a +/// string that is not a URL at all, and a URL whose scheme `object_store` +/// doesn't recognize. +#[test] +fn from_url_opts_rejects_bad_urls() { + // Garbage that fails URL parsing, and a well-formed URL whose scheme + // object_store doesn't recognize. + for bad in ["not a url", "bogus://bucket/prefix"] { + match ObjectStoreTransport::from_url_opts(bad, std::iter::empty::<(&str, &str)>()) { + Err(SnapshotError::Backend(_)) => {} + Err(other) => panic!("url {bad:?}: expected Backend, got {other:?}"), + Ok(_) => panic!("url {bad:?} must not build a transport"), + } + } +} + +/// A destination with no parent directory (a bare relative path) is refused +/// before any remote I/O — there is nowhere to stage the download beside it. +#[tokio::test(flavor = "multi_thread")] +async fn download_rejects_dest_without_parent() { + let (transport, _bucket) = local_transport(); + match transport + .download("anything", Path::new("slipstream-bare-dest")) + .await + { + Err(SnapshotError::ArtifactInvalid(msg)) => { + assert!(msg.contains("no parent"), "{msg}"); + } + other => panic!("expected ArtifactInvalid, got {other:?}"), + } +} + // --- run_export_round ------------------------------------------------------------ // These need a real lease (NATS KV) and a live watch_applied loop. @@ -212,6 +265,7 @@ async fn live_round() -> LiveRound { watcher, WatchScope::All, None, + None, // reader: cursor-expired resync not exercised here Some(fold), Some(ex_rx), BatchConfig::default(), @@ -382,3 +436,171 @@ async fn run_export_round_upload_failure_abandons_lease() { round.shutdown.send(true).unwrap(); round.task.await.unwrap().unwrap(); } + +/// The watch loop is gone before the round begins (panicked / shut down): the +/// round fails with an error naming the loop, nothing is left in scratch, and +/// the lease is abandoned so a replacement node can win the round immediately +/// instead of waiting out the ttl. +#[tokio::test(flavor = "multi_thread")] +async fn run_export_round_dead_loop_abandons_lease() { + let round = live_round().await; + settle_watch(&round).await; + + // Kill the loop definitively; its export receiver drops with it. + round.shutdown.send(true).unwrap(); + round.task.await.unwrap().unwrap(); + + let (transport, _bucket) = local_transport(); + let scratch = round.dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + let lease = ExportLease::new(round.lease_store.as_ref(), "round", "node-a").unwrap(); + + let err = run_export_round( + &lease, + Duration::from_secs(600), // long ttl: only abandon makes re-acquire possible + &round.exports, + &transport, + "latest", + &scratch, + ) + .await + .expect_err("a dead watch loop must fail the round"); + match err { + SnapshotError::Backend(msg) => assert!(msg.contains("watch loop"), "{msg}"), + other => panic!("expected Backend, got {other:?}"), + } + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "scratch cleaned up after the failed round" + ); + + // The lease was abandoned: a replacement node wins immediately, long + // before the 600 s ttl. + let replacement = ExportLease::new(round.lease_store.as_ref(), "round", "node-b").unwrap(); + assert!( + replacement + .try_acquire(Duration::from_secs(60)) + .await + .unwrap() + .is_some(), + "abandoned lease frees the round for a replacement node" + ); +} + +// --- import_remote error paths for the LSM backends (local transport) -------- +// The MinIO tier only exercises these backends with a GOOD artifact; these +// prove the verify gate fires for a tampered one without needing MinIO. + +/// Export a fold, flip a byte in its largest payload file, upload it, and +/// assert the remote import rejects it with nothing at the destination and a +/// clean scratch dir. Generic over the backend's export + import_remote. +#[cfg(any(feature = "fjall", feature = "rocksdb"))] +async fn assert_import_remote_rejects_tampered( + artifact: &std::path::Path, + manifest: &ExportManifest, + transport: &ObjectStoreTransport, +) { + // Tamper with the largest payload file (most likely to hold real data). + let victim = manifest + .files + .iter() + .max_by_key(|f| f.size) + .expect("at least one payload file"); + let victim_path = artifact.join(&victim.path); + let mut bytes = std::fs::read(&victim_path).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 0xFF; + std::fs::write(&victim_path, &bytes).unwrap(); + + // upload() validates only the manifest, so the tampered payload ships. + transport.upload("tampered", artifact).await.unwrap(); +} + +#[cfg(feature = "fjall")] +mod fjall_remote { + use super::*; + use slipstream::snapshot::SnapshotStore; + use slipstream::{FjallConfig, FjallSnapshot}; + + /// `import_remote` re-verifies every payload hash: a tampered artifact + /// shipped through the (untrusted) transport is rejected, nothing lands at + /// the destination, and the downloaded copy is cleaned from scratch. + #[tokio::test(flavor = "multi_thread")] + async fn fjall_import_remote_rejects_tampered_artifact() { + let (transport, _bucket) = local_transport(); + let dir = TempDir::new().unwrap(); + let cfg = FjallConfig { + sync: false, + ..Default::default() + }; + + let (_r, mut fold) = FjallSnapshot::open(&dir.path().join("fold"), cfg).unwrap(); + fold.apply( + &[put("a", b"1", 1), put("b", b"2", 2), put("c", b"3", 3)], + &WatchCursor::from_u64(3), + ) + .unwrap(); + let artifact = dir.path().join("artifact"); + let manifest = fold.export_to(&artifact).unwrap(); + assert_import_remote_rejects_tampered(&artifact, &manifest, &transport).await; + + let scratch = dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + let dest = dir.path().join("imported"); + match FjallSnapshot::import_remote(&transport, "tampered", &scratch, &dest, cfg).await { + Err(SnapshotError::ArtifactInvalid(_)) => {} + Err(other) => panic!("expected ArtifactInvalid, got {other:?}"), + Ok(_) => panic!("tampered artifact must not import"), + } + assert!(!dest.exists(), "nothing lands at the destination"); + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "downloaded artifact cleaned from scratch" + ); + } +} + +#[cfg(feature = "rocksdb")] +mod rocksdb_remote { + use super::*; + use slipstream::snapshot::SnapshotStore; + use slipstream::{RocksDbConfig, RocksDbSnapshot}; + + /// RocksDB twin of the fjall test above. + #[tokio::test(flavor = "multi_thread")] + async fn rocksdb_import_remote_rejects_tampered_artifact() { + let (transport, _bucket) = local_transport(); + let dir = TempDir::new().unwrap(); + let cfg = RocksDbConfig { + sync: false, + ..Default::default() + }; + + let (_r, mut fold) = RocksDbSnapshot::open(&dir.path().join("fold"), cfg).unwrap(); + fold.apply( + &[put("a", b"1", 1), put("b", b"2", 2), put("c", b"3", 3)], + &WatchCursor::from_u64(3), + ) + .unwrap(); + let artifact = dir.path().join("artifact"); + let manifest = fold.export_to(&artifact).unwrap(); + assert_import_remote_rejects_tampered(&artifact, &manifest, &transport).await; + + let scratch = dir.path().join("scratch"); + std::fs::create_dir(&scratch).unwrap(); + let dest = dir.path().join("imported"); + match RocksDbSnapshot::import_remote(&transport, "tampered", &scratch, &dest, cfg).await { + Err(SnapshotError::ArtifactInvalid(_)) => {} + Err(other) => panic!("expected ArtifactInvalid, got {other:?}"), + Ok(_) => panic!("tampered artifact must not import"), + } + assert!(!dest.exists(), "nothing lands at the destination"); + assert_eq!( + std::fs::read_dir(&scratch).unwrap().count(), + 0, + "downloaded artifact cleaned from scratch" + ); + } +} diff --git a/tests/transport_s3.rs b/tests/transport_s3.rs index 80feb35..fec541f 100644 --- a/tests/transport_s3.rs +++ b/tests/transport_s3.rs @@ -82,6 +82,7 @@ fn spawn_node( watcher, WatchScope::All, resume, + None, // reader: cursor-expired resync not exercised here Some(fold), Some(ex_rx), BatchConfig::default(),