feat: implement sequencer recovery for stale batches

.github/actions/setup-guest-toolchain/action.yml

@@ -28,6 +28,10 @@ inputs:
     description: "Foundry version tag"
     required: false
     default: "v1.4.3"
+  rust-toolchain:
+    description: "Rust toolchain version or channel"
+    required: false
+    default: "1.95.0"
   rust-components:
     description: "Extra rustup components (comma-separated)"
     required: false
@@ -109,6 +113,7 @@ runs:
     - name: Install Rust toolchain
       uses: dtolnay/rust-toolchain@stable
       with:
+        toolchain: ${{ inputs.rust-toolchain }}
         components: ${{ inputs.rust-components }}
 
     - name: Cache Rust artifacts

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
 
 env:
+  RUST_TOOLCHAIN: "1.95.0"
   XGENEXT2FS_VERSION: v1.5.6
   XGENEXT2FS_SHA256_AMD64: 996e4e68a638b5dc5967d3410f92ecb8d2f41e32218bbe0f8b4c4474d7eebc59
   XGENEXT2FS_SHA256_ARM64: e5aca81164b762bbe5447bacef41e4fa9e357fd9c8f44e519c5206227d43144d
@@ -25,13 +26,16 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y \
+            faketime \
+            libfaketime \
             lua5.4 \
             liblua5.4-dev \
             libslirp-dev
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
           components: rustfmt, clippy
 
       - name: Cache Rust artifacts
@@ -53,7 +57,7 @@ jobs:
 
       - name: Test
         timeout-minutes: 15
-        run: RUN_ANVIL_TESTS=1 cargo test --workspace --all-targets --all-features --locked
+        run: cargo test --workspace --all-targets --all-features --locked
 
   canonical-guest:
     runs-on: ubuntu-latest
@@ -67,6 +71,7 @@ jobs:
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
           xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }}
           xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }}
           xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }}
@@ -92,6 +97,7 @@ jobs:
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
           xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }}
           xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }}
           xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }}
@@ -100,5 +106,10 @@ jobs:
           cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }}
           install-foundry: "true"
 
+      - name: Install faketime
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y faketime libfaketime
+
       - name: Run rollups E2E tests
         run: just test-rollups-e2e

.github/workflows/release.yml

@@ -20,6 +20,7 @@ permissions:
   contents: write
 
 env:
+  RUST_TOOLCHAIN: "1.95.0"
   XGENEXT2FS_VERSION: v1.5.6
   XGENEXT2FS_SHA256_AMD64: 996e4e68a638b5dc5967d3410f92ecb8d2f41e32218bbe0f8b4c4474d7eebc59
   XGENEXT2FS_SHA256_ARM64: e5aca81164b762bbe5447bacef41e4fa9e357fd9c8f44e519c5206227d43144d
@@ -59,6 +60,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
           targets: ${{ matrix.target }}
 
       - name: Cache Rust artifacts
@@ -123,6 +125,7 @@ jobs:
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
           xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }}
           xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }}
           xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }}

.gitignore

@@ -7,3 +7,4 @@ sequencer.db-wal
 /out/
 /.DS_Store
 soljson-latest.js
+**/states/

CLAUDE.md

@@ -0,0 +1,57 @@
+# CLAUDE.md
+
+Quick reference for working in this repository. For the full guide — architecture, duality, recovery, invariants, threat model, and rules — read [`AGENTS.md`](AGENTS.md).
+
+## Shell Environment
+
+This project uses Nix + direnv. Before running any command that needs project tools (Foundry, TLA+, etc.), activate the direnv environment:
+
+```bash
+eval "$(direnv export bash 2>/dev/null)"
+```
+
+This makes `anvil`, `forge`, `cast`, `tlc`, and other Nix-provided tools available. Cargo and rustc are available without direnv.
+
+## Commands
+
+```bash
+cargo check                                              # compile check
+cargo test --workspace --exclude canonical-test          # run tests (canonical-test needs libslirp)
+cargo fmt --all                                          # format
+cargo clippy --all-targets --all-features -- -D warnings # lint
+cargo test -p sequencer --lib                            # includes Anvil-backed tests (needs Foundry on PATH)
+```
+
+## What This Is
+
+Off-chain sequencer for an app-specific DeFi rollup. Accepts signed user operations, issues low-latency soft confirmations, and posts batches to L1. Currently backed by a placeholder wallet app (transfer, withdrawal). **Security-critical infrastructure** — handle every change accordingly.
+
+Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ encoding.
+
+## Workspace Layout
+
+- `sequencer/` — main sequencer binary and library.
+- `sequencer-core/` — shared domain types consumed by both sequencer and scheduler.
+- `examples/app-core/` — placeholder wallet app implementing `Application`.
+- `examples/canonical-app/` — on-chain scheduler reference implementation.
+- `examples/canonical-test/` — e2e test harness for the canonical app.
+- `sdk/rust-client/` — Rust client library for the sequencer API.
+- `tests/{benchmarks,e2e,harness}/` — test infrastructure.
+
+## Sequencer Module Layout
+
+`sequencer/src/` is organized by writer role; `storage/<role>.rs` holds each role's storage half.
+
+- `runtime/` — bootstrap, config, shutdown, shared clock.
+- `ingress/` — public write path: `api.rs` (`POST /tx`) + `inclusion_lane/` (hot path).
+- `egress/` — internal read path: `api/` (WS subscribe + health) + `l2_tx_feed/`.
+- `l1/` — reader, submitter, provider, partition helper.
+- `recovery/` — startup preemptive-recovery procedure, runtime danger detector, mempool flusher.
+- `storage/` — SQLite persistence, split per writer role.
+- `http.rs` — shared HTTP error type + `axum::serve` orchestration.
+
+## Before You Start Real Work
+
+- **[`AGENTS.md`](AGENTS.md)** — mission, requirements, invariants, duality, recovery, conventions, rules.
+- **[`docs/threat-model/README.md`](docs/threat-model/README.md)** — trust boundaries and in-scope threats.
+- **[`docs/recovery/README.md`](docs/recovery/README.md)** — preemptive recovery design + TLA+ proofs.