From 6acb2436698a5dab925ac785c49697edc32f722d Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sat, 27 Jun 2026 15:37:04 +0530 Subject: [PATCH 01/11] feat: init database crate --- lean_client/Cargo.toml | 1 + lean_client/database/Cargo.toml | 5 +++++ lean_client/database/src/lib.rs | 14 ++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 lean_client/database/Cargo.toml create mode 100644 lean_client/database/src/lib.rs diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 5a394eac..7d7c660d 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -2,6 +2,7 @@ members = [ "chain", "containers", + "database", "env-config", "fork_choice", "http_api", diff --git a/lean_client/database/Cargo.toml b/lean_client/database/Cargo.toml new file mode 100644 index 00000000..64395ced --- /dev/null +++ b/lean_client/database/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "database" +edition = { workspace = true } + +[dependencies] \ No newline at end of file diff --git a/lean_client/database/src/lib.rs b/lean_client/database/src/lib.rs new file mode 100644 index 00000000..b93cf3ff --- /dev/null +++ b/lean_client/database/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From 19c6afd4d79086c6f484b57feabb1c78d0c8716a Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 01:56:59 +0530 Subject: [PATCH 02/11] chore: pull database as a grandine dep and create storage crate --- lean_client/Cargo.lock | 156 +++++++++++++++++-- lean_client/Cargo.toml | 7 +- lean_client/database/Cargo.toml | 5 - lean_client/storage/Cargo.toml | 9 ++ lean_client/{database => storage}/src/lib.rs | 0 5 files changed, 155 insertions(+), 22 deletions(-) delete mode 100644 lean_client/database/Cargo.toml create mode 100644 lean_client/storage/Cargo.toml rename lean_client/{database => storage}/src/lib.rs (100%) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 8f507b99..82f91643 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "rand 0.9.4", "rapidhash", "ruint", - "rustc-hash", + "rustc-hash 2.1.2", "serde", "sha3 0.10.8", ] @@ -627,6 +627,24 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "proc-macro2 1.0.106", + "quote 1.0.45", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.117", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -798,16 +816,33 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" +dependencies = [ + "serde_core", +] + [[package]] name = "cc" -version = "1.2.60" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ - "find-msvc-tools", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -882,6 +917,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.0" @@ -1297,6 +1343,27 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "database" +version = "0.0.0" +source = "git+https://github.com/grandinetech/grandine?rev=64afdee3c6be79fceffb66933dcb69a943f3f1ae#64afdee3c6be79fceffb66933dcb69a943f3f1ae" +dependencies = [ + "anyhow", + "bytesize", + "fs-err", + "futures", + "im", + "itertools 0.14.0", + "logging", + "reth-libmdbx", + "snap", + "std_ext", + "tap", + "thiserror 2.0.18", + "tracing", + "unwrap_none", +] + [[package]] name = "dedicated_executor" version = "0.1.0" @@ -1832,12 +1899,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - [[package]] name = "fixed-hash" version = "0.8.0" @@ -1899,6 +1960,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "funty" version = "2.0.0" @@ -2486,7 +2557,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2949,6 +3020,7 @@ dependencies = [ "parking_lot", "reqwest", "ssz", + "storage", "tikv-jemallocator", "tokio", "tracing", @@ -3077,6 +3149,16 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -4650,9 +4732,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -4670,7 +4752,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -4689,7 +4771,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.52.0", ] @@ -5016,6 +5098,31 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" +[[package]] +name = "reth-libmdbx" +version = "1.3.12" +source = "git+https://github.com/paradigmxyz/reth.git?rev=6f8e7258f4733279080e4bd8345ce50538a40d6e#6f8e7258f4733279080e4bd8345ce50538a40d6e" +dependencies = [ + "bitflags", + "byteorder", + "derive_more", + "indexmap 2.14.0", + "parking_lot", + "reth-mdbx-sys", + "smallvec", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "reth-mdbx-sys" +version = "1.3.12" +source = "git+https://github.com/paradigmxyz/reth.git?rev=6f8e7258f4733279080e4bd8345ce50538a40d6e#6f8e7258f4733279080e4bd8345ce50538a40d6e" +dependencies = [ + "bindgen", + "cc", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -5131,6 +5238,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -5722,6 +5835,13 @@ dependencies = [ "triomphe", ] +[[package]] +name = "storage" +version = "0.0.0" +dependencies = [ + "database", +] + [[package]] name = "strength_reduce" version = "0.2.4" @@ -6441,6 +6561,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap_none" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "461d0c5956fcc728ecc03a3a961e4adc9a7975d86f6f8371389a289517c02ca9" + [[package]] name = "url" version = "2.5.8" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 7d7c660d..4bd5bb8f 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -2,13 +2,13 @@ members = [ "chain", "containers", - "database", "env-config", "fork_choice", "http_api", "metrics", "networking", - "spec_test_fixtures", + "spec_test_fixtures", + "storage", "validator", "xmss", ] @@ -230,6 +230,7 @@ http_api = { path = "./http_api" } metrics = { path = "./metrics" } networking = { path = "./networking" } spec_test_fixtures = { path = "./spec_test_fixtures" } +storage = { path = "./storage" } validator = { path = "./validator" } xmss = { path = "./xmss" } @@ -239,6 +240,7 @@ axum = "0.8.8" bitvec = "1.0.1" bls = { git = "https://github.com/grandinetech/grandine", package = "bls", features = ["blst"], rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } clap = { version = "4", features = ["derive"] } +database = { git = "https://github.com/grandinetech/grandine", package = "database", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } derive_more = "2.1.1" discv5 = "0.10.2" enr = { version = "0.13", features = ["k256"] } @@ -331,6 +333,7 @@ networking = { workspace = true } num_cpus = { workspace = true } parking_lot = { workspace = true } ssz = { workspace = true } +storage = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/lean_client/database/Cargo.toml b/lean_client/database/Cargo.toml deleted file mode 100644 index 64395ced..00000000 --- a/lean_client/database/Cargo.toml +++ /dev/null @@ -1,5 +0,0 @@ -[package] -name = "database" -edition = { workspace = true } - -[dependencies] \ No newline at end of file diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml new file mode 100644 index 00000000..7b623e2f --- /dev/null +++ b/lean_client/storage/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "storage" +edition = { workspace = true } + +[dependencies] +database = { workspace = true } + +[lints] +workspace = true diff --git a/lean_client/database/src/lib.rs b/lean_client/storage/src/lib.rs similarity index 100% rename from lean_client/database/src/lib.rs rename to lean_client/storage/src/lib.rs From 57d0438bca945c0e1702f8fe9a1ac38e3d0b7a90 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 04:42:50 +0530 Subject: [PATCH 03/11] feat: add leanspec consts for db --- lean_client/storage/src/lib.rs | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index b93cf3ff..2dd60d57 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,14 +1,24 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +use database::{Database, DatabaseMode, RestartMessage}; + +const BLOCKS_TABLE_NAME: &str = "blocks"; + +const BLOCKS_CREATE_INDEX: &str = "slots_to_block_roots_index"; + +const STATES_TABLE_NAME: &str = "states"; + +const STATES_CREATE_INDEX: &str = "slots_to_state_roots_index"; + +const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; + +const CHECKPOINTS_KEY_JUSTIFIED: &str = "justified"; + +const CHECKPOINTS_KEY_FINALIZED: &str = "finalized"; + +const CHECKPOINTS_KEY_HEAD: &str = "head"; + +const CHECKPOINTS_KEY_GENESIS_TIME: &str = "genesis_time"; + +const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; + +const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; + From e0233e05e60fb25bac70a2125dd16d84b5c90419 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 16:16:22 +0530 Subject: [PATCH 04/11] feat: add databases for each KV pair defined in spec --- lean_client/storage/Cargo.toml | 2 + lean_client/storage/src/lib.rs | 102 ++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 7b623e2f..2632a54a 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -4,6 +4,8 @@ edition = { workspace = true } [dependencies] database = { workspace = true } +bytesize = { workspace = true } +anyhow = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 2dd60d57..e08f96b1 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,24 +1,96 @@ -use database::{Database, DatabaseMode, RestartMessage}; +use std::path::Path; -const BLOCKS_TABLE_NAME: &str = "blocks"; +use anyhow::Result; +use bytesize::ByteSize; +use database::{Database, DatabaseMode}; -const BLOCKS_CREATE_INDEX: &str = "slots_to_block_roots_index"; +pub struct Storage { + pub blocks: Blocks, + pub states: States, + pub checkpoints: Checkpoints, + pub slot_index: SlotIndex, + pub state_root_index: StateRootIndex, +} -const STATES_TABLE_NAME: &str = "states"; +impl Storage { + pub fn new() -> Result { + Ok(Self { + blocks: Blocks::new()?, + states: States::new()?, + checkpoints: Checkpoints::new()?, + slot_index: SlotIndex::new()?, + state_root_index: StateRootIndex::new()?, + }) + } +} -const STATES_CREATE_INDEX: &str = "slots_to_state_roots_index"; +pub struct Blocks(Database); +pub struct States(Database); +pub struct Checkpoints(Database); +pub struct SlotIndex(Database); +pub struct StateRootIndex(Database); -const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; +impl Blocks { + pub fn new() -> Result { + let db = Database::persistent( + "blocks", + Path::new("./database/blocks"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_JUSTIFIED: &str = "justified"; +impl States { + pub fn new() -> Result { + let db = Database::persistent( + "states", + Path::new("./database/states"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_FINALIZED: &str = "finalized"; +impl Checkpoints { + pub fn new() -> Result { + let db = Database::persistent( + "checkpoints", + Path::new("./database/checkpoints"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_HEAD: &str = "head"; - -const CHECKPOINTS_KEY_GENESIS_TIME: &str = "genesis_time"; - -const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; - -const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; +impl SlotIndex { + pub fn new() -> Result { + let db = Database::persistent( + "slot_index", + Path::new("./database/slot_index"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} +impl StateRootIndex { + pub fn new() -> Result { + let db = Database::persistent( + "state_root_index", + Path::new("./database/state_root_index"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} From 4376cdd657165a0bea0e541ffe592e9dbb18ad5b Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Tue, 30 Jun 2026 04:00:48 +0530 Subject: [PATCH 05/11] feat: add Storage impl with builderplate functions from spec --- lean_client/Cargo.lock | 4 ++ lean_client/Cargo.toml | 1 + lean_client/storage/Cargo.toml | 2 + lean_client/storage/src/lib.rs | 113 ++++++++++++++++++++++++++++----- 4 files changed, 105 insertions(+), 15 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 82f91643..7c16bf00 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -5839,7 +5839,11 @@ dependencies = [ name = "storage" version = "0.0.0" dependencies = [ + "anyhow", + "bytesize", + "containers", "database", + "ssz", ] [[package]] diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 4bd5bb8f..f154849b 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -239,6 +239,7 @@ async-trait = "0.1" axum = "0.8.8" bitvec = "1.0.1" bls = { git = "https://github.com/grandinetech/grandine", package = "bls", features = ["blst"], rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } +bytesize = { version = '2', features = ['serde'] } clap = { version = "4", features = ["derive"] } database = { git = "https://github.com/grandinetech/grandine", package = "database", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } derive_more = "2.1.1" diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 2632a54a..337bde2d 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -6,6 +6,8 @@ edition = { workspace = true } database = { workspace = true } bytesize = { workspace = true } anyhow = { workspace = true } +containers = { workspace = true } +ssz = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index e08f96b1..0de5526b 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -4,12 +4,15 @@ use anyhow::Result; use bytesize::ByteSize; use database::{Database, DatabaseMode}; +use containers::{Block, State, Checkpoint, Slot}; +use ssz::H256; + pub struct Storage { - pub blocks: Blocks, - pub states: States, - pub checkpoints: Checkpoints, - pub slot_index: SlotIndex, - pub state_root_index: StateRootIndex, + blocks: Blocks, + states: States, + checkpoints: Checkpoints, + slot_index: SlotIndex, + state_root_index: StateRootIndex, } impl Storage { @@ -22,16 +25,96 @@ impl Storage { state_root_index: StateRootIndex::new()?, }) } + + pub fn get_block(self, root: H256) -> Result> { + let blocks_db = self.blocks; + + Ok(None) + } + + pub fn put_block(self, block: Block, root: H256) { + let blocks_db = self.blocks; + } + + pub fn get_state(self, root: H256) -> Result> { + let states_db = self.states; + + Ok(None) + } + + pub fn put_state(self, state: State, root: H256) { + let states_db = self.states; + } + + pub fn get_justified_checkpoint(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_justified_checkpoint(self, checkpoint: Checkpoint) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_finalized_checkpoint(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_finalized_checkpoint(self, checkpoint: Checkpoint) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_head_root(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_head_root(self, root: H256) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_block_root_by_slot(self, slot: Slot) -> Result> { + let slot_index_db = self.slot_index; + + Ok(None) + } + + pub fn put_block_root_by_slot(self, slot: Slot, root: H256) { + let slot_index_db = self.slot_index; + } + + pub fn get_block_root_by_state_root(self, state_root: H256) -> Result> { + let state_root_index_db = self.state_root_index; + + Ok(None) + } + + pub fn put_block_root_by_state_root(self, state_root: H256, block_root: H256) { + let state_root_index_db = self.state_root_index; + } + + pub fn get_genesis_time(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_genesis_time(self, genesis_time: u64) { + let checkpoints_db = self.checkpoints; + } } -pub struct Blocks(Database); -pub struct States(Database); -pub struct Checkpoints(Database); -pub struct SlotIndex(Database); -pub struct StateRootIndex(Database); +struct Blocks(Database); +struct States(Database); +struct Checkpoints(Database); +struct SlotIndex(Database); +struct StateRootIndex(Database); impl Blocks { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "blocks", Path::new("./database/blocks"), @@ -44,7 +127,7 @@ impl Blocks { } impl States { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "states", Path::new("./database/states"), @@ -57,7 +140,7 @@ impl States { } impl Checkpoints { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "checkpoints", Path::new("./database/checkpoints"), @@ -70,7 +153,7 @@ impl Checkpoints { } impl SlotIndex { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "slot_index", Path::new("./database/slot_index"), @@ -83,7 +166,7 @@ impl SlotIndex { } impl StateRootIndex { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "state_root_index", Path::new("./database/state_root_index"), From 2a25e9ea9ce94f9750bb46a926a90fe885c7e832 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Wed, 1 Jul 2026 03:55:24 +0530 Subject: [PATCH 06/11] feat: storage spec API functions --- lean_client/storage/src/lib.rs | 123 +++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 50 deletions(-) diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 0de5526b..e9518d0e 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -4,8 +4,8 @@ use anyhow::Result; use bytesize::ByteSize; use database::{Database, DatabaseMode}; -use containers::{Block, State, Checkpoint, Slot}; -use ssz::H256; +use containers::{Block, Checkpoint, Slot, State}; +use ssz::{H256, SszReadDefault as _, SszWrite as _}; pub struct Storage { blocks: Blocks, @@ -26,84 +26,107 @@ impl Storage { }) } - pub fn get_block(self, root: H256) -> Result> { - let blocks_db = self.blocks; - - Ok(None) + pub fn get_block(&self, root: H256) -> Result> { + match self.blocks.0.get(root)? { + Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), + None => Ok(None), + } } - pub fn put_block(self, block: Block, root: H256) { - let blocks_db = self.blocks; + pub fn put_block(&self, block: Block, root: H256) -> Result<()> { + let block_bytes = block.to_ssz()?; + self.blocks.0.put(root, block_bytes)?; + Ok(()) } - pub fn get_state(self, root: H256) -> Result> { - let states_db = self.states; - - Ok(None) + pub fn get_state(&self, root: H256) -> Result> { + match self.states.0.get(root)? { + Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), + None => Ok(None), + } } - pub fn put_state(self, state: State, root: H256) { - let states_db = self.states; + pub fn put_state(&self, state: State, root: H256) -> Result<()> { + let state_bytes = state.to_ssz()?; + self.states.0.put(root, state_bytes)?; + Ok(()) } - pub fn get_justified_checkpoint(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_justified_checkpoint(&self) -> Result> { + match self.checkpoints.0.get("justified")? { + Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + None => Ok(None), + } } - pub fn put_justified_checkpoint(self, checkpoint: Checkpoint) { - let checkpoints_db = self.checkpoints; + pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { + let checkpoint_bytes = checkpoint.to_ssz()?; + self.checkpoints.0.put("justified", checkpoint_bytes)?; + Ok(()) } - pub fn get_finalized_checkpoint(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_finalized_checkpoint(&self) -> Result> { + match self.checkpoints.0.get("finalized")? { + Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + None => Ok(None), + } } - pub fn put_finalized_checkpoint(self, checkpoint: Checkpoint) { - let checkpoints_db = self.checkpoints; + pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { + let checkpoint_bytes = checkpoint.to_ssz()?; + self.checkpoints.0.put("finalized", checkpoint_bytes)?; + Ok(()) } - pub fn get_head_root(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_head_root(&self) -> Result> { + match self.checkpoints.0.get("head")? { + Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), + None => Ok(None), + } } - pub fn put_head_root(self, root: H256) { - let checkpoints_db = self.checkpoints; + pub fn put_head_root(&self, root: H256) -> Result<()> { + self.checkpoints.0.put("head", root)?; + Ok(()) } - pub fn get_block_root_by_slot(self, slot: Slot) -> Result> { - let slot_index_db = self.slot_index; - - Ok(None) + pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { + let slot_bytes = slot.0.to_be_bytes(); + match self.slot_index.0.get(slot_bytes)? { + Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + None => Ok(None), + } } - pub fn put_block_root_by_slot(self, slot: Slot, root: H256) { - let slot_index_db = self.slot_index; + pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { + let slot_bytes = slot.0.to_be_bytes(); + self.slot_index.0.put(slot_bytes, root)?; + Ok(()) } - pub fn get_block_root_by_state_root(self, state_root: H256) -> Result> { - let state_root_index_db = self.state_root_index; - - Ok(None) + pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { + match self.state_root_index.0.get(state_root)? { + Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + None => Ok(None), + } } - pub fn put_block_root_by_state_root(self, state_root: H256, block_root: H256) { - let state_root_index_db = self.state_root_index; + pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { + self.state_root_index.0.put(state_root, block_root)?; + Ok(()) } - pub fn get_genesis_time(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_genesis_time(&self) -> Result> { + match self.checkpoints.0.get("genesis_time")? { + Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), + None => Ok(None), + } } - pub fn put_genesis_time(self, genesis_time: u64) { - let checkpoints_db = self.checkpoints; + pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { + let genesis_time_bytes = genesis_time.to_ssz()?; + self.checkpoints.0.put("genesis_time", genesis_time_bytes)?; + Ok(()) } } From efe3e0a5ab78b67ac63e8ac0e25c23e163932664 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 2 Jul 2026 23:00:24 +0530 Subject: [PATCH 07/11] feat: implement libmdbx with custom compression per database instance --- lean_client/Cargo.lock | 126 ++++-- lean_client/Cargo.toml | 9 + lean_client/storage/Cargo.toml | 17 +- lean_client/storage/src/database.rs | 631 ++++++++++++++++++++++++++++ lean_client/storage/src/lib.rs | 9 +- 5 files changed, 760 insertions(+), 32 deletions(-) create mode 100644 lean_client/storage/src/database.rs diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 7c16bf00..dd763c61 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -636,7 +636,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.106", "quote 1.0.45", "regex", @@ -831,6 +831,8 @@ version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -1340,28 +1342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", -] - -[[package]] -name = "database" -version = "0.0.0" -source = "git+https://github.com/grandinetech/grandine?rev=64afdee3c6be79fceffb66933dcb69a943f3f1ae#64afdee3c6be79fceffb66933dcb69a943f3f1ae" -dependencies = [ - "anyhow", - "bytesize", - "fs-err", - "futures", - "im", - "itertools 0.14.0", - "logging", - "reth-libmdbx", - "snap", - "std_ext", - "tap", - "thiserror 2.0.18", - "tracing", - "unwrap_none", + "syn 2.0.117", ] [[package]] @@ -1967,7 +1948,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" dependencies = [ "autocfg", - "tokio", ] [[package]] @@ -2557,7 +2537,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2936,6 +2916,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -4423,6 +4413,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "polling" version = "3.11.0" @@ -4734,7 +4730,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -4771,7 +4767,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.52.0", ] @@ -5842,8 +5838,19 @@ dependencies = [ "anyhow", "bytesize", "containers", - "database", + "fs-err", + "futures", + "im", + "itertools 0.14.0", + "lz4_flex", + "reth-libmdbx", "ssz", + "tap", + "tempfile", + "test-case", + "thiserror 2.0.18", + "unwrap_none", + "zstd", ] [[package]] @@ -6020,12 +6027,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", ] +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2 1.0.106", + "quote 1.0.45", + "syn 2.0.117", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.45", + "syn 2.0.117", + "test-case-core", +] + [[package]] name = "test-generator" version = "0.3.1" @@ -7429,3 +7469,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index f154849b..7746917a 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -247,6 +247,7 @@ discv5 = "0.10.2" enr = { version = "0.13", features = ["k256"] } eth_ssz = { package = "ethereum_ssz", version = "0.10.0" } ethereum-types = "0.14" +fs-err = "3" futures = "0.3" features = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } git-version = "0.3" @@ -254,11 +255,13 @@ hex = "0.4.3" indexmap = "2" http-body-util = "0.1" http_api_utils = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } +im = "15" k256 = "0.13" rec_aggregation = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } backend = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } leansig = { git = "https://github.com/leanEthereum/leanSig", branch = "devnet4" } leansig_wrapper = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } +libmdbx = { git = "https://github.com/paradigmxyz/reth.git", package = "reth-libmdbx", rev = "6f8e7258f4733279080e4bd8345ce50538a40d6e" } libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", rev = "91e8931e275bcd1c72791d18b09fea8b77209baf", default-features = false, features = [ 'dns', 'gossipsub', @@ -272,6 +275,7 @@ libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", rev = "91e8931e275 'yamux' ] } libp2p-identity = { version = "0.2.13", features = ["secp256k1"] } +lz4_flex = "0.13" dedicated_executor = { path = "dedicated_executor" } num-bigint = "0.4" num-traits = "0.2" @@ -295,6 +299,9 @@ snap = "1.1" ssz = { git = "https://github.com/grandinetech/grandine", package = "ssz", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } ssz-types = "0.3" itertools = "0.14" +tap = "1" +tempfile = "3" +test-case = "3" test-generator = "0.3.1" thiserror = "2" tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } @@ -307,8 +314,10 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } tree-hash = "0.4.0" try_from_iterator = { git = "https://github.com/grandinetech/grandine", package = "try_from_iterator", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } typenum = "1.19" +unwrap_none = "0.1" yamux = "0.12" zeroize = "1.8" +zstd = "0.13.3" [package] name = "lean_client" diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 337bde2d..ba850c47 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -3,11 +3,24 @@ name = "storage" edition = { workspace = true } [dependencies] -database = { workspace = true } -bytesize = { workspace = true } anyhow = { workspace = true } +bytesize = { workspace = true } containers = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +im = { workspace = true } +itertools = { workspace = true } +libmdbx = { workspace = true } +lz4_flex = { workspace = true } ssz = { workspace = true } +tap = { workspace = true } +thiserror = { workspace = true } +unwrap_none = { workspace = true } +zstd = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +test-case = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs new file mode 100644 index 00000000..39b916d7 --- /dev/null +++ b/lean_client/storage/src/database.rs @@ -0,0 +1,631 @@ +use core::ops::{Range, RangeFrom, RangeToInclusive}; +use std::path::Path; +use std::{ + borrow::Cow, + sync::{Arc, Mutex}, +}; + +use anyhow::Result; +use bytesize::ByteSize; +use futures::channel::mpsc::UnboundedSender; +use im::OrdMap; +use itertools::Either; +use libmdbx::{DatabaseFlags, Environment, Geometry, ObjectLength, Stat, WriteFlags}; +use tap::Pipe as _; +use thiserror::Error; +use unwrap_none::UnwrapNone as _; + +const GROWTH_STEP: ByteSize = ByteSize::mib(256); + +const MAX_NAMED_DATABASES: usize = 10; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } + + fn decompress_pair(self, (key, compressed_value): (K, Cow<[u8]>)) -> Result<(K, Vec)> { + let value = self.decompress(&compressed_value)?; + Ok((key, value)) + } +} + +pub trait PrefixableKey { + const PREFIX: &'static str; + + #[must_use] + fn has_prefix(bytes: &[u8]) -> bool { + bytes.starts_with(Self::PREFIX.as_bytes()) + } +} + +#[derive(Debug)] +pub enum RestartMessage { + StorageMapFull(libmdbx::Error), +} + +impl RestartMessage { + pub fn send(self, tx: &UnboundedSender) { + // Nothing to do if the send fails: the receiver has been dropped. + drop(tx.unbounded_send(self)); + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + // + // The UNIX permissions to set on created files. Zero value means to open existing, but do not create. + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } +} + +pub struct Database { + kind: DatabaseKind, + compression: Compression, +} + +impl Database { + pub fn persistent( + name: &str, + directory: impl AsRef, + compression: Compression, + max_size: ByteSize, + mode: DatabaseMode, + restart_tx: Option>, + ) -> Result { + // If a database with the legacy name exists, keep using it. + // Otherwise, create a new database with the specified name. + // This check will not force existing users to resync. + let legacy_name = directory.as_ref().to_str().ok_or(Error)?; + + if !mode.is_read_only() { + fs_err::create_dir_all(&directory)?; + } + + // TODO(Grandine Team): The call to `set_max_dbs` and `MAX_NAMED_DATABASES` should be + // unnecessary if the default database is used. + let environment = Environment::builder() + .set_max_dbs(MAX_NAMED_DATABASES) + .set_geometry(Geometry { + size: Some(..usize::try_from(max_size.as_u64())?), + growth_step: Some(isize::try_from(GROWTH_STEP.as_u64())?), + shrink_threshold: None, + page_size: None, + }) + .open_with_permissions(directory.as_ref(), mode.permissions())?; + + let transaction = environment.begin_rw_txn()?; + let existing_db = transaction.open_db(Some(legacy_name)); + + let database_name = if existing_db.is_err() { + if !mode.is_read_only() { + transaction.create_db(Some(name), DatabaseFlags::default())?; + } + + name + } else { + legacy_name + } + .to_owned(); + + transaction.commit()?; + + Ok(Self { + kind: DatabaseKind::Persistent { + database_name, + environment, + restart_tx, + }, + compression, + }) + } + + #[must_use] + pub fn in_memory() -> Self { + Self { + kind: DatabaseKind::InMemory { + map: Mutex::default(), + }, + compression: Compression::Zstd, + } + } + + pub fn delete(&self, key: impl AsRef<[u8]>) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + transaction.commit()?; + } + } + DatabaseKind::InMemory { map } => { + map.lock() + .expect("in-memory database mutex is poisoned") + .remove(key.as_ref()); + } + } + + Ok(()) + } + + pub fn delete_range(&self, range: Range>) -> Result<()> { + let start = range.start.as_ref(); + let end = range.end.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + let Some((mut key, ())) = cursor.set_range::, _>(start)? else { + return Ok(()); + }; + + while *key < *end { + cursor.del(WriteFlags::default())?; + match cursor.next::, _>()? { + Some((new_key, ())) => key = new_key, + None => break, + } + } + + transaction.commit()?; + } + DatabaseKind::InMemory { map } => { + // Update the map atomically for consistency with `Database::put_batch`. + // This should only make a difference if the method panics between mutations. + // The mutex will be left poisoned either way. + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + let mut new_map = map.clone(); + + let end_pair = map.get_key_value(end); + let (below, _) = new_map.split(start); + let (_, above) = new_map.split(end); + + new_map = below.union(above); + + if let Some((key, value)) = end_pair { + new_map + .insert(key.clone(), value.clone()) + .expect_none("end_pair should have been discarded by OrdMap::split"); + } + + *map = new_map; + } + } + + Ok(()) + } + + pub fn contains_key(&self, key: impl AsRef<[u8]>) -> Result { + let contains_key = match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + transaction + .get::<()>(database.dbi(), key.as_ref())? + .is_some() + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .contains_key(key.as_ref()), + }; + + Ok(contains_key) + } + + pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + transaction + .get::>(database.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get(key.as_ref()) + .map(|compressed| self.compression.decompress(compressed)), + } + .transpose() + } + + pub fn db_stats(&self) -> Result> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + Some(transaction.db_stat(&database)?) + } + DatabaseKind::InMemory { map: _ } => None, + } + .pipe(Ok) + } + + pub fn iterate_all_keys_with_lengths( + &self, + ) -> Result, usize)>>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + core::iter::from_fn(move || cursor.next().transpose()) + .map(|result| { + let (key, ObjectLength(length)) = result?; + Ok((key, length)) + }) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + + let it = map + .clone() + .into_iter() + .map(|(key, value)| Ok((Cow::Owned(key.to_vec()), value.len()))); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + #[expect(clippy::type_complexity)] + pub fn iterator_ascending( + &self, + range: RangeFrom>, + ) -> Result, Vec)>>> { + let start = range.start.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_range(start) + .transpose() + .into_iter() + .chain(core::iter::from_fn(move || cursor.next().transpose())) + .map(|result| self.compression.decompress_pair(result?)) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + let start_pair = map.get_key_value(start); + let (_, mut above) = map.split(start); + + if let Some((key, value)) = start_pair { + above + .insert(key.clone(), value.clone()) + .expect_none("start_pair should have been discarded by OrdMap::split"); + } + + let it = above.into_iter().map(|(key, value)| { + Ok(( + Cow::Owned(key.to_vec()), + self.compression.decompress(value.as_ref())?, + )) + }); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + #[expect(clippy::type_complexity)] + pub fn iterator_descending( + &self, + range: RangeToInclusive>, + ) -> Result, Vec)>>> { + let end = range.end.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_key(end) + .transpose() + .into_iter() + .chain(core::iter::from_fn(move || cursor.prev().transpose())) + .map(|result| self.compression.decompress_pair(result?)) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + let end_pair = map.get_key_value(end); + let (mut below, _) = map.split(end); + + if let Some((key, value)) = end_pair { + below + .insert(key.clone(), value.clone()) + .expect_none("end_pair should have been discarded by OrdMap::split"); + } + + let it = below.into_iter().rev().map(|(key, value)| { + Ok(( + Cow::Owned(key.to_vec()), + self.compression.decompress(value.as_ref())?, + )) + }); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { + self.put_batch(core::iter::once((key, value))) + } + + pub fn put_batch( + &self, + pairs: impl IntoIterator, impl AsRef<[u8]>)>, + ) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + for (key, value) in pairs { + let key = key.as_ref(); + let compressed = self.compression.compress(value.as_ref())?; + transaction + .put(database.dbi(), key, compressed, WriteFlags::default()) + .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; + } + + transaction + .commit() + .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; + } + DatabaseKind::InMemory { map } => { + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + let mut new_map = map.clone(); + + for (key, value) in pairs { + let key = key.as_ref().into(); + let compressed = self.compression.compress(value.as_ref())?.into(); + new_map.insert(key, compressed); + } + + *map = new_map; + } + } + + Ok(()) + } + + /// Returns the first key-value pair whose key is less than or equal to `key`. + /// + /// Behaves like [`im::OrdMap::get_prev`]. + /// + /// [`im::OrdMap::get_prev`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_prev + pub fn prev(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_key(key.as_ref()) + .transpose() + .or_else(|| cursor.prev().transpose()) + .transpose()? + .map(|pair| self.compression.decompress_pair(pair)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get_prev(key.as_ref()) + .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), + } + .transpose() + } + + /// Returns the first key-value pair whose key is greater than or equal to `key`. + /// + /// Behaves like [`im::OrdMap::get_next`]. + /// + /// [`im::OrdMap::get_next`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_next + pub fn next(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_range(key.as_ref())? + .map(|pair| self.compression.decompress_pair(pair)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get_next(key.as_ref()) + .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), + } + .transpose() + } + + const fn kind(&self) -> &DatabaseKind { + &self.kind + } +} + +impl From for Database { + fn from(map: InMemoryMap) -> Self { + Self { + kind: DatabaseKind::InMemory { + map: Mutex::new(map), + }, + compression: Compression::Zstd, + } + } +} + +enum DatabaseKind { + Persistent { + // TODO(Grandine Team): It should be possible to remove `database_name` by using the default + // database (`None`), but that would probably force users to resync. + database_name: String, + environment: Environment, + restart_tx: Option>, + }, + InMemory { + // Various methods of `OrdMap` and `Database` clone the elements of this map, + // so they should be cheaply cloneable. This disqualifies `Vec` and `Box<[u8]>`. + // + // Various methods of `Database` return keys in the form of `Vec` or `Cow<[u8]>`. + // Converting between them and `Arc<[u8]>` is costly due to the reference count before data. + // Returning `Arc<[u8]>` from the methods would require a conversion in the persistent case + // because `libmdbx` cannot decode directly into `std::sync::Arc` or `triomphe::Arc`. + // + // `Bytes` can be cheaply converted to and from `Vec` if its capacity equals its length, + // but `Database` cannot benefit from that with its current API. + // Returning a `Vec` or `Cow` requires copying due to shared ownership. + // Writing requires copying due to the signature of `Database::put`. + // + // Some versions of `libmdbx` (including the one from `reth-libmdbx`) can decode into + // `lifetimed_bytes::Bytes`, which functions like `Cow<[u8]>`, but with the internal + // representation of `Bytes`. `lifetimed_bytes::Bytes` is necessarily distinct from + // `bytes::Bytes`, which makes it harder to use. + map: Mutex, + }, +} + +#[derive(Debug, Error)] +#[error("database directory path should be a valid Unicode string")] +struct Error; + +pub type InMemoryMap = OrdMap, Arc<[u8]>>; + +fn handle_write_error( + error: libmdbx::Error, + restart_tx: Option<&UnboundedSender>, +) -> libmdbx::Error { + if error == libmdbx::Error::MapFull { + if let Some(restart_tx) = restart_tx { + RestartMessage::StorageMapFull(error).send(restart_tx); + } + } + + error +} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index e9518d0e..1f9495fa 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,8 +1,10 @@ +mod database; + use std::path::Path; +use crate::database::{Compression, Database, DatabaseMode}; use anyhow::Result; use bytesize::ByteSize; -use database::{Database, DatabaseMode}; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; @@ -141,6 +143,7 @@ impl Blocks { let db = Database::persistent( "blocks", Path::new("./database/blocks"), + Compression::Lz4, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -154,6 +157,7 @@ impl States { let db = Database::persistent( "states", Path::new("./database/states"), + Compression::Zstd, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -167,6 +171,7 @@ impl Checkpoints { let db = Database::persistent( "checkpoints", Path::new("./database/checkpoints"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -180,6 +185,7 @@ impl SlotIndex { let db = Database::persistent( "slot_index", Path::new("./database/slot_index"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -193,6 +199,7 @@ impl StateRootIndex { let db = Database::persistent( "state_root_index", Path::new("./database/state_root_index"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, From 2da086b6672f72091a7620ee8ae6338feaecc36b Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 00:29:53 +0530 Subject: [PATCH 08/11] chore: tests (all passing) --- lean_client/storage/src/database.rs | 366 +++++++++++++++++++++++++++ lean_client/storage/src/lib.rs | 371 ++++++++++++++++++++++++++-- 2 files changed, 721 insertions(+), 16 deletions(-) diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs index 39b916d7..b8671dd6 100644 --- a/lean_client/storage/src/database.rs +++ b/lean_client/storage/src/database.rs @@ -204,6 +204,38 @@ impl Database { Ok(()) } + pub fn delete_batch(&self, keys: impl IntoIterator>) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + for key in keys { + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + } + } + + transaction.commit()?; + } + DatabaseKind::InMemory { map } => { + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + + for key in keys { + map.remove(key.as_ref()); + } + } + } + + Ok(()) + } + pub fn delete_range(&self, range: Range>) -> Result<()> { let start = range.start.as_ref(); let end = range.end.as_ref(); @@ -629,3 +661,337 @@ fn handle_write_error( error } + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use test_case::test_case; + + use super::*; + + type Constructor = fn() -> Result; + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete("C")?; + database.delete("D")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_batch(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_batch(["A", "C", "D"])?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("B", "2"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_range_inclusive_exclusive(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_range("B".."C")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("C", "3"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_range_between(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_range("D".."F")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("C", "3")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_contains_key(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!(database.contains_key("A")?); + assert!(database.contains_key("B")?); + assert!(database.contains_key("C")?); + assert!(!database.contains_key("D")?); + assert!(database.contains_key("E")?); + assert!(!database.contains_key("F")?); + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterator_ascending(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq( + database.iterator_ascending("0"..)?, + [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("B"..)?, + [("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("C"..)?, + [("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq(database.iterator_ascending("D"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterator_descending(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq( + database.iterator_descending(..="F")?, + [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="E")?, + [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="D")?, + [("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="C")?, + [("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="B")?, + [("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_all_keys_iterator_with_lengths(constructor: Constructor) -> Result<()> { + let database = constructor()?; + let values = database + .iterate_all_keys_with_lengths()? + .map(|result| { + let (key, length) = result?; + let key_string = core::str::from_utf8(key.as_ref())?; + Ok((key_string.to_owned(), length)) + }) + .collect::>>()?; + + let compressed_len = Compression::Zstd.compress(b"A")?.len(); + + let expected = [ + ("A".to_owned(), compressed_len), + ("B".to_owned(), compressed_len), + ("C".to_owned(), compressed_len), + ("E".to_owned(), compressed_len), + ]; + + assert_eq!(values, expected); + + Ok(()) + } + + // This covers a bug we introduced and fixed while implementing in-memory mode. + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterators_do_not_modify_the_database(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_multiple_of_the_same_key(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.put_batch(vec![("A", "1"), ("A", "2"), ("A", "3")])?; + + assert_eq!(database.get("A")?, Some(to_bytes("3"))); + + Ok(()) + } + + // ```text + // 0 A B C D E F + // │ │ ├─┘ ├─┘ + // A B C E + // ``` + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_prev(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!("0" < "A"); + + assert_eq!(database.prev("0")?, None); + assert_eq!(database.prev("A")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.prev("B")?, Some(to_bytes_pair(("B", "2")))); + assert_eq!(database.prev("C")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.prev("D")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.prev("E")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.prev("F")?, Some(to_bytes_pair(("E", "5")))); + + Ok(()) + } + + // ```text + // 0 A B C D E F + // └─┤ │ │ └─┤ + // A B C E + // ``` + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_next(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!("0" < "A"); + + assert_eq!(database.next("0")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.next("A")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.next("B")?, Some(to_bytes_pair(("B", "2")))); + assert_eq!(database.next("C")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.next("D")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.next("E")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.next("F")?, None); + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_isolation(constructor: Constructor) -> Result<()> { + let database = constructor()?; + let iterator = database.iterator_ascending("A"..)?; + + database.delete_range("A".."F")?; + + assert_pairs_eq(iterator, [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")])?; + + Ok(()) + } + + fn build_persistent_database() -> Result { + let database = Database::persistent( + "test_db", + TempDir::new()?, + Compression::Zstd, + ByteSize::mib(1), + DatabaseMode::ReadWrite, + None, + )?; + + populate_database(&database)?; + Ok(database) + } + + fn build_in_memory_database() -> Result { + let database = Database::in_memory(); + populate_database(&database)?; + Ok(database) + } + + fn populate_database(database: &Database) -> Result<()> { + // This indirectly tests `Database::put` and `Database::put_batch`. + database.put_batch(vec![("A", "1"), ("B", "2"), ("C", "3")])?; + database.put("E", "5")?; + Ok(()) + } + + fn assert_pairs_eq<'strings>( + actual_pairs: impl IntoIterator, impl AsRef<[u8]>)>>, + expected_pairs: impl IntoIterator, + ) -> Result<()> { + let actual_pairs = to_string_pairs(actual_pairs)?; + let expected_pairs = to_string_pairs(expected_pairs.into_iter().map(Ok))?; + + assert_eq!(actual_pairs, expected_pairs); + + Ok(()) + } + + fn to_string_pairs( + pairs: impl IntoIterator, impl AsRef<[u8]>)>>, + ) -> Result> { + pairs + .into_iter() + .map(|result| { + let (key, value) = result?; + let key_string = core::str::from_utf8(key.as_ref())?; + let value_string = core::str::from_utf8(value.as_ref())?; + Ok((key_string.to_owned(), value_string.to_owned())) + }) + .collect() + } + + fn to_bytes_pair((key, value): (&str, &str)) -> (Vec, Vec) { + (to_bytes(key), to_bytes(value)) + } + + fn to_bytes(string: &str) -> Vec { + string.as_bytes().to_vec() + } +} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 1f9495fa..5333a297 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,5 +1,9 @@ mod database; +// Dev-dependency reserved for parametrized tests we haven't written yet. +#[cfg(test)] +use ::test_case as _; + use std::path::Path; use crate::database::{Compression, Database, DatabaseMode}; @@ -18,13 +22,14 @@ pub struct Storage { } impl Storage { - pub fn new() -> Result { + pub fn new(base: impl AsRef) -> Result { + let base = base.as_ref(); Ok(Self { - blocks: Blocks::new()?, - states: States::new()?, - checkpoints: Checkpoints::new()?, - slot_index: SlotIndex::new()?, - state_root_index: StateRootIndex::new()?, + blocks: Blocks::new(base)?, + states: States::new(base)?, + checkpoints: Checkpoints::new(base)?, + slot_index: SlotIndex::new(base)?, + state_root_index: StateRootIndex::new(base)?, }) } @@ -139,10 +144,10 @@ struct SlotIndex(Database); struct StateRootIndex(Database); impl Blocks { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "blocks", - Path::new("./database/blocks"), + base.join("blocks"), Compression::Lz4, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -153,10 +158,10 @@ impl Blocks { } impl States { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "states", - Path::new("./database/states"), + base.join("states"), Compression::Zstd, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -167,10 +172,10 @@ impl States { } impl Checkpoints { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "checkpoints", - Path::new("./database/checkpoints"), + base.join("checkpoints"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -181,10 +186,10 @@ impl Checkpoints { } impl SlotIndex { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "slot_index", - Path::new("./database/slot_index"), + base.join("slot_index"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -195,10 +200,10 @@ impl SlotIndex { } impl StateRootIndex { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "state_root_index", - Path::new("./database/state_root_index"), + base.join("state_root_index"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -207,3 +212,337 @@ impl StateRootIndex { Ok(Self(db)) } } + +#[cfg(test)] +mod tests { + use super::*; + + use containers::BlockBody; + use ssz::SszWrite; + use tempfile::TempDir; + + // Each test gets its own persistent `Storage` rooted in a fresh temp dir, so + // tests are isolated, parallel-safe, and cleaned up on drop. This uses the + // real `Storage::new`, so it exercises the real libmdbx path and the real + // per-DB codecs — the only difference from production is the base directory. + // + // The returned `TempDir` MUST be kept alive for as long as the `Storage` is + // used: dropping it deletes the on-disk database files. + fn storage() -> (TempDir, Storage) { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let storage = Storage::new(dir.path()).expect("failed to open storage"); + (dir, storage) + } + + fn h256(byte: u8) -> H256 { + H256::from_slice(&[byte; 32]) + } + + fn sample_block(seed: u8) -> Block { + Block { + slot: Slot(u64::from(seed)), + proposer_index: u64::from(seed), + parent_root: h256(seed), + state_root: h256(seed.wrapping_add(1)), + body: BlockBody::default(), + } + } + + // `Block` and `State` do not implement `PartialEq`, so compare their SSZ + // encodings — the exact bytes the database stores and returns. + fn assert_ssz_eq(left: &T, right: &T) { + assert_eq!( + left.to_ssz().expect("left should serialize"), + right.to_ssz().expect("right should serialize"), + ); + } + + // --- blocks ----------------------------------------------------------- + + #[test] + fn get_block_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_block(h256(9)).unwrap().is_none()); + } + + #[test] + fn put_then_get_block_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(1); + let block = sample_block(1); + + storage.put_block(block.clone(), root).unwrap(); + + let read = storage + .get_block(root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(&block, &read); + } + + #[test] + fn put_and_get_multiple_blocks() { + let (_dir, storage) = storage(); + let blocks: Vec<(H256, Block)> = (0..8).map(|i| (h256(i), sample_block(i))).collect(); + + for (root, block) in &blocks { + storage.put_block(block.clone(), *root).unwrap(); + } + + for (root, block) in &blocks { + let read = storage + .get_block(*root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(block, &read); + } + } + + #[test] + fn put_block_overwrites_existing_root() { + let (_dir, storage) = storage(); + let root = h256(3); + + storage.put_block(sample_block(3), root).unwrap(); + let updated = sample_block(7); + storage.put_block(updated.clone(), root).unwrap(); + + let read = storage + .get_block(root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(&updated, &read); + } + + // --- states ----------------------------------------------------------- + + #[test] + fn get_state_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_state(h256(9)).unwrap().is_none()); + } + + #[test] + fn put_then_get_state_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(2); + let state = State::generate_genesis(1_234, 3); + + storage.put_state(state.clone(), root).unwrap(); + + let read = storage + .get_state(root) + .unwrap() + .expect("state should exist"); + assert_ssz_eq(&state, &read); + } + + #[test] + fn put_and_get_multiple_states() { + let (_dir, storage) = storage(); + let states: Vec<(H256, State)> = (0..4) + .map(|i| (h256(i), State::generate_genesis(u64::from(i), u64::from(i)))) + .collect(); + + for (root, state) in &states { + storage.put_state(state.clone(), *root).unwrap(); + } + + for (root, state) in &states { + let read = storage + .get_state(*root) + .unwrap() + .expect("state should exist"); + assert_ssz_eq(state, &read); + } + } + + // --- checkpoints (justified / finalized / head / genesis_time) --------- + + #[test] + fn justified_checkpoint_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_justified_checkpoint().unwrap().is_none()); + + let checkpoint = Checkpoint { + root: h256(4), + slot: Slot(10), + }; + storage + .put_justified_checkpoint(checkpoint.clone()) + .unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + checkpoint + ); + } + + #[test] + fn finalized_checkpoint_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_finalized_checkpoint().unwrap().is_none()); + + let checkpoint = Checkpoint { + root: h256(5), + slot: Slot(20), + }; + storage + .put_finalized_checkpoint(checkpoint.clone()) + .unwrap(); + + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + checkpoint + ); + } + + #[test] + fn head_root_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_head_root().unwrap().is_none()); + + let root = h256(6); + storage.put_head_root(root).unwrap(); + + assert_eq!(storage.get_head_root().unwrap().unwrap(), root); + } + + #[test] + fn genesis_time_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_genesis_time().unwrap().is_none()); + + storage.put_genesis_time(1_700_000_000).unwrap(); + + assert_eq!(storage.get_genesis_time().unwrap().unwrap(), 1_700_000_000); + } + + // All four values above share the single `checkpoints` database under + // distinct string keys; verify they never clobber one another. + #[test] + fn checkpoints_database_keys_do_not_collide() { + let (_dir, storage) = storage(); + let justified = Checkpoint { + root: h256(1), + slot: Slot(1), + }; + let finalized = Checkpoint { + root: h256(2), + slot: Slot(2), + }; + let head = h256(3); + let genesis_time = 42; + + storage.put_justified_checkpoint(justified.clone()).unwrap(); + storage.put_finalized_checkpoint(finalized.clone()).unwrap(); + storage.put_head_root(head).unwrap(); + storage.put_genesis_time(genesis_time).unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + justified + ); + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + finalized + ); + assert_eq!(storage.get_head_root().unwrap().unwrap(), head); + assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); + } + + // --- slot_index ------------------------------------------------------- + + #[test] + fn block_root_by_slot_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_block_root_by_slot(Slot(0)).unwrap().is_none()); + } + + #[test] + fn block_root_by_slot_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(6); + storage.put_block_root_by_slot(Slot(42), root).unwrap(); + + assert_eq!( + storage.get_block_root_by_slot(Slot(42)).unwrap().unwrap(), + root + ); + } + + #[test] + fn block_root_by_slot_handles_multiple() { + let (_dir, storage) = storage(); + for i in 0..10 { + storage + .put_block_root_by_slot(Slot(u64::from(i)), h256(i)) + .unwrap(); + } + + for i in 0..10 { + assert_eq!( + storage + .get_block_root_by_slot(Slot(u64::from(i))) + .unwrap() + .unwrap(), + h256(i), + ); + } + } + + // --- state_root_index ------------------------------------------------- + + #[test] + fn block_root_by_state_root_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!( + storage + .get_block_root_by_state_root(h256(7)) + .unwrap() + .is_none() + ); + } + + #[test] + fn block_root_by_state_root_roundtrips() { + let (_dir, storage) = storage(); + let state_root = h256(7); + let block_root = h256(8); + storage + .put_block_root_by_state_root(state_root, block_root) + .unwrap(); + + assert_eq!( + storage + .get_block_root_by_state_root(state_root) + .unwrap() + .unwrap(), + block_root, + ); + } + + #[test] + fn block_root_by_state_root_handles_multiple() { + let (_dir, storage) = storage(); + let pairs: Vec<(H256, H256)> = (0..10) + .map(|i| (h256(i), h256(i.wrapping_add(100)))) + .collect(); + + for (state_root, block_root) in &pairs { + storage + .put_block_root_by_state_root(*state_root, *block_root) + .unwrap(); + } + + for (state_root, block_root) in &pairs { + assert_eq!( + storage + .get_block_root_by_state_root(*state_root) + .unwrap() + .unwrap(), + *block_root, + ); + } + } +} From 25902c62bca1fdee9c216ae08a7cc826558d85e3 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 18:27:43 +0530 Subject: [PATCH 09/11] chore: combine database and storage --- lean_client/storage/src/database.rs | 997 ---------------------------- lean_client/storage/src/lib.rs | 299 +++++---- 2 files changed, 170 insertions(+), 1126 deletions(-) delete mode 100644 lean_client/storage/src/database.rs diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs deleted file mode 100644 index b8671dd6..00000000 --- a/lean_client/storage/src/database.rs +++ /dev/null @@ -1,997 +0,0 @@ -use core::ops::{Range, RangeFrom, RangeToInclusive}; -use std::path::Path; -use std::{ - borrow::Cow, - sync::{Arc, Mutex}, -}; - -use anyhow::Result; -use bytesize::ByteSize; -use futures::channel::mpsc::UnboundedSender; -use im::OrdMap; -use itertools::Either; -use libmdbx::{DatabaseFlags, Environment, Geometry, ObjectLength, Stat, WriteFlags}; -use tap::Pipe as _; -use thiserror::Error; -use unwrap_none::UnwrapNone as _; - -const GROWTH_STEP: ByteSize = ByteSize::mib(256); - -const MAX_NAMED_DATABASES: usize = 10; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { - None, - #[default] - Lz4, - Zstd, -} - -impl Compression { - fn compress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), - Self::Zstd => Ok(zstd::encode_all(data, 3)?), - } - } - - fn decompress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), - Self::Zstd => Ok(zstd::decode_all(data)?), - } - } - - fn decompress_pair(self, (key, compressed_value): (K, Cow<[u8]>)) -> Result<(K, Vec)> { - let value = self.decompress(&compressed_value)?; - Ok((key, value)) - } -} - -pub trait PrefixableKey { - const PREFIX: &'static str; - - #[must_use] - fn has_prefix(bytes: &[u8]) -> bool { - bytes.starts_with(Self::PREFIX.as_bytes()) - } -} - -#[derive(Debug)] -pub enum RestartMessage { - StorageMapFull(libmdbx::Error), -} - -impl RestartMessage { - pub fn send(self, tx: &UnboundedSender) { - // Nothing to do if the send fails: the receiver has been dropped. - drop(tx.unbounded_send(self)); - } -} - -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - // - // The UNIX permissions to set on created files. Zero value means to open existing, but do not create. - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - -pub struct Database { - kind: DatabaseKind, - compression: Compression, -} - -impl Database { - pub fn persistent( - name: &str, - directory: impl AsRef, - compression: Compression, - max_size: ByteSize, - mode: DatabaseMode, - restart_tx: Option>, - ) -> Result { - // If a database with the legacy name exists, keep using it. - // Otherwise, create a new database with the specified name. - // This check will not force existing users to resync. - let legacy_name = directory.as_ref().to_str().ok_or(Error)?; - - if !mode.is_read_only() { - fs_err::create_dir_all(&directory)?; - } - - // TODO(Grandine Team): The call to `set_max_dbs` and `MAX_NAMED_DATABASES` should be - // unnecessary if the default database is used. - let environment = Environment::builder() - .set_max_dbs(MAX_NAMED_DATABASES) - .set_geometry(Geometry { - size: Some(..usize::try_from(max_size.as_u64())?), - growth_step: Some(isize::try_from(GROWTH_STEP.as_u64())?), - shrink_threshold: None, - page_size: None, - }) - .open_with_permissions(directory.as_ref(), mode.permissions())?; - - let transaction = environment.begin_rw_txn()?; - let existing_db = transaction.open_db(Some(legacy_name)); - - let database_name = if existing_db.is_err() { - if !mode.is_read_only() { - transaction.create_db(Some(name), DatabaseFlags::default())?; - } - - name - } else { - legacy_name - } - .to_owned(); - - transaction.commit()?; - - Ok(Self { - kind: DatabaseKind::Persistent { - database_name, - environment, - restart_tx, - }, - compression, - }) - } - - #[must_use] - pub fn in_memory() -> Self { - Self { - kind: DatabaseKind::InMemory { - map: Mutex::default(), - }, - compression: Compression::Zstd, - } - } - - pub fn delete(&self, key: impl AsRef<[u8]>) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - if cursor.set::<()>(key.as_ref())?.is_some() { - cursor.del(WriteFlags::default())?; - transaction.commit()?; - } - } - DatabaseKind::InMemory { map } => { - map.lock() - .expect("in-memory database mutex is poisoned") - .remove(key.as_ref()); - } - } - - Ok(()) - } - - pub fn delete_batch(&self, keys: impl IntoIterator>) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - for key in keys { - if cursor.set::<()>(key.as_ref())?.is_some() { - cursor.del(WriteFlags::default())?; - } - } - - transaction.commit()?; - } - DatabaseKind::InMemory { map } => { - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - - for key in keys { - map.remove(key.as_ref()); - } - } - } - - Ok(()) - } - - pub fn delete_range(&self, range: Range>) -> Result<()> { - let start = range.start.as_ref(); - let end = range.end.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - let Some((mut key, ())) = cursor.set_range::, _>(start)? else { - return Ok(()); - }; - - while *key < *end { - cursor.del(WriteFlags::default())?; - match cursor.next::, _>()? { - Some((new_key, ())) => key = new_key, - None => break, - } - } - - transaction.commit()?; - } - DatabaseKind::InMemory { map } => { - // Update the map atomically for consistency with `Database::put_batch`. - // This should only make a difference if the method panics between mutations. - // The mutex will be left poisoned either way. - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - let mut new_map = map.clone(); - - let end_pair = map.get_key_value(end); - let (below, _) = new_map.split(start); - let (_, above) = new_map.split(end); - - new_map = below.union(above); - - if let Some((key, value)) = end_pair { - new_map - .insert(key.clone(), value.clone()) - .expect_none("end_pair should have been discarded by OrdMap::split"); - } - - *map = new_map; - } - } - - Ok(()) - } - - pub fn contains_key(&self, key: impl AsRef<[u8]>) -> Result { - let contains_key = match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - transaction - .get::<()>(database.dbi(), key.as_ref())? - .is_some() - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .contains_key(key.as_ref()), - }; - - Ok(contains_key) - } - - pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - transaction - .get::>(database.dbi(), key.as_ref())? - .map(|compressed| self.compression.decompress(&compressed)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get(key.as_ref()) - .map(|compressed| self.compression.decompress(compressed)), - } - .transpose() - } - - pub fn db_stats(&self) -> Result> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - Some(transaction.db_stat(&database)?) - } - DatabaseKind::InMemory { map: _ } => None, - } - .pipe(Ok) - } - - pub fn iterate_all_keys_with_lengths( - &self, - ) -> Result, usize)>>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - core::iter::from_fn(move || cursor.next().transpose()) - .map(|result| { - let (key, ObjectLength(length)) = result?; - Ok((key, length)) - }) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - - let it = map - .clone() - .into_iter() - .map(|(key, value)| Ok((Cow::Owned(key.to_vec()), value.len()))); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - #[expect(clippy::type_complexity)] - pub fn iterator_ascending( - &self, - range: RangeFrom>, - ) -> Result, Vec)>>> { - let start = range.start.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_range(start) - .transpose() - .into_iter() - .chain(core::iter::from_fn(move || cursor.next().transpose())) - .map(|result| self.compression.decompress_pair(result?)) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - let start_pair = map.get_key_value(start); - let (_, mut above) = map.split(start); - - if let Some((key, value)) = start_pair { - above - .insert(key.clone(), value.clone()) - .expect_none("start_pair should have been discarded by OrdMap::split"); - } - - let it = above.into_iter().map(|(key, value)| { - Ok(( - Cow::Owned(key.to_vec()), - self.compression.decompress(value.as_ref())?, - )) - }); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - #[expect(clippy::type_complexity)] - pub fn iterator_descending( - &self, - range: RangeToInclusive>, - ) -> Result, Vec)>>> { - let end = range.end.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_key(end) - .transpose() - .into_iter() - .chain(core::iter::from_fn(move || cursor.prev().transpose())) - .map(|result| self.compression.decompress_pair(result?)) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - let end_pair = map.get_key_value(end); - let (mut below, _) = map.split(end); - - if let Some((key, value)) = end_pair { - below - .insert(key.clone(), value.clone()) - .expect_none("end_pair should have been discarded by OrdMap::split"); - } - - let it = below.into_iter().rev().map(|(key, value)| { - Ok(( - Cow::Owned(key.to_vec()), - self.compression.decompress(value.as_ref())?, - )) - }); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { - self.put_batch(core::iter::once((key, value))) - } - - pub fn put_batch( - &self, - pairs: impl IntoIterator, impl AsRef<[u8]>)>, - ) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - for (key, value) in pairs { - let key = key.as_ref(); - let compressed = self.compression.compress(value.as_ref())?; - transaction - .put(database.dbi(), key, compressed, WriteFlags::default()) - .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; - } - - transaction - .commit() - .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; - } - DatabaseKind::InMemory { map } => { - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - let mut new_map = map.clone(); - - for (key, value) in pairs { - let key = key.as_ref().into(); - let compressed = self.compression.compress(value.as_ref())?.into(); - new_map.insert(key, compressed); - } - - *map = new_map; - } - } - - Ok(()) - } - - /// Returns the first key-value pair whose key is less than or equal to `key`. - /// - /// Behaves like [`im::OrdMap::get_prev`]. - /// - /// [`im::OrdMap::get_prev`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_prev - pub fn prev(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_key(key.as_ref()) - .transpose() - .or_else(|| cursor.prev().transpose()) - .transpose()? - .map(|pair| self.compression.decompress_pair(pair)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get_prev(key.as_ref()) - .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), - } - .transpose() - } - - /// Returns the first key-value pair whose key is greater than or equal to `key`. - /// - /// Behaves like [`im::OrdMap::get_next`]. - /// - /// [`im::OrdMap::get_next`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_next - pub fn next(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_range(key.as_ref())? - .map(|pair| self.compression.decompress_pair(pair)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get_next(key.as_ref()) - .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), - } - .transpose() - } - - const fn kind(&self) -> &DatabaseKind { - &self.kind - } -} - -impl From for Database { - fn from(map: InMemoryMap) -> Self { - Self { - kind: DatabaseKind::InMemory { - map: Mutex::new(map), - }, - compression: Compression::Zstd, - } - } -} - -enum DatabaseKind { - Persistent { - // TODO(Grandine Team): It should be possible to remove `database_name` by using the default - // database (`None`), but that would probably force users to resync. - database_name: String, - environment: Environment, - restart_tx: Option>, - }, - InMemory { - // Various methods of `OrdMap` and `Database` clone the elements of this map, - // so they should be cheaply cloneable. This disqualifies `Vec` and `Box<[u8]>`. - // - // Various methods of `Database` return keys in the form of `Vec` or `Cow<[u8]>`. - // Converting between them and `Arc<[u8]>` is costly due to the reference count before data. - // Returning `Arc<[u8]>` from the methods would require a conversion in the persistent case - // because `libmdbx` cannot decode directly into `std::sync::Arc` or `triomphe::Arc`. - // - // `Bytes` can be cheaply converted to and from `Vec` if its capacity equals its length, - // but `Database` cannot benefit from that with its current API. - // Returning a `Vec` or `Cow` requires copying due to shared ownership. - // Writing requires copying due to the signature of `Database::put`. - // - // Some versions of `libmdbx` (including the one from `reth-libmdbx`) can decode into - // `lifetimed_bytes::Bytes`, which functions like `Cow<[u8]>`, but with the internal - // representation of `Bytes`. `lifetimed_bytes::Bytes` is necessarily distinct from - // `bytes::Bytes`, which makes it harder to use. - map: Mutex, - }, -} - -#[derive(Debug, Error)] -#[error("database directory path should be a valid Unicode string")] -struct Error; - -pub type InMemoryMap = OrdMap, Arc<[u8]>>; - -fn handle_write_error( - error: libmdbx::Error, - restart_tx: Option<&UnboundedSender>, -) -> libmdbx::Error { - if error == libmdbx::Error::MapFull { - if let Some(restart_tx) = restart_tx { - RestartMessage::StorageMapFull(error).send(restart_tx); - } - } - - error -} - -#[cfg(test)] -mod tests { - use tempfile::TempDir; - use test_case::test_case; - - use super::*; - - type Constructor = fn() -> Result; - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete("C")?; - database.delete("D")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_batch(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_batch(["A", "C", "D"])?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("B", "2"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_range_inclusive_exclusive(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_range("B".."C")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("C", "3"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_range_between(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_range("D".."F")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("C", "3")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_contains_key(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!(database.contains_key("A")?); - assert!(database.contains_key("B")?); - assert!(database.contains_key("C")?); - assert!(!database.contains_key("D")?); - assert!(database.contains_key("E")?); - assert!(!database.contains_key("F")?); - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterator_ascending(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq( - database.iterator_ascending("0"..)?, - [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("B"..)?, - [("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("C"..)?, - [("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq(database.iterator_ascending("D"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterator_descending(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq( - database.iterator_descending(..="F")?, - [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="E")?, - [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="D")?, - [("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="C")?, - [("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="B")?, - [("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_all_keys_iterator_with_lengths(constructor: Constructor) -> Result<()> { - let database = constructor()?; - let values = database - .iterate_all_keys_with_lengths()? - .map(|result| { - let (key, length) = result?; - let key_string = core::str::from_utf8(key.as_ref())?; - Ok((key_string.to_owned(), length)) - }) - .collect::>>()?; - - let compressed_len = Compression::Zstd.compress(b"A")?.len(); - - let expected = [ - ("A".to_owned(), compressed_len), - ("B".to_owned(), compressed_len), - ("C".to_owned(), compressed_len), - ("E".to_owned(), compressed_len), - ]; - - assert_eq!(values, expected); - - Ok(()) - } - - // This covers a bug we introduced and fixed while implementing in-memory mode. - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterators_do_not_modify_the_database(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_multiple_of_the_same_key(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.put_batch(vec![("A", "1"), ("A", "2"), ("A", "3")])?; - - assert_eq!(database.get("A")?, Some(to_bytes("3"))); - - Ok(()) - } - - // ```text - // 0 A B C D E F - // │ │ ├─┘ ├─┘ - // A B C E - // ``` - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_prev(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!("0" < "A"); - - assert_eq!(database.prev("0")?, None); - assert_eq!(database.prev("A")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.prev("B")?, Some(to_bytes_pair(("B", "2")))); - assert_eq!(database.prev("C")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.prev("D")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.prev("E")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.prev("F")?, Some(to_bytes_pair(("E", "5")))); - - Ok(()) - } - - // ```text - // 0 A B C D E F - // └─┤ │ │ └─┤ - // A B C E - // ``` - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_next(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!("0" < "A"); - - assert_eq!(database.next("0")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.next("A")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.next("B")?, Some(to_bytes_pair(("B", "2")))); - assert_eq!(database.next("C")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.next("D")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.next("E")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.next("F")?, None); - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_isolation(constructor: Constructor) -> Result<()> { - let database = constructor()?; - let iterator = database.iterator_ascending("A"..)?; - - database.delete_range("A".."F")?; - - assert_pairs_eq(iterator, [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")])?; - - Ok(()) - } - - fn build_persistent_database() -> Result { - let database = Database::persistent( - "test_db", - TempDir::new()?, - Compression::Zstd, - ByteSize::mib(1), - DatabaseMode::ReadWrite, - None, - )?; - - populate_database(&database)?; - Ok(database) - } - - fn build_in_memory_database() -> Result { - let database = Database::in_memory(); - populate_database(&database)?; - Ok(database) - } - - fn populate_database(database: &Database) -> Result<()> { - // This indirectly tests `Database::put` and `Database::put_batch`. - database.put_batch(vec![("A", "1"), ("B", "2"), ("C", "3")])?; - database.put("E", "5")?; - Ok(()) - } - - fn assert_pairs_eq<'strings>( - actual_pairs: impl IntoIterator, impl AsRef<[u8]>)>>, - expected_pairs: impl IntoIterator, - ) -> Result<()> { - let actual_pairs = to_string_pairs(actual_pairs)?; - let expected_pairs = to_string_pairs(expected_pairs.into_iter().map(Ok))?; - - assert_eq!(actual_pairs, expected_pairs); - - Ok(()) - } - - fn to_string_pairs( - pairs: impl IntoIterator, impl AsRef<[u8]>)>>, - ) -> Result> { - pairs - .into_iter() - .map(|result| { - let (key, value) = result?; - let key_string = core::str::from_utf8(key.as_ref())?; - let value_string = core::str::from_utf8(value.as_ref())?; - Ok((key_string.to_owned(), value_string.to_owned())) - }) - .collect() - } - - fn to_bytes_pair((key, value): (&str, &str)) -> (Vec, Vec) { - (to_bytes(key), to_bytes(value)) - } - - fn to_bytes(string: &str) -> Vec { - string.as_bytes().to_vec() - } -} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 5333a297..d870883d 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,40 +1,176 @@ -mod database; +use std::{ + borrow::Cow, + path::Path, + sync::Arc, +}; -// Dev-dependency reserved for parametrized tests we haven't written yet. -#[cfg(test)] -use ::test_case as _; - -use std::path::Path; - -use crate::database::{Compression, Database, DatabaseMode}; use anyhow::Result; use bytesize::ByteSize; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; +use libmdbx::{ + DatabaseFlags, Environment, Geometry, RW, Transaction, WriteFlags, +}; + pub struct Storage { - blocks: Blocks, - states: States, - checkpoints: Checkpoints, - slot_index: SlotIndex, - state_root_index: StateRootIndex, + environment: Arc, + blocks: Database, + states: Database, + checkpoints: Database, + slot_index: Database, + state_root_index: Database, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } +} + +pub struct Database { + environment: Arc, + name: String, + compression: Compression, +} + +impl Database { + pub fn new(env: Arc, name: &str, compression: Compression) -> Result { + Ok(Self { + environment: env, + name: name.to_owned(), + compression, + }) + } + + pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { + let txn = self.environment.begin_ro_txn()?; + let db = txn.open_db(Some(&self.name))?; + + txn.get::>(db.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + .transpose() + } + + pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { + let txn = self.environment.begin_rw_txn()?; + self.put_in(&txn, key, value)?; + txn.commit()?; + Ok(()) + } + + fn put_in( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + value: impl AsRef<[u8]>, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let compressed = self.compression.compress(value.as_ref())?; + txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; + Ok(()) + } } impl Storage { pub fn new(base: impl AsRef) -> Result { let base = base.as_ref(); + fs_err::create_dir_all(base)?; + + let environment = Arc::new( + Environment::builder() + .set_max_dbs(5) + .set_geometry(Geometry { + size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), + growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), + shrink_threshold: None, + page_size: None, + }) + .open_with_permissions(base, DatabaseMode::ReadWrite.permissions())?, + ); + + let txn = environment.begin_rw_txn()?; + for name in [ + "blocks", + "states", + "checkpoints", + "slot_index", + "state_root_index", + ] { + txn.create_db(Some(name), DatabaseFlags::default())?; + } + txn.commit()?; + Ok(Self { - blocks: Blocks::new(base)?, - states: States::new(base)?, - checkpoints: Checkpoints::new(base)?, - slot_index: SlotIndex::new(base)?, - state_root_index: StateRootIndex::new(base)?, + blocks: Database::new(environment.clone(), "blocks", Compression::Lz4)?, + states: Database::new(environment.clone(), "states", Compression::Zstd)?, + checkpoints: Database::new(environment.clone(), "checkpoints", Compression::None)?, + slot_index: Database::new(environment.clone(), "slot_index", Compression::None)?, + state_root_index: Database::new( + environment.clone(), + "state_root_index", + Compression::None, + )?, + environment, }) } pub fn get_block(&self, root: H256) -> Result> { - match self.blocks.0.get(root)? { + match self.blocks.get(root)? { Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), None => Ok(None), } @@ -42,12 +178,12 @@ impl Storage { pub fn put_block(&self, block: Block, root: H256) -> Result<()> { let block_bytes = block.to_ssz()?; - self.blocks.0.put(root, block_bytes)?; + self.blocks.put(root, block_bytes)?; Ok(()) } pub fn get_state(&self, root: H256) -> Result> { - match self.states.0.get(root)? { + match self.states.get(root)? { Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), None => Ok(None), } @@ -55,12 +191,12 @@ impl Storage { pub fn put_state(&self, state: State, root: H256) -> Result<()> { let state_bytes = state.to_ssz()?; - self.states.0.put(root, state_bytes)?; + self.states.put(root, state_bytes)?; Ok(()) } pub fn get_justified_checkpoint(&self) -> Result> { - match self.checkpoints.0.get("justified")? { + match self.checkpoints.get("justified")? { Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), None => Ok(None), } @@ -68,12 +204,12 @@ impl Storage { pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.0.put("justified", checkpoint_bytes)?; + self.checkpoints.put("justified", checkpoint_bytes)?; Ok(()) } pub fn get_finalized_checkpoint(&self) -> Result> { - match self.checkpoints.0.get("finalized")? { + match self.checkpoints.get("finalized")? { Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), None => Ok(None), } @@ -81,25 +217,25 @@ impl Storage { pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.0.put("finalized", checkpoint_bytes)?; + self.checkpoints.put("finalized", checkpoint_bytes)?; Ok(()) } pub fn get_head_root(&self) -> Result> { - match self.checkpoints.0.get("head")? { + match self.checkpoints.get("head")? { Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), None => Ok(None), } } pub fn put_head_root(&self, root: H256) -> Result<()> { - self.checkpoints.0.put("head", root)?; + self.checkpoints.put("head", root)?; Ok(()) } pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { let slot_bytes = slot.0.to_be_bytes(); - match self.slot_index.0.get(slot_bytes)? { + match self.slot_index.get(slot_bytes)? { Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), None => Ok(None), } @@ -107,24 +243,24 @@ impl Storage { pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { let slot_bytes = slot.0.to_be_bytes(); - self.slot_index.0.put(slot_bytes, root)?; + self.slot_index.put(slot_bytes, root)?; Ok(()) } pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { - match self.state_root_index.0.get(state_root)? { + match self.state_root_index.get(state_root)? { Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), None => Ok(None), } } pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { - self.state_root_index.0.put(state_root, block_root)?; + self.state_root_index.put(state_root, block_root)?; Ok(()) } pub fn get_genesis_time(&self) -> Result> { - match self.checkpoints.0.get("genesis_time")? { + match self.checkpoints.get("genesis_time")? { Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), None => Ok(None), } @@ -132,87 +268,11 @@ impl Storage { pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { let genesis_time_bytes = genesis_time.to_ssz()?; - self.checkpoints.0.put("genesis_time", genesis_time_bytes)?; + self.checkpoints.put("genesis_time", genesis_time_bytes)?; Ok(()) } } -struct Blocks(Database); -struct States(Database); -struct Checkpoints(Database); -struct SlotIndex(Database); -struct StateRootIndex(Database); - -impl Blocks { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "blocks", - base.join("blocks"), - Compression::Lz4, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl States { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "states", - base.join("states"), - Compression::Zstd, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl Checkpoints { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "checkpoints", - base.join("checkpoints"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl SlotIndex { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "slot_index", - base.join("slot_index"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl StateRootIndex { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "state_root_index", - base.join("state_root_index"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - #[cfg(test)] mod tests { use super::*; @@ -221,13 +281,6 @@ mod tests { use ssz::SszWrite; use tempfile::TempDir; - // Each test gets its own persistent `Storage` rooted in a fresh temp dir, so - // tests are isolated, parallel-safe, and cleaned up on drop. This uses the - // real `Storage::new`, so it exercises the real libmdbx path and the real - // per-DB codecs — the only difference from production is the base directory. - // - // The returned `TempDir` MUST be kept alive for as long as the `Storage` is - // used: dropping it deletes the on-disk database files. fn storage() -> (TempDir, Storage) { let dir = tempfile::tempdir().expect("failed to create temp dir"); let storage = Storage::new(dir.path()).expect("failed to open storage"); @@ -248,8 +301,6 @@ mod tests { } } - // `Block` and `State` do not implement `PartialEq`, so compare their SSZ - // encodings — the exact bytes the database stores and returns. fn assert_ssz_eq(left: &T, right: &T) { assert_eq!( left.to_ssz().expect("left should serialize"), @@ -257,8 +308,6 @@ mod tests { ); } - // --- blocks ----------------------------------------------------------- - #[test] fn get_block_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -314,8 +363,6 @@ mod tests { assert_ssz_eq(&updated, &read); } - // --- states ----------------------------------------------------------- - #[test] fn get_state_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -357,8 +404,6 @@ mod tests { } } - // --- checkpoints (justified / finalized / head / genesis_time) --------- - #[test] fn justified_checkpoint_roundtrips() { let (_dir, storage) = storage(); @@ -418,8 +463,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), 1_700_000_000); } - // All four values above share the single `checkpoints` database under - // distinct string keys; verify they never clobber one another. #[test] fn checkpoints_database_keys_do_not_collide() { let (_dir, storage) = storage(); @@ -451,7 +494,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); } - // --- slot_index ------------------------------------------------------- #[test] fn block_root_by_slot_returns_none_when_absent() { @@ -491,7 +533,6 @@ mod tests { } } - // --- state_root_index ------------------------------------------------- #[test] fn block_root_by_state_root_returns_none_when_absent() { From eaffc221d6cc7f26d923dea19b369cded0769ebc Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 22:38:49 +0530 Subject: [PATCH 10/11] feat: fully spec-compliant storage API --- lean_client/storage/Cargo.toml | 7 - lean_client/storage/src/lib.rs | 690 +++++++++++++++++++++++++-------- 2 files changed, 530 insertions(+), 167 deletions(-) diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index ba850c47..2a05808a 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -7,20 +7,13 @@ anyhow = { workspace = true } bytesize = { workspace = true } containers = { workspace = true } fs-err = { workspace = true } -futures = { workspace = true } -im = { workspace = true } -itertools = { workspace = true } libmdbx = { workspace = true } lz4_flex = { workspace = true } ssz = { workspace = true } -tap = { workspace = true } -thiserror = { workspace = true } -unwrap_none = { workspace = true } zstd = { workspace = true } [dev-dependencies] tempfile = { workspace = true } -test-case = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index d870883d..cba84a52 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,8 +1,4 @@ -use std::{ - borrow::Cow, - path::Path, - sync::Arc, -}; +use std::{borrow::Cow, cell::RefCell, collections::HashSet, path::Path, sync::Arc}; use anyhow::Result; use bytesize::ByteSize; @@ -10,120 +6,31 @@ use bytesize::ByteSize; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; -use libmdbx::{ - DatabaseFlags, Environment, Geometry, RW, Transaction, WriteFlags, -}; +use libmdbx::{DatabaseFlags, Environment, Geometry, RW, Transaction, TransactionKind, WriteFlags}; + +const BLOCKS_TABLE_NAME: &str = "blocks"; +const STATES_TABLE_NAME: &str = "states"; +const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; +const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; +const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; +const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; +const STATES_BY_SLOT_TABLE_NAME: &str = "states_by_slot"; + +const JUSTIFIED: u8 = 0; +const FINALIZED: u8 = 1; +const HEAD: u8 = 2; +const GENESIS_TIME: u8 = 3; pub struct Storage { environment: Arc, + transaction: RefCell>>, blocks: Database, states: Database, checkpoints: Database, slot_index: Database, state_root_index: Database, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { - None, - #[default] - Lz4, - Zstd, -} - -impl Compression { - fn compress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), - Self::Zstd => Ok(zstd::encode_all(data, 3)?), - } - } - - fn decompress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), - Self::Zstd => Ok(zstd::decode_all(data)?), - } - } -} - -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - -pub struct Database { - environment: Arc, - name: String, - compression: Compression, -} - -impl Database { - pub fn new(env: Arc, name: &str, compression: Compression) -> Result { - Ok(Self { - environment: env, - name: name.to_owned(), - compression, - }) - } - - pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { - let txn = self.environment.begin_ro_txn()?; - let db = txn.open_db(Some(&self.name))?; - - txn.get::>(db.dbi(), key.as_ref())? - .map(|compressed| self.compression.decompress(&compressed)) - .transpose() - } - - pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { - let txn = self.environment.begin_rw_txn()?; - self.put_in(&txn, key, value)?; - txn.commit()?; - Ok(()) - } - - fn put_in( - &self, - txn: &Transaction, - key: impl AsRef<[u8]>, - value: impl AsRef<[u8]>, - ) -> Result<()> { - let db = txn.open_db(Some(&self.name))?; - let compressed = self.compression.compress(value.as_ref())?; - txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; - Ok(()) - } + blocks_by_slot: Database, + states_by_slot: Database, } impl Storage { @@ -133,7 +40,7 @@ impl Storage { let environment = Arc::new( Environment::builder() - .set_max_dbs(5) + .set_max_dbs(7) .set_geometry(Geometry { size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), @@ -145,132 +52,383 @@ impl Storage { let txn = environment.begin_rw_txn()?; for name in [ - "blocks", - "states", - "checkpoints", - "slot_index", - "state_root_index", + BLOCKS_TABLE_NAME, + STATES_TABLE_NAME, + CHECKPOINTS_TABLE_NAME, + SLOT_INDEX_TABLE_NAME, + STATE_ROOT_INDEX_TABLE_NAME, + BLOCKS_BY_SLOT_TABLE_NAME, + STATES_BY_SLOT_TABLE_NAME, ] { txn.create_db(Some(name), DatabaseFlags::default())?; } txn.commit()?; Ok(Self { - blocks: Database::new(environment.clone(), "blocks", Compression::Lz4)?, - states: Database::new(environment.clone(), "states", Compression::Zstd)?, - checkpoints: Database::new(environment.clone(), "checkpoints", Compression::None)?, - slot_index: Database::new(environment.clone(), "slot_index", Compression::None)?, - state_root_index: Database::new( - environment.clone(), - "state_root_index", - Compression::None, - )?, + blocks: Database::new(BLOCKS_TABLE_NAME, Compression::Lz4), + states: Database::new(STATES_TABLE_NAME, Compression::Zstd), + checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), + slot_index: Database::new(SLOT_INDEX_TABLE_NAME, Compression::None), + state_root_index: Database::new(STATE_ROOT_INDEX_TABLE_NAME, Compression::None), + blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), + states_by_slot: Database::new(STATES_BY_SLOT_TABLE_NAME, Compression::None), environment, + transaction: RefCell::new(None), }) } + pub fn batch_write(&self, f: impl FnOnce() -> Result<()>) -> Result<()> { + let txn = self.environment.begin_rw_txn()?; + *self.transaction.borrow_mut() = Some(txn); + + let result = f(); + + let txn = self + .transaction + .borrow_mut() + .take() + .expect("batch_write installed the ambient transaction"); + + match result { + Ok(()) => { + txn.commit()?; + Ok(()) + } + Err(error) => { + drop(txn); + Err(error) + } + } + } + + fn write(&self, f: impl FnOnce(&Transaction) -> Result) -> Result { + if let Some(txn) = self.transaction.borrow().as_ref() { + return f(txn); + } + let txn = self.environment.begin_rw_txn()?; + let result = f(&txn)?; + txn.commit()?; + Ok(result) + } + + fn read_bytes(&self, db: &Database, key: impl AsRef<[u8]>) -> Result>> { + if let Some(txn) = self.transaction.borrow().as_ref() { + return db.get(txn, key); + } + let txn = self.environment.begin_ro_txn()?; + let bytes = db.get(&txn, key)?; + txn.commit()?; + Ok(bytes) + } + pub fn get_block(&self, root: H256) -> Result> { - match self.blocks.get(root)? { - Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), + match self.read_bytes(&self.blocks, root)? { + Some(bytes) => Ok(Some(Block::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_block(&self, block: Block, root: H256) -> Result<()> { let block_bytes = block.to_ssz()?; - self.blocks.put(root, block_bytes)?; - Ok(()) + let index_key = slot_root_key(block.slot, root); + self.write(|txn| { + self.blocks.put(txn, root, block_bytes)?; + self.blocks_by_slot.put(txn, index_key, b"")?; + Ok(()) + }) } pub fn get_state(&self, root: H256) -> Result> { - match self.states.get(root)? { - Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), + match self.read_bytes(&self.states, root)? { + Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_state(&self, state: State, root: H256) -> Result<()> { let state_bytes = state.to_ssz()?; - self.states.put(root, state_bytes)?; - Ok(()) + let index_key = slot_root_key(state.slot, root); + self.write(|txn| { + self.states.put(txn, root, state_bytes)?; + self.states_by_slot.put(txn, index_key, b"")?; + Ok(()) + }) } pub fn get_justified_checkpoint(&self) -> Result> { - match self.checkpoints.get("justified")? { - Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + match self.read_bytes(&self.checkpoints, [JUSTIFIED])? { + Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.put("justified", checkpoint_bytes)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [JUSTIFIED], checkpoint_bytes)) } pub fn get_finalized_checkpoint(&self) -> Result> { - match self.checkpoints.get("finalized")? { - Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + match self.read_bytes(&self.checkpoints, [FINALIZED])? { + Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.put("finalized", checkpoint_bytes)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [FINALIZED], checkpoint_bytes)) } pub fn get_head_root(&self) -> Result> { - match self.checkpoints.get("head")? { - Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), + match self.read_bytes(&self.checkpoints, [HEAD])? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_head_root(&self, root: H256) -> Result<()> { - self.checkpoints.put("head", root)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [HEAD], root)) } pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { - let slot_bytes = slot.0.to_be_bytes(); - match self.slot_index.get(slot_bytes)? { - Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + match self.read_bytes(&self.slot_index, slot.0.to_be_bytes())? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { let slot_bytes = slot.0.to_be_bytes(); - self.slot_index.put(slot_bytes, root)?; - Ok(()) + self.write(|txn| self.slot_index.put(txn, slot_bytes, root)) } pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { - match self.state_root_index.get(state_root)? { - Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + match self.read_bytes(&self.state_root_index, state_root)? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { - self.state_root_index.put(state_root, block_root)?; - Ok(()) + self.write(|txn| self.state_root_index.put(txn, state_root, block_root)) } pub fn get_genesis_time(&self) -> Result> { - match self.checkpoints.get("genesis_time")? { - Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), + match self.read_bytes(&self.checkpoints, [GENESIS_TIME])? { + Some(bytes) => Ok(Some(u64::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { let genesis_time_bytes = genesis_time.to_ssz()?; - self.checkpoints.put("genesis_time", genesis_time_bytes)?; + self.write(|txn| { + self.checkpoints + .put(txn, [GENESIS_TIME], genesis_time_bytes) + }) + } + + pub fn prune_before_slot(&self, slot: Slot, keep_roots: &HashSet) -> Result { + let bound = slot.0; + self.write(|txn| { + let mut block_roots: Vec = Vec::new(); + let mut block_index_keys: Vec> = Vec::new(); + self.blocks_by_slot.for_each(txn, |key, _value| { + if slot_of(key) >= bound { + return Ok(false); + } + let root = H256::from_slice(&key[8..]); + if !keep_roots.contains(&root) { + block_roots.push(root); + block_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut state_roots: Vec = Vec::new(); + let mut state_index_keys: Vec> = Vec::new(); + self.states_by_slot.for_each(txn, |key, _value| { + if slot_of(key) >= bound { + return Ok(false); + } + let root = H256::from_slice(&key[8..]); + if !keep_roots.contains(&root) { + state_roots.push(root); + state_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut slot_index_keys: Vec> = Vec::new(); + self.slot_index.for_each(txn, |key, value| { + if slot_of(key) >= bound { + return Ok(false); + } + let canonical_root = H256::from_slice(value); + if !keep_roots.contains(&canonical_root) { + slot_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut removed = 0; + removed += self.blocks.delete_batch(txn, &block_roots)?; + removed += self.blocks_by_slot.delete_batch(txn, &block_index_keys)?; + removed += self.slot_index.delete_batch(txn, &slot_index_keys)?; + removed += self.states.delete_batch(txn, &state_roots)?; + removed += self.states_by_slot.delete_batch(txn, &state_index_keys)?; + removed += self.state_root_index.delete_batch(txn, &state_roots)?; + + Ok(removed) + }) + } +} + +fn slot_root_key(slot: Slot, root: H256) -> [u8; 40] { + let mut key = [0u8; 40]; + key[..8].copy_from_slice(&slot.0.to_be_bytes()); + key[8..].copy_from_slice(root.as_ref()); + key +} + +fn slot_of(key: &[u8]) -> u64 { + let mut slot = [0u8; 8]; + slot.copy_from_slice(&key[..8]); + u64::from_be_bytes(slot) +} + +pub struct Database { + name: String, + compression: Compression, +} + +impl Database { + pub fn new(name: &str, compression: Compression) -> Self { + Self { + name: name.to_owned(), + compression, + } + } + + pub fn get( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + ) -> Result>> { + let db = txn.open_db(Some(&self.name))?; + + txn.get::>(db.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + .transpose() + } + + pub fn put( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + value: impl AsRef<[u8]>, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let compressed = self.compression.compress(value.as_ref())?; + txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; Ok(()) } + + pub fn for_each( + &self, + txn: &Transaction, + mut f: impl FnMut(&[u8], &[u8]) -> Result, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let mut cursor = txn.cursor(&db)?; + + while let Some((key, value)) = cursor.next::, Cow<[u8]>>()? { + let value = self.compression.decompress(&value)?; + if !f(key.as_ref(), &value)? { + break; + } + } + + Ok(()) + } + + pub fn delete_batch( + &self, + txn: &Transaction, + keys: impl IntoIterator>, + ) -> Result { + let db = txn.open_db(Some(&self.name))?; + let mut cursor = txn.cursor(&db)?; + + let mut deleted = 0; + for key in keys { + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + deleted += 1; + } + } + + Ok(deleted) + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } } #[cfg(test)] @@ -494,7 +652,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); } - #[test] fn block_root_by_slot_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -533,7 +690,6 @@ mod tests { } } - #[test] fn block_root_by_state_root_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -586,4 +742,218 @@ mod tests { ); } } + + fn state_at_slot(slot: u64) -> State { + let mut state = State::generate_genesis(1_234, 3); + state.slot = Slot(slot); + state + } + + #[test] + fn prune_removes_blocks_below_slot_and_keeps_the_rest() { + let (_dir, storage) = storage(); + for i in 0..10u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); + assert_eq!(removed, 10); + + for i in 0..5u8 { + assert!(storage.get_block(h256(i)).unwrap().is_none()); + } + for i in 5..10u8 { + assert!(storage.get_block(h256(i)).unwrap().is_some()); + } + } + + #[test] + fn prune_keeps_frozen_block_root() { + let (_dir, storage) = storage(); + for i in 0..5u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let mut keep = HashSet::new(); + keep.insert(h256(2)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block(h256(2)).unwrap().is_some()); + for i in [0u8, 1, 3, 4] { + assert!(storage.get_block(h256(i)).unwrap().is_none()); + } + } + + #[test] + fn prune_removes_forked_blocks_at_same_slot() { + let (_dir, storage) = storage(); + let mut fork = sample_block(2); + fork.proposer_index = 99; + storage.put_block(sample_block(2), h256(20)).unwrap(); + storage.put_block(fork, h256(21)).unwrap(); + + let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); + + assert_eq!(removed, 4); + assert!(storage.get_block(h256(20)).unwrap().is_none()); + assert!(storage.get_block(h256(21)).unwrap().is_none()); + } + + #[test] + fn prune_cleans_slot_index_unless_frozen() { + let (_dir, storage) = storage(); + storage.put_block(sample_block(2), h256(2)).unwrap(); + storage.put_block_root_by_slot(Slot(2), h256(2)).unwrap(); + storage.put_block(sample_block(3), h256(3)).unwrap(); + storage.put_block_root_by_slot(Slot(3), h256(3)).unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(3)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block_root_by_slot(Slot(2)).unwrap().is_none()); + assert_eq!( + storage.get_block_root_by_slot(Slot(3)).unwrap().unwrap(), + h256(3), + ); + } + + #[test] + fn prune_removes_states_below_slot_and_keeps_frozen() { + let (_dir, storage) = storage(); + storage.put_state(state_at_slot(1), h256(1)).unwrap(); + storage.put_state(state_at_slot(2), h256(2)).unwrap(); + storage + .put_block_root_by_state_root(h256(1), h256(101)) + .unwrap(); + storage + .put_block_root_by_state_root(h256(2), h256(102)) + .unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(2)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_state(h256(1)).unwrap().is_none()); + assert!(storage.get_state(h256(2)).unwrap().is_some()); + assert!( + storage + .get_block_root_by_state_root(h256(1)) + .unwrap() + .is_none() + ); + assert_eq!( + storage + .get_block_root_by_state_root(h256(2)) + .unwrap() + .unwrap(), + h256(102), + ); + } + + #[test] + fn prune_protection_is_per_entity() { + let (_dir, storage) = storage(); + storage.put_block(sample_block(1), h256(1)).unwrap(); + storage.put_state(state_at_slot(1), h256(1)).unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(1)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_state(h256(1)).unwrap().is_some()); + } + + #[test] + fn prune_below_lowest_slot_is_a_noop() { + let (_dir, storage) = storage(); + for i in 0..5u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let removed = storage.prune_before_slot(Slot(0), &HashSet::new()).unwrap(); + assert_eq!(removed, 0); + for i in 0..5u8 { + assert!(storage.get_block(h256(i)).unwrap().is_some()); + } + } + + #[test] + fn batch_write_commits_all_on_success() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| { + storage.put_block(sample_block(1), h256(1))?; + storage.put_state(state_at_slot(1), h256(2))?; + storage.put_head_root(h256(3))?; + Ok(()) + }) + .unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_state(h256(2)).unwrap().is_some()); + assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); + } + + #[test] + fn batch_write_rolls_back_every_write_on_error() { + let (_dir, storage) = storage(); + + let result = storage.batch_write(|| { + storage.put_block(sample_block(1), h256(1))?; + storage.put_state(state_at_slot(1), h256(2))?; + Err(anyhow::anyhow!("intentional failure")) + }); + + assert!(result.is_err()); + assert!(storage.get_block(h256(1)).unwrap().is_none()); + assert!(storage.get_state(h256(2)).unwrap().is_none()); + } + + #[test] + fn storage_is_usable_after_batch_write() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| storage.put_block(sample_block(1), h256(1))) + .unwrap(); + + storage.put_block(sample_block(2), h256(2)).unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_block(h256(2)).unwrap().is_some()); + } + + #[test] + fn storage_is_usable_after_failed_batch_write() { + let (_dir, storage) = storage(); + + assert!( + storage + .batch_write(|| Err(anyhow::anyhow!("boom"))) + .is_err() + ); + + storage.put_block(sample_block(5), h256(5)).unwrap(); + assert!(storage.get_block(h256(5)).unwrap().is_some()); + } + + #[test] + fn batch_write_reads_see_pending_writes() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| { + storage.put_block(sample_block(7), h256(7)).unwrap(); + assert!(storage.get_block(h256(7)).unwrap().is_some()); + Ok(()) + }) + .unwrap(); + } } From 5f1fdb88da20f6e836fe5dd5056b0763adb2fba8 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 22:59:38 +0530 Subject: [PATCH 11/11] chore: privatized Database and removed DatabaseMode --- lean_client/Cargo.lock | 46 ------------------------------- lean_client/Cargo.toml | 4 --- lean_client/storage/src/lib.rs | 49 ++++++---------------------------- 3 files changed, 8 insertions(+), 91 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index dd763c61..b52568f7 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -5839,17 +5839,10 @@ dependencies = [ "bytesize", "containers", "fs-err", - "futures", - "im", - "itertools 0.14.0", "lz4_flex", "reth-libmdbx", "ssz", - "tap", "tempfile", - "test-case", - "thiserror 2.0.18", - "unwrap_none", "zstd", ] @@ -6033,39 +6026,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "test-case" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" -dependencies = [ - "test-case-macros", -] - -[[package]] -name = "test-case-core" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" -dependencies = [ - "cfg-if", - "proc-macro2 1.0.106", - "quote 1.0.45", - "syn 2.0.117", -] - -[[package]] -name = "test-case-macros" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" -dependencies = [ - "proc-macro2 1.0.106", - "quote 1.0.45", - "syn 2.0.117", - "test-case-core", -] - [[package]] name = "test-generator" version = "0.3.1" @@ -6605,12 +6565,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unwrap_none" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461d0c5956fcc728ecc03a3a961e4adc9a7975d86f6f8371389a289517c02ca9" - [[package]] name = "url" version = "2.5.8" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 7746917a..ac52e68a 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -255,7 +255,6 @@ hex = "0.4.3" indexmap = "2" http-body-util = "0.1" http_api_utils = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } -im = "15" k256 = "0.13" rec_aggregation = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } backend = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } @@ -299,9 +298,7 @@ snap = "1.1" ssz = { git = "https://github.com/grandinetech/grandine", package = "ssz", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } ssz-types = "0.3" itertools = "0.14" -tap = "1" tempfile = "3" -test-case = "3" test-generator = "0.3.1" thiserror = "2" tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } @@ -314,7 +311,6 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } tree-hash = "0.4.0" try_from_iterator = { git = "https://github.com/grandinetech/grandine", package = "try_from_iterator", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } typenum = "1.19" -unwrap_none = "0.1" yamux = "0.12" zeroize = "1.8" zstd = "0.13.3" diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index cba84a52..19436339 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -47,7 +47,7 @@ impl Storage { shrink_threshold: None, page_size: None, }) - .open_with_permissions(base, DatabaseMode::ReadWrite.permissions())?, + .open_with_permissions(base, 0o600)?, ); let txn = environment.begin_rw_txn()?; @@ -297,20 +297,20 @@ fn slot_of(key: &[u8]) -> u64 { u64::from_be_bytes(slot) } -pub struct Database { +struct Database { name: String, compression: Compression, } impl Database { - pub fn new(name: &str, compression: Compression) -> Self { + fn new(name: &str, compression: Compression) -> Self { Self { name: name.to_owned(), compression, } } - pub fn get( + fn get( &self, txn: &Transaction, key: impl AsRef<[u8]>, @@ -322,7 +322,7 @@ impl Database { .transpose() } - pub fn put( + fn put( &self, txn: &Transaction, key: impl AsRef<[u8]>, @@ -334,7 +334,7 @@ impl Database { Ok(()) } - pub fn for_each( + fn for_each( &self, txn: &Transaction, mut f: impl FnMut(&[u8], &[u8]) -> Result, @@ -352,7 +352,7 @@ impl Database { Ok(()) } - pub fn delete_batch( + fn delete_batch( &self, txn: &Transaction, keys: impl IntoIterator>, @@ -373,7 +373,7 @@ impl Database { } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { +enum Compression { None, #[default] Lz4, @@ -398,39 +398,6 @@ impl Compression { } } -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - #[cfg(test)] mod tests { use super::*;