diff --git a/Cargo.lock b/Cargo.lock index 2d94c4c0bcd..b10d62ef57c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -366,12 +366,14 @@ dependencies = [ "alloy-network", "alloy-network-primitives", "alloy-primitives", + "alloy-pubsub", "alloy-rpc-client", "alloy-rpc-types-eth", "alloy-signer", "alloy-sol-types", "alloy-transport", "alloy-transport-http", + "alloy-transport-ws", "async-stream", "async-trait", "auto_impl", @@ -392,6 +394,28 @@ dependencies = [ "wasmtimer", ] +[[package]] +name = "alloy-pubsub" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd4c64eb250a18101d22ae622357c6b505e158e9165d4c7974d59082a600c5e" +dependencies = [ + "alloy-json-rpc", + "alloy-primitives", + "alloy-transport", + "auto_impl", + "bimap", + "futures", + "parking_lot", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tracing", + "wasmtimer", +] + [[package]] name = "alloy-rlp" version = "0.3.12" @@ -422,8 +446,10 @@ checksum = "d0882e72d2c1c0c79dcf4ab60a67472d3f009a949f774d4c17d0bdb669cfde05" dependencies = [ "alloy-json-rpc", "alloy-primitives", + "alloy-pubsub", "alloy-transport", "alloy-transport-http", + "alloy-transport-ws", "futures", "pin-project", "reqwest", @@ -619,6 +645,24 @@ dependencies = [ "url", ] +[[package]] +name = "alloy-transport-ws" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad2344a12398d7105e3722c9b7a7044ea837128e11d453604dec6e3731a86e2" +dependencies = [ + "alloy-pubsub", + "alloy-transport", + "futures", + "http 1.3.1", + "rustls 0.23.35", + "serde_json", + "tokio", + "tokio-tungstenite", + "tracing", + "ws_stream_wasm", +] + [[package]] name = "alloy-trie" version = "0.9.1" @@ -698,7 +742,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -709,7 +753,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1081,6 +1125,17 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures", + "pharos", + "rustc_version 0.4.1", +] + [[package]] name = "asynchronous-codec" version = "0.7.0" @@ -1373,6 +1428,12 @@ dependencies = [ "types", ] +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "bincode" version = "1.3.3" @@ -3159,7 +3220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3376,6 +3437,22 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-client" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ddc25e1ad2cc0106d5e2d967397b4fb2068a66677ee9b0eea4600e5cfe8fb4" +dependencies = [ + "futures", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "hyper-timeout 0.4.1", + "log", + "pin-project", + "rand 0.8.5", + "tokio", +] + [[package]] name = "eventsource-stream" version = "0.2.3" @@ -3387,6 +3464,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "execution-witness-sentry" +version = "0.1.0" +dependencies = [ + "alloy-provider", + "alloy-rpc-types-eth", + "anyhow", + "clap", + "discv5", + "eventsource-client", + "flate2", + "futures", + "reqwest", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "toml", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -4403,6 +4503,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -4417,7 +4533,19 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots", + "webpki-roots 1.0.4", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.32", + "pin-project-lite", + "tokio", + "tokio-io-timeout", ] [[package]] @@ -4803,7 +4931,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6396,7 +6524,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6850,6 +6978,16 @@ dependencies = [ "ucd-trie", ] +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures", + "rustc_version 0.4.1", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -7060,7 +7198,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit", + "toml_edit 0.23.7", ] [[package]] @@ -7572,7 +7710,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.8.1", - "hyper-rustls", + "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", "js-sys", @@ -7599,7 +7737,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 1.0.4", ] [[package]] @@ -7697,9 +7835,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.0" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68df0380e5c9d20ce49534f292a36a7514ae21350726efe1865bdb1fa91d278" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" dependencies = [ "alloy-rlp", "arbitrary", @@ -7843,7 +7981,19 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", ] [[package]] @@ -7875,6 +8025,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.2" @@ -7887,6 +8049,15 @@ dependencies = [ "security-framework 3.5.1", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -7906,6 +8077,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -8053,6 +8234,16 @@ dependencies = [ "sha2 0.9.9", ] +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sec1" version = "0.7.3" @@ -8153,6 +8344,12 @@ dependencies = [ "pest", ] +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "sensitive_url" version = "0.1.0" @@ -8237,6 +8434,15 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -8933,7 +9139,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9172,6 +9378,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.6.0" @@ -9193,6 +9409,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -9226,6 +9452,22 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "rustls 0.23.35", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.17" @@ -9241,6 +9483,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.3" @@ -9250,6 +9513,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.12.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow", +] + [[package]] name = "toml_edit" version = "0.23.7" @@ -9257,7 +9534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ "indexmap 2.12.0", - "toml_datetime", + "toml_datetime 0.7.3", "toml_parser", "winnow", ] @@ -9271,6 +9548,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.12.3" @@ -9287,7 +9570,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.8.1", - "hyper-timeout", + "hyper-timeout 0.5.2", "hyper-util", "percent-encoding", "pin-project", @@ -9314,12 +9597,12 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.8.1", - "hyper-timeout", + "hyper-timeout 0.5.2", "hyper-util", "percent-encoding", "pin-project", "prost", - "rustls-native-certs", + "rustls-native-certs 0.8.2", "tokio", "tokio-rustls 0.26.4", "tokio-stream", @@ -9554,6 +9837,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http 1.3.1", + "httparse", + "log", + "rand 0.9.2", + "rustls 0.23.35", + "rustls-pki-types", + "sha1", + "thiserror 2.0.17", + "utf-8", +] + [[package]] name = "typenum" version = "1.19.0" @@ -9733,6 +10035,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -10042,7 +10350,7 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "scoped-tls", "serde", "serde_json", @@ -10223,6 +10531,15 @@ dependencies = [ "zip", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.4", +] + [[package]] name = "webpki-roots" version = "1.0.4" @@ -10278,7 +10595,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -10723,6 +11040,25 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "ws_stream_wasm" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures", + "js-sys", + "log", + "pharos", + "rustc_version 0.4.1", + "send_wrapper", + "thiserror 2.0.17", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index ba2316bb034..19158eb29e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ members = [ "crypto/kzg", "database_manager", "dummy_el", + "execution-witness-sentry", "lcli", "lighthouse", "lighthouse/environment", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index dfd28cd9572..d6990894018 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4149,6 +4149,24 @@ impl BeaconChain { // This prevents inconsistency between the two at the expense of concurrency. drop(fork_choice); + // Persist execution proofs to the database if zkvm is enabled and proofs are cached. + // This is done after the block is successfully stored so we don't lose proofs on cache eviction. + if let Some(proofs) = self + .data_availability_checker + .get_execution_proofs(&block_root) + && !proofs.is_empty() + { + let proofs_owned: Vec<_> = proofs.iter().map(|p| (**p).clone()).collect(); + if let Err(e) = self.store.put_execution_proofs(&block_root, &proofs_owned) { + // Log but don't fail block import - proofs can still be served from cache + warn!( + %block_root, + error = ?e, + "Failed to persist execution proofs to database" + ); + } + } + // We're declaring the block "imported" at this point, since fork choice and the DB know // about it. let block_time_imported = timestamp_now(); diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index bc5b41b09e1..feabcd5f44a 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1159,6 +1159,13 @@ where .process_prune_blobs(data_availability_boundary); } + // Prune execution proofs older than the execution proof boundary in the background. + if let Some(execution_proof_boundary) = beacon_chain.execution_proof_boundary() { + beacon_chain + .store_migrator + .process_prune_execution_proofs(execution_proof_boundary); + } + Ok(beacon_chain) } } diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 228e5eb2d27..17dc227430b 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1034,6 +1034,12 @@ impl BeaconChain { .process_prune_blobs(data_availability_boundary); } + // Prune execution proofs in the background. + if let Some(execution_proof_boundary) = self.execution_proof_boundary() { + self.store_migrator + .process_prune_execution_proofs(execution_proof_boundary); + } + // Take a write-lock on the canonical head and signal for it to prune. self.canonical_head.fork_choice_write_lock().prune()?; diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index bd232f2e8a2..e290cf510f0 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -120,6 +120,7 @@ pub enum Notification { Finalization(FinalizationNotification), Reconstruction, PruneBlobs(Epoch), + PruneExecutionProofs(Epoch), ManualFinalization(ManualFinalizationNotification), ManualCompaction, } @@ -251,6 +252,28 @@ impl, Cold: ItemStore> BackgroundMigrator>, + execution_proof_boundary: Epoch, + ) { + if let Err(e) = db.try_prune_execution_proofs(false, execution_proof_boundary) { + error!( + error = ?e, + "Execution proof pruning failed" + ); + } + } + /// If configured to run in the background, send `notif` to the background thread. /// /// Return `None` if the message was sent to the background thread, `Some(notif)` otherwise. @@ -440,11 +463,15 @@ impl, Cold: ItemStore> BackgroundMigrator reconstruction_notif = Some(notif), Notification::Finalization(fin) => finalization_notif = Some(fin), Notification::ManualFinalization(fin) => manual_finalization_notif = Some(fin), Notification::PruneBlobs(dab) => prune_blobs_notif = Some(dab), + Notification::PruneExecutionProofs(epb) => { + prune_execution_proofs_notif = Some(epb) + } Notification::ManualCompaction => manual_compaction_notif = Some(notif), } // Read the rest of the messages in the channel, taking the best of each type. @@ -475,6 +502,10 @@ impl, Cold: ItemStore> BackgroundMigrator { prune_blobs_notif = std::cmp::max(prune_blobs_notif, Some(dab)); } + Notification::PruneExecutionProofs(epb) => { + prune_execution_proofs_notif = + std::cmp::max(prune_execution_proofs_notif, Some(epb)); + } } } // Run finalization and blob pruning migrations first, then a reconstruction batch. @@ -489,6 +520,9 @@ impl, Cold: ItemStore> BackgroundMigrator = DBColumn::iter().map(|c| c.as_str()).collect(); let expected_columns = vec![ - "bma", "blk", "blb", "bdc", "bdi", "ste", "hsd", "hsn", "bsn", "bsd", "bss", "bs3", "bcs", - "bst", "exp", "bch", "opo", "etc", "frk", "pkc", "brp", "bsx", "bsr", "bbx", "bbr", "bhr", - "brm", "dht", "cus", "otb", "bhs", "olc", "lcu", "scb", "scm", "dmy", + "bma", "blk", "blb", "bdc", "bdi", "bep", "ste", "hsd", "hsn", "bsn", "bsd", "bss", "bs3", + "bcs", "bst", "exp", "bch", "opo", "etc", "frk", "pkc", "brp", "bsx", "bsr", "bbx", "bbr", + "bhr", "brm", "dht", "cus", "otb", "bhs", "olc", "lcu", "scb", "scm", "dmy", ]; assert_eq!(expected_columns, current_columns); } diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 6f5170be300..f98d57e5cb6 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -129,6 +129,7 @@ pub struct BeaconProcessorQueueLengths { block_broots_queue: usize, blob_broots_queue: usize, execution_proof_broots_queue: usize, + execution_proof_brange_queue: usize, blob_brange_queue: usize, dcbroots_queue: usize, dcbrange_queue: usize, @@ -198,6 +199,7 @@ impl BeaconProcessorQueueLengths { block_broots_queue: 1024, blob_broots_queue: 1024, execution_proof_broots_queue: 1024, + execution_proof_brange_queue: 1024, blob_brange_queue: 1024, dcbroots_queue: 1024, dcbrange_queue: 1024, @@ -620,6 +622,7 @@ pub enum Work { BlobsByRangeRequest(BlockingFn), BlobsByRootsRequest(BlockingFn), ExecutionProofsByRootsRequest(BlockingFn), + ExecutionProofsByRangeRequest(BlockingFn), DataColumnsByRootsRequest(BlockingFn), DataColumnsByRangeRequest(BlockingFn), GossipBlsToExecutionChange(BlockingFn), @@ -675,6 +678,7 @@ pub enum WorkType { BlobsByRangeRequest, BlobsByRootsRequest, ExecutionProofsByRootsRequest, + ExecutionProofsByRangeRequest, DataColumnsByRootsRequest, DataColumnsByRangeRequest, GossipBlsToExecutionChange, @@ -728,6 +732,7 @@ impl Work { Work::BlobsByRangeRequest(_) => WorkType::BlobsByRangeRequest, Work::BlobsByRootsRequest(_) => WorkType::BlobsByRootsRequest, Work::ExecutionProofsByRootsRequest(_) => WorkType::ExecutionProofsByRootsRequest, + Work::ExecutionProofsByRangeRequest(_) => WorkType::ExecutionProofsByRangeRequest, Work::DataColumnsByRootsRequest(_) => WorkType::DataColumnsByRootsRequest, Work::DataColumnsByRangeRequest(_) => WorkType::DataColumnsByRangeRequest, Work::LightClientBootstrapRequest(_) => WorkType::LightClientBootstrapRequest, @@ -901,6 +906,8 @@ impl BeaconProcessor { let mut blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); let mut execution_proof_broots_queue = FifoQueue::new(queue_lengths.execution_proof_broots_queue); + let mut execution_proof_brange_queue = + FifoQueue::new(queue_lengths.execution_proof_brange_queue); let mut blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); let mut dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); let mut dcbrange_queue = FifoQueue::new(queue_lengths.dcbrange_queue); @@ -1226,6 +1233,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = execution_proof_broots_queue.pop() { Some(item) + } else if let Some(item) = execution_proof_brange_queue.pop() { + Some(item) } else if let Some(item) = dcbroots_queue.pop() { Some(item) } else if let Some(item) = dcbrange_queue.pop() { @@ -1430,6 +1439,9 @@ impl BeaconProcessor { Work::ExecutionProofsByRootsRequest { .. } => { execution_proof_broots_queue.push(work, work_id) } + Work::ExecutionProofsByRangeRequest { .. } => { + execution_proof_brange_queue.push(work, work_id) + } Work::DataColumnsByRootsRequest { .. } => { dcbroots_queue.push(work, work_id) } @@ -1489,6 +1501,9 @@ impl BeaconProcessor { WorkType::ExecutionProofsByRootsRequest => { execution_proof_broots_queue.len() } + WorkType::ExecutionProofsByRangeRequest => { + execution_proof_brange_queue.len() + } WorkType::DataColumnsByRootsRequest => dcbroots_queue.len(), WorkType::DataColumnsByRangeRequest => dcbrange_queue.len(), WorkType::GossipBlsToExecutionChange => { @@ -1649,6 +1664,7 @@ impl BeaconProcessor { Work::BlobsByRangeRequest(process_fn) | Work::BlobsByRootsRequest(process_fn) | Work::ExecutionProofsByRootsRequest(process_fn) + | Work::ExecutionProofsByRangeRequest(process_fn) | Work::DataColumnsByRootsRequest(process_fn) | Work::DataColumnsByRangeRequest(process_fn) => { task_spawner.spawn_blocking(process_fn) diff --git a/beacon_node/http_api/src/beacon/pool.rs b/beacon_node/http_api/src/beacon/pool.rs index 63b1a95b2ed..50a257db01b 100644 --- a/beacon_node/http_api/src/beacon/pool.rs +++ b/beacon_node/http_api/src/beacon/pool.rs @@ -530,6 +530,7 @@ pub fn post_beacon_pool_attestations_v2( /// Submits an execution proof to the beacon node. /// The proof will be validated and stored in the data availability checker. /// If valid, the proof will be published to the gossip network. +/// If the proof makes a block available, the block will be imported. pub fn post_beacon_pool_execution_proofs( network_tx_filter: &NetworkTxFilter, beacon_pool_path: &BeaconPoolPathFilter, @@ -541,81 +542,92 @@ pub fn post_beacon_pool_execution_proofs( .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |task_spawner: TaskSpawner, + |_task_spawner: TaskSpawner, chain: Arc>, proof: ExecutionProof, - network_tx: UnboundedSender>| { - task_spawner.blocking_json_task(Priority::P0, move || { - let proof = Arc::new(proof); - - // Validate the proof using the same logic as gossip validation - let verified_proof: GossipVerifiedExecutionProof = - GossipVerifiedExecutionProof::new(proof.clone(), &chain).map_err(|e| { - match e { - GossipExecutionProofError::PriorKnown { - slot, - block_root, - proof_id, - } => { - debug!( - %slot, - %block_root, - %proof_id, - "Execution proof already known" - ); - warp_utils::reject::custom_bad_request(format!( - "proof already known for slot {} block_root {} proof_id {}", - slot, block_root, proof_id - )) - } - GossipExecutionProofError::PriorKnownUnpublished => { - // Proof is valid but was received via non-gossip source - // It's in the DA checker, so we should publish it to gossip - warp_utils::reject::custom_bad_request( - "proof already received but not yet published".to_string(), - ) - } - _ => warp_utils::reject::object_invalid(format!( - "proof verification failed: {:?}", - e - )), - } - })?; - - let slot = verified_proof.slot(); - let block_root = verified_proof.block_root(); - let proof_id = verified_proof.subnet_id(); - - // Publish the proof to the gossip network - utils::publish_pubsub_message( - &network_tx, - PubsubMessage::ExecutionProof(verified_proof.clone().into_inner()), - )?; - - // Store the proof in the data availability checker - if let Err(e) = chain - .data_availability_checker - .put_rpc_execution_proofs(block_root, vec![verified_proof.into_inner()]) - { - warn!( - %slot, - %block_root, - %proof_id, - error = ?e, - "Failed to store execution proof in DA checker" - ); - } - - info!( - %slot, - %block_root, - %proof_id, - "Execution proof submitted and published" - ); - - Ok(()) - }) + network_tx: UnboundedSender>| async move { + let result = publish_execution_proof(chain, proof, network_tx).await; + convert_rejection(result.map(|()| warp::reply::json(&()))).await }, ) .boxed() } + +/// Validate, publish, and process an execution proof. +async fn publish_execution_proof( + chain: Arc>, + proof: ExecutionProof, + network_tx: UnboundedSender>, +) -> Result<(), warp::Rejection> { + let proof = Arc::new(proof); + + // Validate the proof using the same logic as gossip validation + let verified_proof: GossipVerifiedExecutionProof = + GossipVerifiedExecutionProof::new(proof.clone(), &chain).map_err(|e| match e { + GossipExecutionProofError::PriorKnown { + slot, + block_root, + proof_id, + } => { + debug!( + %slot, + %block_root, + %proof_id, + "Execution proof already known" + ); + warp_utils::reject::custom_bad_request(format!( + "proof already known for slot {} block_root {} proof_id {}", + slot, block_root, proof_id + )) + } + GossipExecutionProofError::PriorKnownUnpublished => { + // Proof is valid but was received via non-gossip source + // It's in the DA checker, so we should publish it to gossip + warp_utils::reject::custom_bad_request( + "proof already received but not yet published".to_string(), + ) + } + _ => warp_utils::reject::object_invalid(format!("proof verification failed: {:?}", e)), + })?; + + let slot = verified_proof.slot(); + let block_root = verified_proof.block_root(); + let proof_id = verified_proof.subnet_id(); + + // Publish the proof to the gossip network + utils::publish_pubsub_message( + &network_tx, + PubsubMessage::ExecutionProof(verified_proof.clone().into_inner()), + )?; + + // Store the proof in the data availability checker and check if block is now available. + // This properly triggers block import if all components are now available. + match chain + .process_rpc_execution_proofs(slot, block_root, vec![verified_proof.into_inner()]) + .await + { + Ok(status) => { + info!( + %slot, + %block_root, + %proof_id, + ?status, + "Execution proof submitted and published" + ); + } + Err(e) => { + // Log the error but don't fail the request - the proof was already + // published to gossip and stored in the DA checker. The error is + // likely due to the block already being imported or similar. + debug!( + %slot, + %block_root, + %proof_id, + error = ?e, + "Error processing execution proof availability (proof was still published)" + ); + } + } + + Ok(()) +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 1b280d54035..52b98d4d3c7 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -606,6 +606,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => PeerAction::MidToleranceError, Protocol::DataColumnsByRange => PeerAction::MidToleranceError, Protocol::ExecutionProofsByRoot => PeerAction::MidToleranceError, + Protocol::ExecutionProofsByRange => PeerAction::MidToleranceError, Protocol::Goodbye => PeerAction::LowToleranceError, Protocol::MetaData => PeerAction::LowToleranceError, Protocol::Status => PeerAction::LowToleranceError, @@ -627,6 +628,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => return, Protocol::DataColumnsByRange => return, Protocol::ExecutionProofsByRoot => return, + Protocol::ExecutionProofsByRange => return, Protocol::Goodbye => return, Protocol::LightClientBootstrap => return, Protocol::LightClientOptimisticUpdate => return, @@ -651,6 +653,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => PeerAction::MidToleranceError, Protocol::DataColumnsByRange => PeerAction::MidToleranceError, Protocol::ExecutionProofsByRoot => PeerAction::MidToleranceError, + Protocol::ExecutionProofsByRange => PeerAction::MidToleranceError, Protocol::LightClientBootstrap => return, Protocol::LightClientOptimisticUpdate => return, Protocol::LightClientFinalityUpdate => return, diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index b3401038df8..aa0fe8a3d9d 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -81,6 +81,7 @@ impl SSZSnappyInboundCodec { RpcSuccessResponse::DataColumnsByRoot(res) => res.as_ssz_bytes(), RpcSuccessResponse::DataColumnsByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::ExecutionProofsByRoot(res) => res.as_ssz_bytes(), + RpcSuccessResponse::ExecutionProofsByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientBootstrap(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientOptimisticUpdate(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientFinalityUpdate(res) => res.as_ssz_bytes(), @@ -362,6 +363,7 @@ impl Encoder> for SSZSnappyOutboundCodec { RequestType::DataColumnsByRange(req) => req.as_ssz_bytes(), RequestType::DataColumnsByRoot(req) => req.data_column_ids.as_ssz_bytes(), RequestType::ExecutionProofsByRoot(req) => req.as_ssz_bytes(), + RequestType::ExecutionProofsByRange(req) => req.as_ssz_bytes(), RequestType::Ping(req) => req.as_ssz_bytes(), RequestType::LightClientBootstrap(req) => req.as_ssz_bytes(), RequestType::LightClientUpdatesByRange(req) => req.as_ssz_bytes(), @@ -578,6 +580,11 @@ fn handle_rpc_request( Ok(Some(RequestType::ExecutionProofsByRoot(request))) } + SupportedProtocol::ExecutionProofsByRangeV1 => { + Ok(Some(RequestType::ExecutionProofsByRange( + ExecutionProofsByRangeRequest::from_ssz_bytes(decoded_buffer)?, + ))) + } SupportedProtocol::PingV1 => Ok(Some(RequestType::Ping(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -746,6 +753,11 @@ fn handle_rpc_response( ExecutionProof::from_ssz_bytes(decoded_buffer)?, )))) } + SupportedProtocol::ExecutionProofsByRangeV1 => { + Ok(Some(RpcSuccessResponse::ExecutionProofsByRange(Arc::new( + ExecutionProof::from_ssz_bytes(decoded_buffer)?, + )))) + } SupportedProtocol::PingV1 => Ok(Some(RpcSuccessResponse::Pong(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -1295,6 +1307,12 @@ mod tests { RequestType::ExecutionProofsByRoot(exec_proofs) => { assert_eq!(decoded, RequestType::ExecutionProofsByRoot(exec_proofs)) } + RequestType::ExecutionProofsByRange(exec_proofs_range) => { + assert_eq!( + decoded, + RequestType::ExecutionProofsByRange(exec_proofs_range) + ) + } RequestType::Ping(ping) => { assert_eq!(decoded, RequestType::Ping(ping)) } diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index d23c16f8fa1..99c0f33da31 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -94,6 +94,7 @@ pub struct RateLimiterConfig { pub(super) data_columns_by_root_quota: Quota, pub(super) data_columns_by_range_quota: Quota, pub(super) execution_proofs_by_root_quota: Quota, + pub(super) execution_proofs_by_range_quota: Quota, pub(super) light_client_bootstrap_quota: Quota, pub(super) light_client_optimistic_update_quota: Quota, pub(super) light_client_finality_update_quota: Quota, @@ -126,6 +127,9 @@ impl RateLimiterConfig { // TODO(zkproofs): Configure this to be less arbitrary pub const DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); + // TODO(zkproofs): Configure this to be less arbitrary + pub const DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA: Quota = + Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_FINALITY_UPDATE_QUOTA: Quota = Quota::one_every(10); @@ -146,6 +150,7 @@ impl Default for RateLimiterConfig { data_columns_by_root_quota: Self::DEFAULT_DATA_COLUMNS_BY_ROOT_QUOTA, data_columns_by_range_quota: Self::DEFAULT_DATA_COLUMNS_BY_RANGE_QUOTA, execution_proofs_by_root_quota: Self::DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA, + execution_proofs_by_range_quota: Self::DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA, light_client_bootstrap_quota: Self::DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA, light_client_optimistic_update_quota: Self::DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA, @@ -184,6 +189,14 @@ impl Debug for RateLimiterConfig { "data_columns_by_root", fmt_q!(&self.data_columns_by_root_quota), ) + .field( + "execution_proofs_by_root", + fmt_q!(&self.execution_proofs_by_root_quota), + ) + .field( + "execution_proofs_by_range", + fmt_q!(&self.execution_proofs_by_range_quota), + ) .finish() } } @@ -207,6 +220,7 @@ impl FromStr for RateLimiterConfig { let mut data_columns_by_root_quota = None; let mut data_columns_by_range_quota = None; let mut execution_proofs_by_root_quota = None; + let mut execution_proofs_by_range_quota = None; let mut light_client_bootstrap_quota = None; let mut light_client_optimistic_update_quota = None; let mut light_client_finality_update_quota = None; @@ -231,6 +245,9 @@ impl FromStr for RateLimiterConfig { Protocol::ExecutionProofsByRoot => { execution_proofs_by_root_quota = execution_proofs_by_root_quota.or(quota) } + Protocol::ExecutionProofsByRange => { + execution_proofs_by_range_quota = execution_proofs_by_range_quota.or(quota) + } Protocol::Ping => ping_quota = ping_quota.or(quota), Protocol::MetaData => meta_data_quota = meta_data_quota.or(quota), Protocol::LightClientBootstrap => { @@ -268,6 +285,8 @@ impl FromStr for RateLimiterConfig { .unwrap_or(Self::DEFAULT_DATA_COLUMNS_BY_RANGE_QUOTA), execution_proofs_by_root_quota: execution_proofs_by_root_quota .unwrap_or(Self::DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA), + execution_proofs_by_range_quota: execution_proofs_by_range_quota + .unwrap_or(Self::DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA), light_client_bootstrap_quota: light_client_bootstrap_quota .unwrap_or(Self::DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA), light_client_optimistic_update_quota: light_client_optimistic_update_quota diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 9ba8f66dafa..966106b6f69 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -603,6 +603,36 @@ impl ExecutionProofsByRootRequest { } } +/// Request execution proofs for a range of slots. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct ExecutionProofsByRangeRequest { + /// The starting slot to request execution proofs. + pub start_slot: u64, + /// The number of slots from the start slot. + pub count: u64, +} + +impl ExecutionProofsByRangeRequest { + pub fn max_proofs_requested(&self) -> u64 { + // Each slot could have up to MAX_PROOFS execution proofs + self.count + .saturating_mul(types::execution_proof::MAX_PROOFS as u64) + } + + pub fn ssz_min_len() -> usize { + ExecutionProofsByRangeRequest { + start_slot: 0, + count: 0, + } + .as_ssz_bytes() + .len() + } + + pub fn ssz_max_len() -> usize { + Self::ssz_min_len() + } +} + /// Request a number of beacon data columns from a peer. #[derive(Encode, Decode, Clone, Debug, PartialEq)] pub struct LightClientUpdatesByRangeRequest { @@ -673,6 +703,9 @@ pub enum RpcSuccessResponse { /// A response to a get EXECUTION_PROOFS_BY_ROOT request. ExecutionProofsByRoot(Arc), + /// A response to a get EXECUTION_PROOFS_BY_RANGE request. + ExecutionProofsByRange(Arc), + /// A PONG response to a PING request. Pong(Ping), @@ -704,6 +737,9 @@ pub enum ResponseTermination { /// Execution proofs by root stream termination. ExecutionProofsByRoot, + /// Execution proofs by range stream termination. + ExecutionProofsByRange, + /// Light client updates by range stream termination. LightClientUpdatesByRange, } @@ -718,6 +754,7 @@ impl ResponseTermination { ResponseTermination::DataColumnsByRoot => Protocol::DataColumnsByRoot, ResponseTermination::DataColumnsByRange => Protocol::DataColumnsByRange, ResponseTermination::ExecutionProofsByRoot => Protocol::ExecutionProofsByRoot, + ResponseTermination::ExecutionProofsByRange => Protocol::ExecutionProofsByRange, ResponseTermination::LightClientUpdatesByRange => Protocol::LightClientUpdatesByRange, } } @@ -814,6 +851,7 @@ impl RpcSuccessResponse { RpcSuccessResponse::DataColumnsByRoot(_) => Protocol::DataColumnsByRoot, RpcSuccessResponse::DataColumnsByRange(_) => Protocol::DataColumnsByRange, RpcSuccessResponse::ExecutionProofsByRoot(_) => Protocol::ExecutionProofsByRoot, + RpcSuccessResponse::ExecutionProofsByRange(_) => Protocol::ExecutionProofsByRange, RpcSuccessResponse::Pong(_) => Protocol::Ping, RpcSuccessResponse::MetaData(_) => Protocol::MetaData, RpcSuccessResponse::LightClientBootstrap(_) => Protocol::LightClientBootstrap, @@ -840,6 +878,7 @@ impl RpcSuccessResponse { Self::LightClientUpdatesByRange(r) => Some(r.attested_header_slot()), // TODO(zkproofs): Change this when we add Slot to ExecutionProof Self::ExecutionProofsByRoot(_) + | Self::ExecutionProofsByRange(_) | Self::MetaData(_) | Self::Status(_) | Self::Pong(_) => None, @@ -905,6 +944,13 @@ impl std::fmt::Display for RpcSuccessResponse { RpcSuccessResponse::ExecutionProofsByRoot(proof) => { write!(f, "ExecutionProofsByRoot: Block root: {}", proof.block_root) } + RpcSuccessResponse::ExecutionProofsByRange(proof) => { + write!( + f, + "ExecutionProofsByRange: Block root: {}", + proof.block_root + ) + } RpcSuccessResponse::Pong(ping) => write!(f, "Pong: {}", ping.data), RpcSuccessResponse::MetaData(metadata) => { write!(f, "Metadata: {}", metadata.seq_number()) @@ -1027,3 +1073,13 @@ impl std::fmt::Display for ExecutionProofsByRootRequest { ) } } + +impl std::fmt::Display for ExecutionProofsByRangeRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Request: ExecutionProofsByRange: Start Slot: {}, Count: {}", + self.start_slot, self.count + ) + } +} diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index dfa44976390..0a37db0d210 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -254,6 +254,9 @@ pub enum Protocol { /// The `ExecutionProofsByRoot` protocol name. #[strum(serialize = "execution_proofs_by_root")] ExecutionProofsByRoot, + /// The `ExecutionProofsByRange` protocol name. + #[strum(serialize = "execution_proofs_by_range")] + ExecutionProofsByRange, /// The `Ping` protocol name. Ping, /// The `MetaData` protocol name. @@ -285,6 +288,7 @@ impl Protocol { Protocol::DataColumnsByRoot => Some(ResponseTermination::DataColumnsByRoot), Protocol::DataColumnsByRange => Some(ResponseTermination::DataColumnsByRange), Protocol::ExecutionProofsByRoot => Some(ResponseTermination::ExecutionProofsByRoot), + Protocol::ExecutionProofsByRange => Some(ResponseTermination::ExecutionProofsByRange), Protocol::Ping => None, Protocol::MetaData => None, Protocol::LightClientBootstrap => None, @@ -316,6 +320,7 @@ pub enum SupportedProtocol { DataColumnsByRootV1, DataColumnsByRangeV1, ExecutionProofsByRootV1, + ExecutionProofsByRangeV1, PingV1, MetaDataV1, MetaDataV2, @@ -341,6 +346,7 @@ impl SupportedProtocol { SupportedProtocol::DataColumnsByRootV1 => "1", SupportedProtocol::DataColumnsByRangeV1 => "1", SupportedProtocol::ExecutionProofsByRootV1 => "1", + SupportedProtocol::ExecutionProofsByRangeV1 => "1", SupportedProtocol::PingV1 => "1", SupportedProtocol::MetaDataV1 => "1", SupportedProtocol::MetaDataV2 => "2", @@ -366,6 +372,7 @@ impl SupportedProtocol { SupportedProtocol::DataColumnsByRootV1 => Protocol::DataColumnsByRoot, SupportedProtocol::DataColumnsByRangeV1 => Protocol::DataColumnsByRange, SupportedProtocol::ExecutionProofsByRootV1 => Protocol::ExecutionProofsByRoot, + SupportedProtocol::ExecutionProofsByRangeV1 => Protocol::ExecutionProofsByRange, SupportedProtocol::PingV1 => Protocol::Ping, SupportedProtocol::MetaDataV1 => Protocol::MetaData, SupportedProtocol::MetaDataV2 => Protocol::MetaData, @@ -417,10 +424,16 @@ impl SupportedProtocol { ]); } if fork_context.spec.is_zkvm_enabled() { - supported.push(ProtocolId::new( - SupportedProtocol::ExecutionProofsByRootV1, - Encoding::SSZSnappy, - )); + supported.extend_from_slice(&[ + ProtocolId::new( + SupportedProtocol::ExecutionProofsByRootV1, + Encoding::SSZSnappy, + ), + ProtocolId::new( + SupportedProtocol::ExecutionProofsByRangeV1, + Encoding::SSZSnappy, + ), + ]); } supported } @@ -535,6 +548,10 @@ impl ProtocolId { DataColumnsByRangeRequest::ssz_max_len::(), ), Protocol::ExecutionProofsByRoot => RpcLimits::new(0, spec.max_blocks_by_root_request), + Protocol::ExecutionProofsByRange => RpcLimits::new( + ExecutionProofsByRangeRequest::ssz_min_len(), + ExecutionProofsByRangeRequest::ssz_max_len(), + ), Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -572,6 +589,7 @@ impl ProtocolId { rpc_data_column_limits::(fork_context.current_fork_epoch(), &fork_context.spec) } Protocol::ExecutionProofsByRoot => rpc_execution_proof_limits(), + Protocol::ExecutionProofsByRange => rpc_execution_proof_limits(), Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -614,6 +632,7 @@ impl ProtocolId { | SupportedProtocol::BlocksByRootV1 | SupportedProtocol::BlocksByRangeV1 | SupportedProtocol::ExecutionProofsByRootV1 + | SupportedProtocol::ExecutionProofsByRangeV1 | SupportedProtocol::PingV1 | SupportedProtocol::MetaDataV1 | SupportedProtocol::MetaDataV2 @@ -748,6 +767,7 @@ pub enum RequestType { DataColumnsByRoot(DataColumnsByRootRequest), DataColumnsByRange(DataColumnsByRangeRequest), ExecutionProofsByRoot(ExecutionProofsByRootRequest), + ExecutionProofsByRange(ExecutionProofsByRangeRequest), LightClientBootstrap(LightClientBootstrapRequest), LightClientOptimisticUpdate, LightClientFinalityUpdate, @@ -772,6 +792,7 @@ impl RequestType { RequestType::DataColumnsByRoot(req) => req.max_requested() as u64, RequestType::DataColumnsByRange(req) => req.max_requested::(), RequestType::ExecutionProofsByRoot(req) => req.max_requested() as u64, + RequestType::ExecutionProofsByRange(req) => req.max_proofs_requested(), RequestType::Ping(_) => 1, RequestType::MetaData(_) => 1, RequestType::LightClientBootstrap(_) => 1, @@ -802,6 +823,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => SupportedProtocol::DataColumnsByRootV1, RequestType::DataColumnsByRange(_) => SupportedProtocol::DataColumnsByRangeV1, RequestType::ExecutionProofsByRoot(_) => SupportedProtocol::ExecutionProofsByRootV1, + RequestType::ExecutionProofsByRange(_) => SupportedProtocol::ExecutionProofsByRangeV1, RequestType::Ping(_) => SupportedProtocol::PingV1, RequestType::MetaData(req) => match req { MetadataRequest::V1(_) => SupportedProtocol::MetaDataV1, @@ -834,6 +856,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => ResponseTermination::DataColumnsByRoot, RequestType::DataColumnsByRange(_) => ResponseTermination::DataColumnsByRange, RequestType::ExecutionProofsByRoot(_) => ResponseTermination::ExecutionProofsByRoot, + RequestType::ExecutionProofsByRange(_) => ResponseTermination::ExecutionProofsByRange, RequestType::Status(_) => unreachable!(), RequestType::Goodbye(_) => unreachable!(), RequestType::Ping(_) => unreachable!(), @@ -884,6 +907,10 @@ impl RequestType { SupportedProtocol::ExecutionProofsByRootV1, Encoding::SSZSnappy, )], + RequestType::ExecutionProofsByRange(_) => vec![ProtocolId::new( + SupportedProtocol::ExecutionProofsByRangeV1, + Encoding::SSZSnappy, + )], RequestType::Ping(_) => vec![ProtocolId::new( SupportedProtocol::PingV1, Encoding::SSZSnappy, @@ -923,6 +950,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => false, RequestType::DataColumnsByRange(_) => false, RequestType::ExecutionProofsByRoot(_) => false, + RequestType::ExecutionProofsByRange(_) => false, RequestType::Ping(_) => true, RequestType::MetaData(_) => true, RequestType::LightClientBootstrap(_) => true, @@ -1039,6 +1067,9 @@ impl std::fmt::Display for RequestType { RequestType::ExecutionProofsByRoot(req) => { write!(f, "Execution proofs by root: {:?}", req) } + RequestType::ExecutionProofsByRange(req) => { + write!(f, "Execution proofs by range: {:?}", req) + } RequestType::Ping(ping) => write!(f, "Ping: {}", ping.data), RequestType::MetaData(_) => write!(f, "MetaData request"), RequestType::LightClientBootstrap(bootstrap) => { diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index f70b29cfe45..9dfbc668c89 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -107,6 +107,8 @@ pub struct RPCRateLimiter { dcbrange_rl: Limiter, /// ExecutionProofsByRoot rate limiter. execution_proofs_by_root_rl: Limiter, + /// ExecutionProofsByRange rate limiter. + execution_proofs_by_range_rl: Limiter, /// LightClientBootstrap rate limiter. lc_bootstrap_rl: Limiter, /// LightClientOptimisticUpdate rate limiter. @@ -152,6 +154,8 @@ pub struct RPCRateLimiterBuilder { dcbrange_quota: Option, /// Quota for the ExecutionProofsByRoot protocol. execution_proofs_by_root_quota: Option, + /// Quota for the ExecutionProofsByRange protocol. + execution_proofs_by_range_quota: Option, /// Quota for the LightClientBootstrap protocol. lcbootstrap_quota: Option, /// Quota for the LightClientOptimisticUpdate protocol. @@ -178,6 +182,7 @@ impl RPCRateLimiterBuilder { Protocol::DataColumnsByRoot => self.dcbroot_quota = q, Protocol::DataColumnsByRange => self.dcbrange_quota = q, Protocol::ExecutionProofsByRoot => self.execution_proofs_by_root_quota = q, + Protocol::ExecutionProofsByRange => self.execution_proofs_by_range_quota = q, Protocol::LightClientBootstrap => self.lcbootstrap_quota = q, Protocol::LightClientOptimisticUpdate => self.lc_optimistic_update_quota = q, Protocol::LightClientFinalityUpdate => self.lc_finality_update_quota = q, @@ -230,6 +235,10 @@ impl RPCRateLimiterBuilder { .execution_proofs_by_root_quota .ok_or("ExecutionProofsByRoot quota not specified")?; + let execution_proofs_by_range_quota = self + .execution_proofs_by_range_quota + .ok_or("ExecutionProofsByRange quota not specified")?; + // create the rate limiters let ping_rl = Limiter::from_quota(ping_quota)?; let metadata_rl = Limiter::from_quota(metadata_quota)?; @@ -242,6 +251,7 @@ impl RPCRateLimiterBuilder { let dcbroot_rl = Limiter::from_quota(dcbroot_quota)?; let dcbrange_rl = Limiter::from_quota(dcbrange_quota)?; let execution_proofs_by_root_rl = Limiter::from_quota(execution_proofs_by_root_quota)?; + let execution_proofs_by_range_rl = Limiter::from_quota(execution_proofs_by_range_quota)?; let lc_bootstrap_rl = Limiter::from_quota(lc_bootstrap_quota)?; let lc_optimistic_update_rl = Limiter::from_quota(lc_optimistic_update_quota)?; let lc_finality_update_rl = Limiter::from_quota(lc_finality_update_quota)?; @@ -266,6 +276,7 @@ impl RPCRateLimiterBuilder { dcbroot_rl, dcbrange_rl, execution_proofs_by_root_rl, + execution_proofs_by_range_rl, lc_bootstrap_rl, lc_optimistic_update_rl, lc_finality_update_rl, @@ -320,6 +331,7 @@ impl RPCRateLimiter { data_columns_by_root_quota, data_columns_by_range_quota, execution_proofs_by_root_quota, + execution_proofs_by_range_quota, light_client_bootstrap_quota, light_client_optimistic_update_quota, light_client_finality_update_quota, @@ -341,6 +353,10 @@ impl RPCRateLimiter { Protocol::ExecutionProofsByRoot, execution_proofs_by_root_quota, ) + .set_quota( + Protocol::ExecutionProofsByRange, + execution_proofs_by_range_quota, + ) .set_quota(Protocol::LightClientBootstrap, light_client_bootstrap_quota) .set_quota( Protocol::LightClientOptimisticUpdate, @@ -389,6 +405,7 @@ impl RPCRateLimiter { Protocol::DataColumnsByRoot => &mut self.dcbroot_rl, Protocol::DataColumnsByRange => &mut self.dcbrange_rl, Protocol::ExecutionProofsByRoot => &mut self.execution_proofs_by_root_rl, + Protocol::ExecutionProofsByRange => &mut self.execution_proofs_by_range_rl, Protocol::LightClientBootstrap => &mut self.lc_bootstrap_rl, Protocol::LightClientOptimisticUpdate => &mut self.lc_optimistic_update_rl, Protocol::LightClientFinalityUpdate => &mut self.lc_finality_update_rl, @@ -414,6 +431,7 @@ impl RPCRateLimiter { dcbroot_rl, dcbrange_rl, execution_proofs_by_root_rl, + execution_proofs_by_range_rl, lc_bootstrap_rl, lc_optimistic_update_rl, lc_finality_update_rl, @@ -432,6 +450,7 @@ impl RPCRateLimiter { dcbrange_rl.prune(time_since_start); dcbroot_rl.prune(time_since_start); execution_proofs_by_root_rl.prune(time_since_start); + execution_proofs_by_range_rl.prune(time_since_start); lc_bootstrap_rl.prune(time_since_start); lc_optimistic_update_rl.prune(time_since_start); lc_finality_update_rl.prune(time_since_start); diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index d97506653b5..ca3a5b78bd9 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -32,6 +32,8 @@ pub enum SyncRequestId { BlobsByRange(BlobsByRangeRequestId), /// Data columns by range request DataColumnsByRange(DataColumnsByRangeRequestId), + /// Execution proofs by range request + ExecutionProofsByRange(ExecutionProofsByRangeRequestId), } /// Request ID for data_columns_by_root requests. Block lookups do not issue this request directly. @@ -77,6 +79,17 @@ pub enum DataColumnsByRangeRequester { CustodyBackfillSync(CustodyBackFillBatchRequestId), } +/// Request ID for execution_proofs_by_range requests during range sync. +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +pub struct ExecutionProofsByRangeRequestId { + /// Id to identify this attempt at an execution_proofs_by_range request for `parent_request_id` + pub id: Id, + /// The Id of the overall By Range request. + pub parent_request_id: ComponentsByRangeRequestId, + /// The peer id associated with the request. + pub peer: PeerId, +} + /// Block components by range request for range sync. Includes an ID for downstream consumers to /// handle retries and tie all their sub requests together. #[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] @@ -168,6 +181,8 @@ pub enum Response { DataColumnsByRoot(Option>>), /// A response to a get EXECUTION_PROOFS_BY_ROOT request. ExecutionProofsByRoot(Option>), + /// A response to a get EXECUTION_PROOFS_BY_RANGE request. + ExecutionProofsByRange(Option>), /// A response to a LightClientUpdate request. LightClientBootstrap(Arc>), /// A response to a LightClientOptimisticUpdate request. @@ -209,6 +224,10 @@ impl std::convert::From> for RpcResponse { Some(p) => RpcResponse::Success(RpcSuccessResponse::ExecutionProofsByRoot(p)), None => RpcResponse::StreamTermination(ResponseTermination::ExecutionProofsByRoot), }, + Response::ExecutionProofsByRange(r) => match r { + Some(p) => RpcResponse::Success(RpcSuccessResponse::ExecutionProofsByRange(p)), + None => RpcResponse::StreamTermination(ResponseTermination::ExecutionProofsByRange), + }, Response::Status(s) => RpcResponse::Success(RpcSuccessResponse::Status(s)), Response::LightClientBootstrap(b) => { RpcResponse::Success(RpcSuccessResponse::LightClientBootstrap(b)) @@ -245,6 +264,12 @@ macro_rules! impl_display { impl_display!(BlocksByRangeRequestId, "{}/{}", id, parent_request_id); impl_display!(BlobsByRangeRequestId, "{}/{}", id, parent_request_id); impl_display!(DataColumnsByRangeRequestId, "{}/{}", id, parent_request_id); +impl_display!( + ExecutionProofsByRangeRequestId, + "{}/{}", + id, + parent_request_id +); impl_display!(ComponentsByRangeRequestId, "{}/{}", id, requester); impl_display!(DataColumnsByRootRequestId, "{}/{}", id, requester); impl_display!(SingleLookupReqId, "{}/Lookup/{}", req_id, lookup_id); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9f1530ec732..4e8be98a509 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -411,7 +411,7 @@ impl Network { }; let peer_manager = { - let peer_manager_cfg = PeerManagerCfg { + let mut peer_manager_cfg = PeerManagerCfg { discovery_enabled: !config.disable_discovery, quic_enabled: !config.disable_quic_support, metrics_enabled: config.metrics_enabled, @@ -419,6 +419,15 @@ impl Network { execution_proof_enabled: ctx.chain_spec.is_zkvm_enabled(), ..Default::default() }; + // TODO(zkproofs): We decrease the slot time, so we want to + // correspondingly decrease the status interval at which a node will + // check if it needs to sync with others. + let epoch_secs = ctx + .chain_spec + .seconds_per_slot + .saturating_mul(E::slots_per_epoch()) + .max(1); + peer_manager_cfg.status_interval = peer_manager_cfg.status_interval.min(epoch_secs); PeerManager::new(peer_manager_cfg, network_globals.clone())? }; @@ -1580,6 +1589,17 @@ impl Network { request_type, }) } + RequestType::ExecutionProofsByRange(_) => { + metrics::inc_counter_vec( + &metrics::TOTAL_RPC_REQUESTS, + &["execution_proofs_by_range"], + ); + Some(NetworkEvent::RequestReceived { + peer_id, + inbound_request_id, + request_type, + }) + } RequestType::LightClientBootstrap(_) => { metrics::inc_counter_vec( &metrics::TOTAL_RPC_REQUESTS, @@ -1670,6 +1690,11 @@ impl Network { peer_id, Response::ExecutionProofsByRoot(Some(resp)), ), + RpcSuccessResponse::ExecutionProofsByRange(resp) => self.build_response( + id, + peer_id, + Response::ExecutionProofsByRange(Some(resp)), + ), // Should never be reached RpcSuccessResponse::LightClientBootstrap(bootstrap) => { self.build_response(id, peer_id, Response::LightClientBootstrap(bootstrap)) @@ -1702,6 +1727,9 @@ impl Network { ResponseTermination::ExecutionProofsByRoot => { Response::ExecutionProofsByRoot(None) } + ResponseTermination::ExecutionProofsByRange => { + Response::ExecutionProofsByRange(None) + } ResponseTermination::LightClientUpdatesByRange => { Response::LightClientUpdatesByRange(None) } diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs index dd9e9f1ebb2..9ca5afbcf9c 100644 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ b/beacon_node/lighthouse_tracing/src/lib.rs @@ -41,6 +41,8 @@ pub const SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST: &str = "handle_blocks_by_root_requ pub const SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST: &str = "handle_blobs_by_root_request"; pub const SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST: &str = "handle_execution_proofs_by_root_request"; +pub const SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST: &str = + "handle_execution_proofs_by_range_request"; pub const SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST: &str = "handle_data_columns_by_root_request"; pub const SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE: &str = "handle_light_client_updates_by_range"; pub const SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP: &str = "handle_light_client_bootstrap"; @@ -73,6 +75,7 @@ pub const LH_BN_ROOT_SPAN_NAMES: &[&str] = &[ SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, + SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index c8440a6bbf4..5d2203ee380 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -820,13 +820,29 @@ impl NetworkBeaconProcessor { debug!( %block_root, %proof_id, - "Gossip execution proof already processed via the EL. Accepting the proof without re-processing." + "Gossip execution proof already processed via the EL. Checking availability." ); self.propagate_validation_result( message_id, peer_id, MessageAcceptance::Accept, ); + + // The proof is already in the DA checker (from HTTP API). + // Check if this makes any pending blocks complete and import them. + let slot = execution_proof.slot; + if let Err(e) = self + .chain + .process_rpc_execution_proofs(slot, block_root, vec![execution_proof]) + .await + { + debug!( + %block_root, + %proof_id, + error = ?e, + "Failed to process availability for prior known execution proof" + ); + } } GossipExecutionProofError::PriorKnown { block_root, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7db2790920e..ffac53e522a 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -14,7 +14,7 @@ use beacon_processor::{ use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - ExecutionProofsByRootRequest, LightClientUpdatesByRangeRequest, + ExecutionProofsByRangeRequest, ExecutionProofsByRootRequest, LightClientUpdatesByRangeRequest, }; use lighthouse_network::service::api_types::CustodyBackfillBatchId; use lighthouse_network::{ @@ -699,6 +699,24 @@ impl NetworkBeaconProcessor { }) } + /// Create a new work event to process `ExecutionProofsByRangeRequest`s from the RPC network. + pub fn send_execution_proofs_by_range_request( + self: &Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: ExecutionProofsByRangeRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.handle_execution_proofs_by_range_request(peer_id, inbound_request_id, request) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::ExecutionProofsByRangeRequest(Box::new(process_fn)), + }) + } + /// Create a new work event to process `DataColumnsByRootRequest`s from the RPC network. pub fn send_data_columns_by_roots_request( self: &Arc, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index f063d7e8380..17ee4076731 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -7,7 +7,7 @@ use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessStatus, WhenS use itertools::{Itertools, process_results}; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - ExecutionProofsByRootRequest, + ExecutionProofsByRangeRequest, ExecutionProofsByRootRequest, }; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, ReportSource, Response, SyncInfo}; @@ -15,9 +15,9 @@ use lighthouse_tracing::{ SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, - SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, - SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, - SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, + SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, + SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, + SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, }; use methods::LightClientUpdatesByRangeRequest; use slot_clock::SlotClock; @@ -436,19 +436,39 @@ impl NetworkBeaconProcessor { request.already_have.iter().copied().collect(); let count_needed = request.count_needed as usize; - // Get all execution proofs we have for this block from the DA checker - let Some(available_proofs) = self + // Get all execution proofs we have for this block from the DA checker, falling back to the + // store (which checks the store cache/DB). + let available_proofs = match self .chain .data_availability_checker .get_execution_proofs(&block_root) - else { - // No proofs available for this block - debug!( - %peer_id, - %block_root, - "No execution proofs available for peer" - ); - return Ok(()); + { + Some(proofs) => proofs, + None => match self.chain.store.get_execution_proofs(&block_root) { + Ok(proofs) => { + if proofs.is_empty() { + debug!( + %peer_id, + %block_root, + "No execution proofs available for peer" + ); + return Ok(()); + } + proofs + } + Err(e) => { + error!( + %peer_id, + %block_root, + error = ?e, + "Error fetching execution proofs for block root" + ); + return Err(( + RpcErrorResponse::ServerError, + "Error fetching execution proofs", + )); + } + }, }; // Filter out proofs the peer already has and send up to count_needed @@ -486,6 +506,137 @@ impl NetworkBeaconProcessor { Ok(()) } + /// Handle an `ExecutionProofsByRange` request from the peer. + #[instrument( + name = SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, + parent = None, + level = "debug", + skip_all, + fields(peer_id = %peer_id, client = tracing::field::Empty) + )] + pub fn handle_execution_proofs_by_range_request( + &self, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + req: ExecutionProofsByRangeRequest, + ) { + let client = self.network_globals.client(&peer_id); + Span::current().record("client", field::display(client.kind)); + + self.terminate_response_stream( + peer_id, + inbound_request_id, + self.handle_execution_proofs_by_range_request_inner(peer_id, inbound_request_id, req), + Response::ExecutionProofsByRange, + ); + } + + /// Handle an `ExecutionProofsByRange` request from the peer. + fn handle_execution_proofs_by_range_request_inner( + &self, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + req: ExecutionProofsByRangeRequest, + ) -> Result<(), (RpcErrorResponse, &'static str)> { + debug!( + %peer_id, + count = req.count, + start_slot = req.start_slot, + "Received ExecutionProofsByRange Request" + ); + + let request_start_slot = Slot::from(req.start_slot); + + // Check if zkvm is enabled and get the execution proof boundary + let execution_proof_boundary_slot = match self.chain.execution_proof_boundary() { + Some(boundary) => boundary.start_slot(T::EthSpec::slots_per_epoch()), + None => { + debug!("ZKVM fork is disabled"); + return Err((RpcErrorResponse::InvalidRequest, "ZKVM fork is disabled")); + } + }; + + // Get the oldest execution proof slot from the store + let oldest_execution_proof_slot = self + .chain + .store + .get_execution_proof_info() + .oldest_execution_proof_slot + .unwrap_or(execution_proof_boundary_slot); + + if request_start_slot < oldest_execution_proof_slot { + debug!( + %request_start_slot, + %oldest_execution_proof_slot, + %execution_proof_boundary_slot, + "Range request start slot is older than the oldest execution proof slot." + ); + + return if execution_proof_boundary_slot < oldest_execution_proof_slot { + Err(( + RpcErrorResponse::ResourceUnavailable, + "execution proofs pruned within boundary", + )) + } else { + Err(( + RpcErrorResponse::InvalidRequest, + "Req outside availability period", + )) + }; + } + + let block_roots = self.get_block_roots_for_slot_range( + req.start_slot, + req.count, + "ExecutionProofsByRange", + )?; + let mut proofs_sent = 0; + + for root in block_roots { + // Get execution proofs from the database (like BlobsByRange does for blobs) + match self.chain.store.get_execution_proofs(&root) { + Ok(proofs) => { + for proof in proofs { + // Due to skip slots, proofs could be out of the range + if proof.slot >= request_start_slot + && proof.slot < request_start_slot + req.count + { + proofs_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + inbound_request_id, + response: Response::ExecutionProofsByRange(Some(proof)), + }); + } + } + } + Err(e) => { + error!( + request = ?req, + %peer_id, + block_root = ?root, + error = ?e, + "Error fetching execution proofs for block root" + ); + return Err(( + RpcErrorResponse::ServerError, + "Failed fetching execution proofs from database", + )); + } + } + } + + debug!( + %peer_id, + start_slot = req.start_slot, + count = req.count, + sent = proofs_sent, + "ExecutionProofsByRange outgoing response processed" + ); + + Ok(()) + } + /// Handle a `DataColumnsByRoot` request from the peer. #[instrument( name = SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index eb02ddad921..f5bf65c9777 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -283,6 +283,15 @@ impl Router { request, ), ), + RequestType::ExecutionProofsByRange(request) => self + .handle_beacon_processor_send_result( + self.network_beacon_processor + .send_execution_proofs_by_range_request( + peer_id, + inbound_request_id, + request, + ), + ), _ => {} } } @@ -323,6 +332,13 @@ impl Router { Response::ExecutionProofsByRoot(execution_proof) => { self.on_execution_proofs_by_root_response(peer_id, app_request_id, execution_proof); } + Response::ExecutionProofsByRange(execution_proof) => { + self.on_execution_proofs_by_range_response( + peer_id, + app_request_id, + execution_proof, + ); + } // Light client responses should not be received Response::LightClientBootstrap(_) | Response::LightClientOptimisticUpdate(_) @@ -727,6 +743,30 @@ impl Router { }); } + /// Handle an `ExecutionProofsByRange` response from the peer. + pub fn on_execution_proofs_by_range_response( + &mut self, + peer_id: PeerId, + app_request_id: AppRequestId, + execution_proof: Option>, + ) { + trace!( + %peer_id, + "Received ExecutionProofsByRange Response" + ); + + if let AppRequestId::Sync(sync_request_id) = app_request_id { + self.send_to_sync(SyncMessage::RpcExecutionProof { + peer_id, + sync_request_id, + execution_proof, + seen_timestamp: timestamp_now(), + }); + } else { + crit!("All execution proofs by range responses should belong to sync"); + } + } + /// Handle a `DataColumnsByRoot` response from the peer. pub fn on_data_columns_by_root_response( &mut self, diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 6c0cbd7e554..441e9b0a6d9 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -348,6 +348,23 @@ impl BackFillSync { CouplingError::BlobPeerFailure(msg) => { tracing::debug!(?batch_id, msg, "Blob peer failure"); } + CouplingError::ExecutionProofPeerFailure { + error, + peer, + exceeded_retries, + } => { + tracing::debug!(?batch_id, ?peer, error, "Execution proof peer failure"); + if !*exceeded_retries { + let mut failed_peers = HashSet::new(); + failed_peers.insert(*peer); + return self.retry_execution_proof_batch( + network, + batch_id, + request_id, + failed_peers, + ); + } + } CouplingError::InternalError(msg) => { error!(?batch_id, msg, "Block components coupling internal error"); } @@ -1001,6 +1018,46 @@ impl BackFillSync { Ok(()) } + /// Retries execution proof requests within the batch by creating a new proofs request. + pub fn retry_execution_proof_batch( + &mut self, + network: &mut SyncNetworkContext, + batch_id: BatchId, + id: Id, + mut failed_peers: HashSet, + ) -> Result<(), BackFillError> { + if let Some(batch) = self.batches.get_mut(&batch_id) { + failed_peers.extend(&batch.failed_peers()); + let req = batch.to_blocks_by_range_request().0; + + let synced_peers = network + .network_globals() + .peers + .read() + .synced_peers_for_epoch(batch_id) + .cloned() + .collect::>(); + + match network.retry_execution_proofs_by_range(id, &synced_peers, &failed_peers, req) { + Ok(()) => { + debug!( + ?batch_id, + id, "Retried execution proof requests from different peers" + ); + return Ok(()); + } + Err(e) => { + debug!(?batch_id, id, e, "Failed to retry execution proof batch"); + } + } + } else { + return Err(BackFillError::InvalidSyncState( + "Batch should exist to be retried".to_string(), + )); + } + Ok(()) + } + /// When resuming a chain, this function searches for batches that need to be re-downloaded and /// transitions their state to redownload the batch. fn resume_batches(&mut self, network: &mut SyncNetworkContext) -> Result<(), BackFillError> { diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index ed9a11a03de..faa2fac949c 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -5,6 +5,7 @@ use lighthouse_network::{ PeerId, service::api_types::{ BlobsByRangeRequestId, BlocksByRangeRequestId, DataColumnsByRangeRequestId, + ExecutionProofsByRangeRequestId, }, }; use ssz_types::RuntimeVariableList; @@ -12,7 +13,7 @@ use std::{collections::HashMap, sync::Arc}; use tracing::{Span, debug}; use types::{ BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, - Hash256, SignedBeaconBlock, + ExecutionProof, Hash256, SignedBeaconBlock, }; use crate::sync::network_context::MAX_COLUMN_RETRIES; @@ -24,6 +25,7 @@ use crate::sync::network_context::MAX_COLUMN_RETRIES; /// - Blocks themselves (always required) /// - Blob sidecars (pre-Fulu fork) /// - Data columns (Fulu fork and later) +/// - Execution proofs (for zkvm-enabled nodes) /// /// It accumulates responses until all expected components are received, then couples /// them together and returns complete `RpcBlock`s ready for processing. Handles validation @@ -33,10 +35,25 @@ pub struct RangeBlockComponentsRequest { blocks_request: ByRangeRequest>>>, /// Sidecars we have received awaiting for their corresponding block. block_data_request: RangeBlockDataRequest, + /// Execution proofs request (for zkvm-enabled nodes). + execution_proofs_request: Option>, /// Span to track the range request and all children range requests. pub(crate) request_span: Span, } +/// Tracks execution proofs requests during range sync. +struct ExecutionProofsRequest { + /// The request tracking state. + request: ByRangeRequest>>, + /// The peer we requested proofs from. + peer: PeerId, + /// Number of proofs required per block. + min_proofs_required: usize, + /// Number of proof attempts completed for this batch. + attempt: usize, + _phantom: std::marker::PhantomData, +} + pub enum ByRangeRequest { Active(I), Complete(T), @@ -67,6 +84,12 @@ pub(crate) enum CouplingError { exceeded_retries: bool, }, BlobPeerFailure(String), + /// The peer we requested execution proofs from was faulty/malicious + ExecutionProofPeerFailure { + error: String, + peer: PeerId, + exceeded_retries: bool, + }, } impl RangeBlockComponentsRequest { @@ -76,6 +99,7 @@ impl RangeBlockComponentsRequest { /// * `blocks_req_id` - Request ID for the blocks /// * `blobs_req_id` - Optional request ID for blobs (pre-Fulu fork) /// * `data_columns` - Optional tuple of (request_id->column_indices pairs, expected_custody_columns) for Fulu fork + /// * `execution_proofs` - Optional tuple of (request_id, peer, min_proofs_required) for zkvm-enabled nodes #[allow(clippy::type_complexity)] pub fn new( blocks_req_id: BlocksByRangeRequestId, @@ -84,6 +108,7 @@ impl RangeBlockComponentsRequest { Vec<(DataColumnsByRangeRequestId, Vec)>, Vec, )>, + execution_proofs: Option<(ExecutionProofsByRangeRequestId, usize)>, request_span: Span, ) -> Self { let block_data_request = if let Some(blobs_req_id) = blobs_req_id { @@ -103,9 +128,19 @@ impl RangeBlockComponentsRequest { RangeBlockDataRequest::NoData }; + let execution_proofs_request = + execution_proofs.map(|(req_id, min_proofs_required)| ExecutionProofsRequest { + request: ByRangeRequest::Active(req_id), + peer: req_id.peer, + min_proofs_required, + attempt: 0, + _phantom: std::marker::PhantomData, + }); + Self { blocks_request: ByRangeRequest::Active(blocks_req_id), block_data_request, + execution_proofs_request, request_span, } } @@ -187,6 +222,30 @@ impl RangeBlockComponentsRequest { } } + /// Adds received execution proofs to the request. + /// + /// Returns an error if this request doesn't expect execution proofs, + /// or if the request ID doesn't match. + pub fn add_execution_proofs( + &mut self, + req_id: ExecutionProofsByRangeRequestId, + proofs: Vec>, + ) -> Result<(), String> { + match &mut self.execution_proofs_request { + Some(exec_proofs_req) => { + exec_proofs_req.request.finish(req_id, proofs)?; + exec_proofs_req.attempt += 1; + Ok(()) + } + None => Err("received execution proofs but none were expected".to_owned()), + } + } + + /// Returns true if this request expects execution proofs. + pub fn expects_execution_proofs(&self) -> bool { + self.execution_proofs_request.is_some() + } + /// Attempts to construct RPC blocks from all received components. /// /// Returns `None` if not all expected requests have completed. @@ -200,6 +259,13 @@ impl RangeBlockComponentsRequest { return None; }; + // Check if execution proofs are required but not yet complete + if let Some(exec_proofs_req) = &self.execution_proofs_request + && exec_proofs_req.request.to_finished().is_none() + { + return None; + } + // Increment the attempt once this function returns the response or errors match &mut self.block_data_request { RangeBlockDataRequest::NoData => { @@ -269,6 +335,50 @@ impl RangeBlockComponentsRequest { } } + /// Returns the collected execution proofs if available. + /// This should be called after `responses()` returns `Some`. + pub fn get_execution_proofs(&self) -> Option>> { + self.execution_proofs_request + .as_ref() + .and_then(|req| req.request.to_finished().cloned()) + } + + /// Returns the peer that was responsible for providing execution proofs. + pub fn execution_proofs_peer(&self) -> Option { + self.execution_proofs_request.as_ref().map(|req| req.peer) + } + + /// Returns the minimum number of execution proofs required per block, if any. + pub fn min_execution_proofs_required(&self) -> Option { + self.execution_proofs_request + .as_ref() + .map(|req| req.min_proofs_required) + } + + /// Returns the number of completed proof attempts for this batch, if any. + pub fn execution_proofs_attempt(&self) -> Option { + self.execution_proofs_request + .as_ref() + .map(|req| req.attempt) + } + + /// Resets the execution proofs request to retry with a new peer. + pub fn reinsert_execution_proofs_request( + &mut self, + req_id: ExecutionProofsByRangeRequestId, + min_proofs_required: usize, + ) -> Result<(), String> { + match &mut self.execution_proofs_request { + Some(exec_proofs_req) => { + exec_proofs_req.request = ByRangeRequest::Active(req_id); + exec_proofs_req.peer = req_id.peer; + exec_proofs_req.min_proofs_required = min_proofs_required; + Ok(()) + } + None => Err("execution proofs request not present".to_owned()), + } + } + fn responses_with_blobs( blocks: Vec>>, blobs: Vec>>, @@ -529,7 +639,7 @@ mod tests { let blocks_req_id = blocks_id(components_id()); let mut info = - RangeBlockComponentsRequest::::new(blocks_req_id, None, None, Span::none()); + RangeBlockComponentsRequest::::new(blocks_req_id, None, None, None, Span::none()); // Send blocks and complete terminate response info.add_blocks(blocks_req_id, blocks).unwrap(); @@ -557,6 +667,7 @@ mod tests { blocks_req_id, Some(blobs_req_id), None, + None, Span::none(), ); @@ -606,6 +717,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expects_custody_columns.clone())), + None, Span::none(), ); // Send blocks and complete terminate response @@ -674,6 +786,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expects_custody_columns.clone())), + None, Span::none(), ); @@ -762,6 +875,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); @@ -848,6 +962,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); @@ -941,6 +1056,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index c0af69d7a40..6c41d3d9c75 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -61,7 +61,8 @@ use lighthouse_network::service::api_types::{ BlobsByRangeRequestId, BlocksByRangeRequestId, ComponentsByRangeRequestId, CustodyBackFillBatchRequestId, CustodyBackfillBatchId, CustodyRequester, DataColumnsByRangeRequestId, DataColumnsByRangeRequester, DataColumnsByRootRequestId, - DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, + DataColumnsByRootRequester, ExecutionProofsByRangeRequestId, Id, SingleLookupReqId, + SyncRequestId, }; use lighthouse_network::types::{NetworkGlobals, SyncState}; use lighthouse_network::{PeerAction, PeerId}; @@ -518,6 +519,8 @@ impl SyncManager { SyncRequestId::DataColumnsByRange(req_id) => { self.on_data_columns_by_range_response(req_id, peer_id, RpcEvent::RPCError(error)) } + SyncRequestId::ExecutionProofsByRange(req_id) => self + .on_execution_proofs_by_range_response(req_id, peer_id, RpcEvent::RPCError(error)), } } @@ -1225,6 +1228,12 @@ impl SyncManager { peer_id, RpcEvent::from_chunk(execution_proof, seen_timestamp), ), + SyncRequestId::ExecutionProofsByRange(req_id) => self + .on_execution_proofs_by_range_response( + req_id, + peer_id, + RpcEvent::from_chunk(execution_proof, seen_timestamp), + ), _ => { crit!(%peer_id, "bad request id for execution_proof"); } @@ -1352,6 +1361,28 @@ impl SyncManager { } } + /// Handles a response for an execution proofs by range request. + /// + /// Note: This is currently a stub. Execution proofs by range requests are not yet issued + /// during range sync. + fn on_execution_proofs_by_range_response( + &mut self, + id: ExecutionProofsByRangeRequestId, + peer_id: PeerId, + proof: RpcEvent>, + ) { + if let Some(resp) = self + .network + .on_execution_proofs_by_range_response(id, peer_id, proof) + { + self.on_range_components_response( + id.parent_request_id, + peer_id, + RangeBlockComponent::ExecutionProofs(id, resp), + ); + } + } + fn on_custody_by_root_result( &mut self, requester: CustodyRequester, diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 0943787c925..65ae25bfd3c 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -17,18 +17,21 @@ use crate::sync::block_lookups::SingleLookupId; use crate::sync::block_sidecar_coupling::CouplingError; use crate::sync::network_context::requests::BlobsByRootSingleBlockRequest; use crate::sync::range_data_column_batch_request::RangeDataColumnBatchRequest; -use beacon_chain::block_verification_types::RpcBlock; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessStatus, EngineState}; use custody::CustodyRequestResult; use fnv::FnvHashMap; -use lighthouse_network::rpc::methods::{BlobsByRangeRequest, DataColumnsByRangeRequest}; +use lighthouse_network::rpc::methods::{ + BlobsByRangeRequest, DataColumnsByRangeRequest, ExecutionProofsByRangeRequest, +}; use lighthouse_network::rpc::{BlocksByRangeRequest, GoodbyeReason, RPCError, RequestType}; pub use lighthouse_network::service::api_types::RangeRequestId; use lighthouse_network::service::api_types::{ AppRequestId, BlobsByRangeRequestId, BlocksByRangeRequestId, ComponentsByRangeRequestId, CustodyBackFillBatchRequestId, CustodyBackfillBatchId, CustodyId, CustodyRequester, DataColumnsByRangeRequestId, DataColumnsByRangeRequester, DataColumnsByRootRequestId, - DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, + DataColumnsByRootRequester, ExecutionProofsByRangeRequestId, Id, SingleLookupReqId, + SyncRequestId, }; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Subnet}; use lighthouse_tracing::{SPAN_OUTGOING_BLOCK_BY_ROOT_REQUEST, SPAN_OUTGOING_RANGE_REQUEST}; @@ -37,7 +40,8 @@ pub use requests::LookupVerifyError; use requests::{ ActiveRequests, BlobsByRangeRequestItems, BlobsByRootRequestItems, BlocksByRangeRequestItems, BlocksByRootRequestItems, DataColumnsByRangeRequestItems, DataColumnsByRootRequestItems, - ExecutionProofsByRootRequestItems, ExecutionProofsByRootSingleBlockRequest, + ExecutionProofsByRangeRequestItems, ExecutionProofsByRootRequestItems, + ExecutionProofsByRootSingleBlockRequest, }; #[cfg(test)] use slot_clock::SlotClock; @@ -73,6 +77,8 @@ macro_rules! new_range_request_span { /// Max retries for block components after which we fail the batch. pub const MAX_COLUMN_RETRIES: usize = 3; +/// Max retries for execution proofs after which we fail the batch. +pub const MAX_EXECUTION_PROOF_RETRIES: usize = 3; #[derive(Debug)] pub enum RpcEvent { @@ -118,6 +124,7 @@ pub enum RpcRequestSendError { pub enum NoPeerError { BlockPeer, CustodyPeer(ColumnIndex), + ExecutionProofPeer, } #[derive(Debug, PartialEq, Eq)] @@ -217,6 +224,9 @@ pub struct SyncNetworkContext { /// A mapping of active DataColumnsByRange requests data_columns_by_range_requests: ActiveRequests>, + /// A mapping of active ExecutionProofsByRange requests + execution_proofs_by_range_requests: + ActiveRequests, /// Mapping of active custody column requests for a block root custody_by_root_requests: FnvHashMap>, @@ -254,6 +264,17 @@ pub enum RangeBlockComponent { DataColumnsByRangeRequestId, RpcResponseResult>>>, ), + ExecutionProofs( + ExecutionProofsByRangeRequestId, + RpcResponseResult>>, + ), +} + +struct RangeExecutionProofInputs { + min_proofs_required: usize, + proofs_peer: PeerId, + proofs: Vec>, + attempt: usize, } #[cfg(test)] @@ -303,6 +324,7 @@ impl SyncNetworkContext { blocks_by_range_requests: ActiveRequests::new("blocks_by_range"), blobs_by_range_requests: ActiveRequests::new("blobs_by_range"), data_columns_by_range_requests: ActiveRequests::new("data_columns_by_range"), + execution_proofs_by_range_requests: ActiveRequests::new("execution_proofs_by_range"), custody_by_root_requests: <_>::default(), components_by_range_requests: FnvHashMap::default(), custody_backfill_data_column_batch_requests: FnvHashMap::default(), @@ -332,6 +354,7 @@ impl SyncNetworkContext { blocks_by_range_requests, blobs_by_range_requests, data_columns_by_range_requests, + execution_proofs_by_range_requests, // custody_by_root_requests is a meta request of data_columns_by_root_requests custody_by_root_requests: _, // components_by_range_requests is a meta request of various _by_range requests @@ -371,6 +394,10 @@ impl SyncNetworkContext { .active_requests_of_peer(peer_id) .into_iter() .map(|req_id| SyncRequestId::DataColumnsByRange(*req_id)); + let execution_proofs_by_range_ids = execution_proofs_by_range_requests + .active_requests_of_peer(peer_id) + .into_iter() + .map(|req_id| SyncRequestId::ExecutionProofsByRange(*req_id)); blocks_by_root_ids .chain(blobs_by_root_ids) .chain(data_column_by_root_ids) @@ -378,6 +405,7 @@ impl SyncNetworkContext { .chain(blocks_by_range_ids) .chain(blobs_by_range_ids) .chain(data_column_by_range_ids) + .chain(execution_proofs_by_range_ids) .collect() } @@ -435,6 +463,7 @@ impl SyncNetworkContext { blocks_by_range_requests, blobs_by_range_requests, data_columns_by_range_requests, + execution_proofs_by_range_requests, // custody_by_root_requests is a meta request of data_columns_by_root_requests custody_by_root_requests: _, // components_by_range_requests is a meta request of various _by_range requests @@ -458,6 +487,7 @@ impl SyncNetworkContext { .chain(blocks_by_range_requests.iter_request_peers()) .chain(blobs_by_range_requests.iter_request_peers()) .chain(data_columns_by_range_requests.iter_request_peers()) + .chain(execution_proofs_by_range_requests.iter_request_peers()) { *active_request_count_by_peer.entry(peer_id).or_default() += 1; } @@ -547,6 +577,83 @@ impl SyncNetworkContext { Ok(()) } + /// Retries execution proofs by range by requesting the proofs again from a different peer. + pub fn retry_execution_proofs_by_range( + &mut self, + id: Id, + peers: &HashSet, + peers_to_deprioritize: &HashSet, + request: BlocksByRangeRequest, + ) -> Result<(), String> { + let Some((requester, parent_request_span)) = self + .components_by_range_requests + .iter() + .find_map(|(key, value)| { + if key.id == id { + Some((key.requester, value.request_span.clone())) + } else { + None + } + }) + else { + return Err("request id not present".to_string()); + }; + + let active_request_count_by_peer = self.active_request_count_by_peer(); + + let proof_peer = self + .select_execution_proofs_peer( + peers, + &active_request_count_by_peer, + peers_to_deprioritize, + ) + .ok_or_else(|| "no zkvm-enabled peer available for execution proofs".to_string())?; + + let epoch = Slot::new(*request.start_slot()).epoch(T::EthSpec::slots_per_epoch()); + let min_proofs_required = self.chain.spec.zkvm_min_proofs_required().ok_or_else(|| { + "zkvm enabled but min proofs requirement is not configured".to_string() + })?; + if !self.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + return Err("execution proofs retry requested for pre-zkvm epoch".to_string()); + } + + debug!( + id, + ?requester, + ?proof_peer, + "Retrying execution proofs by range from a different peer" + ); + + let id = ComponentsByRangeRequestId { id, requester }; + let req_id = self + .send_execution_proofs_by_range_request( + proof_peer, + ExecutionProofsByRangeRequest { + start_slot: *request.start_slot(), + count: *request.count(), + }, + id, + new_range_request_span!( + self, + "outgoing_proofs_by_range_retry", + parent_request_span.clone(), + proof_peer + ), + ) + .map_err(|e| format!("{:?}", e))?; + + let Some(range_request) = self.components_by_range_requests.get_mut(&id) else { + return Err( + "retrying execution proofs for range request that does not exist".to_string(), + ); + }; + + range_request + .reinsert_execution_proofs_request(req_id, min_proofs_required) + .map_err(|e| format!("{e:?}"))?; + Ok(()) + } + /// A blocks by range request sent by the range sync algorithm pub fn block_components_by_range_request( &mut self, @@ -672,6 +779,43 @@ impl SyncNetworkContext { .transpose()?; let epoch = Slot::new(*request.start_slot()).epoch(T::EthSpec::slots_per_epoch()); + + // Request execution proofs only if zkvm is enabled for this epoch. + let execution_proofs_request = if self.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + let min_proofs_required = + self.chain.spec.zkvm_min_proofs_required().ok_or_else(|| { + RpcRequestSendError::InternalError( + "zkvm enabled but min proofs requirement is not configured".to_string(), + ) + })?; + + // Find a zkvm-enabled peer from block_peers or column_peers + let zkvm_peer = self.find_zkvm_enabled_peer(block_peers, column_peers); + + if let Some(proofs_peer) = zkvm_peer { + let proofs_request = ExecutionProofsByRangeRequest { + start_slot: *request.start_slot(), + count: *request.count(), + }; + let req_id = self.send_execution_proofs_by_range_request( + proofs_peer, + proofs_request, + id, + new_range_request_span!( + self, + "outgoing_proofs_by_range", + range_request_span.clone(), + proofs_peer + ), + )?; + Some((req_id, min_proofs_required)) + } else { + return Err(RpcRequestSendError::NoPeer(NoPeerError::ExecutionProofPeer)); + } + } else { + None + }; + let info = RangeBlockComponentsRequest::new( blocks_req_id, blobs_req_id, @@ -681,6 +825,7 @@ impl SyncNetworkContext { self.chain.sampling_columns_for_epoch(epoch).to_vec(), ) }), + execution_proofs_request, range_request_span, ); self.components_by_range_requests.insert(id, info); @@ -743,6 +888,33 @@ impl SyncNetworkContext { Ok(columns_to_request_by_peer) } + fn select_execution_proofs_peer( + &self, + peers: &HashSet, + active_request_count_by_peer: &HashMap, + peers_to_deprioritize: &HashSet, + ) -> Option { + let peers_db = self.network_globals().peers.read(); + peers + .iter() + .filter(|peer| { + peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + }) + .map(|peer| { + ( + peers_to_deprioritize.contains(peer), + active_request_count_by_peer.get(peer).copied().unwrap_or(0), + rand::random::(), + peer, + ) + }) + .min() + .map(|(_, _, _, peer)| *peer) + } + /// Received a blocks by range or blobs by range response for a request that couples blocks ' /// and blobs. pub fn range_block_component_response( @@ -750,13 +922,14 @@ impl SyncNetworkContext { id: ComponentsByRangeRequestId, range_block_component: RangeBlockComponent, ) -> Option>, RpcResponseError>> { - let Entry::Occupied(mut entry) = self.components_by_range_requests.entry(id) else { - metrics::inc_counter_vec(&metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, &["range_blocks"]); - return None; - }; - - if let Err(e) = { - let request = entry.get_mut(); + let add_result = { + let Some(request) = self.components_by_range_requests.get_mut(&id) else { + metrics::inc_counter_vec( + &metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, + &["range_blocks"], + ); + return None; + }; match range_block_component { RangeBlockComponent::Block(req_id, resp) => resp.and_then(|(blocks, _)| { request.add_blocks(req_id, blocks).map_err(|e| { @@ -783,14 +956,104 @@ impl SyncNetworkContext { }) }) } + RangeBlockComponent::ExecutionProofs(req_id, resp) => { + let expects_execution_proofs = request.expects_execution_proofs(); + // Handle execution proofs response, treating UnsupportedProtocol as an error + // if proofs are required. + let proofs = match resp { + Ok((proofs, _)) => proofs, + Err(RpcResponseError::RpcError(RPCError::UnsupportedProtocol)) + if expects_execution_proofs => + { + return Some(Err(RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + error: "Peer doesn't support execution_proofs_by_range" + .to_string(), + peer: req_id.peer, + exceeded_retries: false, + }, + ))); + } + Err(RpcResponseError::RpcError(RPCError::UnsupportedProtocol)) => { + debug!( + req_id = ?req_id, + "Peer doesn't support execution_proofs_by_range, treating as empty response" + ); + vec![] + } + Err(e) => return Some(Err(e)), + }; + request.add_execution_proofs(req_id, proofs).map_err(|e| { + RpcResponseError::BlockComponentCouplingError(CouplingError::InternalError( + e, + )) + }) + } } - } { - entry.remove(); + }; + + if let Err(e) = add_result { + self.components_by_range_requests.remove(&id); return Some(Err(e)); } - let range_req = entry.get_mut(); - if let Some(blocks_result) = range_req.responses(&self.chain.spec) { + let (blocks_result, min_proofs_required, proofs_peer, proofs, proofs_attempt) = { + let Some(range_req) = self.components_by_range_requests.get_mut(&id) else { + metrics::inc_counter_vec( + &metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, + &["range_blocks"], + ); + return None; + }; + let blocks_result = range_req.responses(&self.chain.spec); + let min_proofs_required = range_req.min_execution_proofs_required(); + let proofs_peer = range_req.execution_proofs_peer(); + let proofs = range_req.get_execution_proofs().unwrap_or_default(); + let proofs_attempt = range_req.execution_proofs_attempt().unwrap_or(0); + ( + blocks_result, + min_proofs_required, + proofs_peer, + proofs, + proofs_attempt, + ) + }; + + if let Some(Ok(blocks)) = &blocks_result + && let Some(min_proofs_required) = min_proofs_required + { + let Some(proofs_peer) = proofs_peer else { + self.components_by_range_requests.remove(&id); + return Some(Err(RpcResponseError::BlockComponentCouplingError( + CouplingError::InternalError( + "execution proofs request completed without a peer".to_string(), + ), + ))); + }; + let proof_inputs = RangeExecutionProofInputs { + min_proofs_required, + proofs_peer, + proofs, + attempt: proofs_attempt, + }; + if let Err(err) = self.process_range_execution_proofs(proof_inputs, blocks) { + let remove_entry = !matches!( + err, + RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + exceeded_retries: false, + .. + } + ) + ); + if remove_entry { + self.components_by_range_requests.remove(&id); + } + return Some(Err(err)); + } + } + + if let Some(blocks_result) = blocks_result { if let Err(CouplingError::DataColumnPeerFailure { error, faulty_peers: _, @@ -800,16 +1063,16 @@ impl SyncNetworkContext { // Remove the entry if it's a peer failure **and** retry counter is exceeded if *exceeded_retries { debug!( - entry=?entry.key(), + entry = ?id, msg = error, "Request exceeded max retries, failing batch" ); - entry.remove(); - }; + self.components_by_range_requests.remove(&id); + } } else { - // also remove the entry only if it coupled successfully + // Also remove the entry only if it coupled successfully // or if it isn't a column peer failure. - entry.remove(); + self.components_by_range_requests.remove(&id); } // If the request is finished, dequeue everything Some(blocks_result.map_err(RpcResponseError::BlockComponentCouplingError)) @@ -818,6 +1081,138 @@ impl SyncNetworkContext { } } + fn process_range_execution_proofs( + &self, + inputs: RangeExecutionProofInputs, + blocks: &[RpcBlock], + ) -> Result<(), RpcResponseError> { + let RangeExecutionProofInputs { + min_proofs_required, + proofs_peer, + proofs, + attempt, + } = inputs; + let exceeded_retries = attempt >= MAX_EXECUTION_PROOF_RETRIES; + let mut proofs_by_root: HashMap>> = HashMap::new(); + for proof in proofs { + proofs_by_root + .entry(proof.block_root) + .or_default() + .push(proof); + } + + let proof_error = |error: String| { + RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + error, + peer: proofs_peer, + exceeded_retries, + }, + ) + }; + + for block in blocks { + let block_root = block.block_root(); + if !self.chain.spec.is_zkvm_enabled_for_epoch(block.epoch()) { + proofs_by_root.remove(&block_root); + continue; + } + let existing_count = self + .chain + .data_availability_checker + .get_existing_proof_ids(&block_root) + .map(|ids| ids.len()) + .unwrap_or(0); + + let proofs_for_block = proofs_by_root.remove(&block_root).unwrap_or_default(); + if existing_count >= min_proofs_required { + if !proofs_for_block.is_empty() { + debug!( + ?block_root, + existing_count, + min_proofs_required, + "Ignoring execution proofs because cache already satisfies requirement" + ); + } + continue; + } + + let payload_hash = block + .message() + .body() + .execution_payload() + .ok() + .map(|payload| payload.execution_payload_ref().block_hash()) + .ok_or_else(|| { + RpcResponseError::BlockComponentCouplingError(CouplingError::InternalError( + "execution payload missing for zkvm proofs".to_string(), + )) + })?; + + let mut verified_proofs = Vec::new(); + for proof in proofs_for_block { + if proof.block_root != block_root { + return Err(proof_error(format!( + "proof block_root mismatch: expected {block_root:?} got {:?}", + proof.block_root + ))); + } + if proof.block_hash != payload_hash { + return Err(proof_error(format!( + "proof execution payload hash mismatch for {block_root:?}" + ))); + } + match self + .chain + .data_availability_checker + .verify_execution_proof_for_gossip(&proof) + { + Ok(true) => verified_proofs.push((*proof).clone()), + Ok(false) => { + return Err(proof_error(format!( + "execution proof verification failed for {block_root:?}" + ))); + } + Err(e) => { + return Err(proof_error(format!( + "execution proof verification error for {block_root:?}: {e:?}" + ))); + } + } + } + + if !verified_proofs.is_empty() + && let Err(e) = self + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, verified_proofs) + { + return Err(proof_error(format!( + "failed to store execution proofs for {block_root:?}: {e:?}" + ))); + } + + let updated_count = self + .chain + .data_availability_checker + .get_existing_proof_ids(&block_root) + .map(|ids| ids.len()) + .unwrap_or(0); + if updated_count < min_proofs_required { + return Err(proof_error(format!( + "missing execution proofs for {block_root:?}: have {updated_count}, need {min_proofs_required}" + ))); + } + } + + if !proofs_by_root.is_empty() { + let unknown_roots: Vec<_> = proofs_by_root.keys().collect(); + debug!(?unknown_roots, "Execution proofs for unknown block roots"); + } + + Ok(()) + } + /// Request block of `block_root` if necessary by checking: /// - If the da_checker has a pending block from gossip or a previous request /// @@ -1402,6 +1797,75 @@ impl SyncNetworkContext { Ok((id, requested_columns)) } + /// Find a zkvm-enabled peer from the given peer sets. + /// + /// Peers advertise zkvm support via their ENR's zkvm flag. This function + /// checks both block_peers and column_peers to find any peer that supports + /// the execution_proofs_by_range protocol. + fn find_zkvm_enabled_peer( + &self, + block_peers: &HashSet, + column_peers: &HashSet, + ) -> Option { + let peers_db = self.network_globals().peers.read(); + + // First try block_peers, then column_peers + let all_peers = block_peers.iter().chain(column_peers.iter()); + + for peer in all_peers { + if peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + { + return Some(*peer); + } + } + + None + } + + fn send_execution_proofs_by_range_request( + &mut self, + peer_id: PeerId, + request: ExecutionProofsByRangeRequest, + parent_request_id: ComponentsByRangeRequestId, + request_span: Span, + ) -> Result { + let id = ExecutionProofsByRangeRequestId { + id: self.next_id(), + parent_request_id, + peer: peer_id, + }; + + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request: RequestType::ExecutionProofsByRange(request.clone()), + app_request_id: AppRequestId::Sync(SyncRequestId::ExecutionProofsByRange(id)), + }) + .map_err(|_| RpcRequestSendError::InternalError("network send error".to_owned()))?; + + debug!( + method = "ExecutionProofsByRange", + slots = request.count, + epoch = %Slot::new(request.start_slot).epoch(T::EthSpec::slots_per_epoch()), + peer = %peer_id, + %id, + "Sync RPC request sent" + ); + + self.execution_proofs_by_range_requests.insert( + id, + peer_id, + // false = do not enforce max_requests are returned for *_by_range methods. We don't + // know how many proofs to expect per block. + false, + ExecutionProofsByRangeRequestItems::new(request), + request_span, + ); + Ok(id) + } + pub fn is_execution_engine_online(&self) -> bool { self.execution_engine_state == EngineState::Online } @@ -1640,6 +2104,20 @@ impl SyncNetworkContext { self.on_rpc_response_result(id, "DataColumnsByRange", resp, peer_id, |d| d.len()) } + /// Handles a response for an execution proofs by range request. + #[allow(clippy::type_complexity)] + pub(crate) fn on_execution_proofs_by_range_response( + &mut self, + id: ExecutionProofsByRangeRequestId, + peer_id: PeerId, + rpc_event: RpcEvent>, + ) -> Option>>> { + let resp = self + .execution_proofs_by_range_requests + .on_response(id, rpc_event); + self.on_rpc_response_result(id, "ExecutionProofsByRange", resp, peer_id, |p| p.len()) + } + fn on_rpc_response_result usize>( &mut self, id: I, diff --git a/beacon_node/network/src/sync/network_context/requests.rs b/beacon_node/network/src/sync/network_context/requests.rs index 63249ed2a4b..238e551659d 100644 --- a/beacon_node/network/src/sync/network_context/requests.rs +++ b/beacon_node/network/src/sync/network_context/requests.rs @@ -15,6 +15,7 @@ pub use data_columns_by_range::DataColumnsByRangeRequestItems; pub use data_columns_by_root::{ DataColumnsByRootRequestItems, DataColumnsByRootSingleBlockRequest, }; +pub use execution_proofs_by_range::ExecutionProofsByRangeRequestItems; pub use execution_proofs_by_root::{ ExecutionProofsByRootRequestItems, ExecutionProofsByRootSingleBlockRequest, }; @@ -29,6 +30,7 @@ mod blocks_by_range; mod blocks_by_root; mod data_columns_by_range; mod data_columns_by_root; +mod execution_proofs_by_range; mod execution_proofs_by_root; #[derive(Debug, PartialEq, Eq, IntoStaticStr)] diff --git a/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs b/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs new file mode 100644 index 00000000000..179f08a6547 --- /dev/null +++ b/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs @@ -0,0 +1,54 @@ +use super::{ActiveRequestItems, LookupVerifyError}; +use lighthouse_network::rpc::methods::ExecutionProofsByRangeRequest; +use std::sync::Arc; +use types::{ExecutionProof, Slot}; + +/// Accumulates results of an execution_proofs_by_range request. Only returns items after receiving +/// the stream termination. +pub struct ExecutionProofsByRangeRequestItems { + request: ExecutionProofsByRangeRequest, + items: Vec>, +} + +impl ExecutionProofsByRangeRequestItems { + pub fn new(request: ExecutionProofsByRangeRequest) -> Self { + Self { + request, + items: vec![], + } + } +} + +impl ActiveRequestItems for ExecutionProofsByRangeRequestItems { + type Item = Arc; + + fn add(&mut self, proof: Self::Item) -> Result { + let proof_slot = proof.slot; + + // Verify the proof is within the requested slot range + if proof_slot < Slot::new(self.request.start_slot) + || proof_slot >= Slot::new(self.request.start_slot + self.request.count) + { + return Err(LookupVerifyError::UnrequestedSlot(proof_slot)); + } + + // Check for duplicate proofs (same slot and proof_id) + if self + .items + .iter() + .any(|existing| existing.slot == proof_slot && existing.proof_id == proof.proof_id) + { + return Err(LookupVerifyError::DuplicatedProofIDs(proof.proof_id)); + } + + self.items.push(proof); + + // We don't know exactly how many proofs to expect (depends on block content), + // so we never return true here - rely on stream termination + Ok(false) + } + + fn consume(&mut self) -> Vec { + std::mem::take(&mut self.items) + } +} diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 4ce10e23ca1..251dff9ffb7 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -6,12 +6,14 @@ use crate::sync::batch::{ BatchConfig, BatchInfo, BatchOperationOutcome, BatchProcessingResult, BatchState, }; use crate::sync::block_sidecar_coupling::CouplingError; -use crate::sync::network_context::{RangeRequestId, RpcRequestSendError, RpcResponseError}; +use crate::sync::network_context::{ + NoPeerError, RangeRequestId, RpcRequestSendError, RpcResponseError, +}; use crate::sync::{BatchProcessResult, network_context::SyncNetworkContext}; use beacon_chain::BeaconChainTypes; use beacon_chain::block_verification_types::RpcBlock; use lighthouse_network::service::api_types::Id; -use lighthouse_network::{PeerAction, PeerId}; +use lighthouse_network::{PeerAction, PeerId, Subnet}; use lighthouse_tracing::SPAN_SYNCING_CHAIN; use logging::crit; use std::collections::{BTreeMap, HashSet, btree_map::Entry}; @@ -463,6 +465,12 @@ impl SyncingChain { // target when there is no sampling peers available. This is a valid state and should not // return an error. return Ok(KeepChain); + } else if !self.good_peers_on_execution_proof_subnet(self.processing_target, network) { + debug!( + src = "process_completed_batches", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); } else { // NOTE: It is possible that the batch doesn't exist for the processing id. This can happen // when we complete a batch and attempt to download a new batch but there are: @@ -944,6 +952,31 @@ impl SyncingChain { CouplingError::BlobPeerFailure(msg) => { tracing::debug!(?batch_id, msg, "Blob peer failure"); } + CouplingError::ExecutionProofPeerFailure { + error, + peer, + exceeded_retries, + } => { + tracing::debug!(?batch_id, ?peer, error, "Execution proof peer failure"); + if !*exceeded_retries { + if let BatchOperationOutcome::Failed { blacklist } = + batch.downloading_to_awaiting_download()? + { + return Err(RemoveChain::ChainFailed { + blacklist, + failing_batch: batch_id, + }); + } + let mut failed_peers = HashSet::new(); + failed_peers.insert(*peer); + return self.retry_execution_proof_batch( + network, + batch_id, + request_id, + failed_peers, + ); + } + } CouplingError::InternalError(msg) => { tracing::error!(?batch_id, msg, "Block components coupling internal error"); } @@ -1020,6 +1053,13 @@ impl SyncingChain { for batch_id in awaiting_downloads { if self.good_peers_on_sampling_subnets(batch_id, network) { + if !self.good_peers_on_execution_proof_subnet(batch_id, network) { + debug!( + src = "attempt_send_awaiting_download_batches", + "Waiting for zkvm-enabled peers for execution proofs" + ); + continue; + } self.send_batch(network, batch_id)?; } else { debug!( @@ -1083,6 +1123,13 @@ impl SyncingChain { return Ok(KeepChain); } Err(e) => match e { + RpcRequestSendError::NoPeer(NoPeerError::ExecutionProofPeer) => { + debug!( + %batch_id, + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); + } // TODO(das): Handle the NoPeer case explicitly and don't drop the batch. For // sync to work properly it must be okay to have "stalled" batches in // AwaitingDownload state. Currently it will error with invalid state if @@ -1163,6 +1210,45 @@ impl SyncingChain { Ok(KeepChain) } + /// Retries execution proof requests within the batch by creating a new proofs request. + fn retry_execution_proof_batch( + &mut self, + network: &mut SyncNetworkContext, + batch_id: BatchId, + id: Id, + mut failed_peers: HashSet, + ) -> ProcessingResult { + let _guard = self.span.clone().entered(); + debug!(%batch_id, %id, ?failed_peers, "Retrying execution proof requests"); + if let Some(batch) = self.batches.get_mut(&batch_id) { + failed_peers.extend(&batch.failed_peers()); + let req = batch.to_blocks_by_range_request().0; + + let synced_peers = network + .network_globals() + .peers + .read() + .synced_peers_for_epoch(batch_id) + .cloned() + .collect::>(); + + match network.retry_execution_proofs_by_range(id, &synced_peers, &failed_peers, req) { + Ok(()) => { + batch.start_downloading(id)?; + debug!( + ?batch_id, + id, "Retried execution proof requests from other peers" + ); + return Ok(KeepChain); + } + Err(e) => { + debug!(?batch_id, id, e, "Failed to retry execution proof batch"); + } + } + } + Ok(KeepChain) + } + /// Returns true if this chain is currently syncing. pub fn is_syncing(&self) -> bool { match self.state { @@ -1206,6 +1292,13 @@ impl SyncingChain { ); return Ok(KeepChain); } + if !self.good_peers_on_execution_proof_subnet(epoch, network) { + debug!( + src = "request_batches_optimistic", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); + } if let Entry::Vacant(entry) = self.batches.entry(epoch) { let batch_type = network.batch_type(epoch); @@ -1252,6 +1345,27 @@ impl SyncingChain { } } + /// Returns true if there is at least one zkvm-enabled peer for execution proofs. + fn good_peers_on_execution_proof_subnet( + &self, + epoch: Epoch, + network: &SyncNetworkContext, + ) -> bool { + if !network.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + return true; + } + + let peers_db = network.network_globals().peers.read(); + let synced_peers: HashSet<_> = peers_db.synced_peers_for_epoch(epoch).cloned().collect(); + + self.peers.iter().chain(synced_peers.iter()).any(|peer| { + peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + }) + } + /// Creates the next required batch from the chain. If there are no more batches required, /// `false` is returned. fn include_next_batch(&mut self, network: &mut SyncNetworkContext) -> Option { @@ -1294,6 +1408,13 @@ impl SyncingChain { ); return None; } + if !self.good_peers_on_execution_proof_subnet(self.to_be_downloaded, network) { + debug!( + src = "include_next_batch", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return None; + } // If no batch needs a retry, attempt to send the batch of the next epoch to download let next_batch_id = self.to_be_downloaded; diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 8e190da2b9d..fb4adbcee65 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -61,8 +61,16 @@ impl TestRig { } fn test_setup_with_spec(spec: ChainSpec) -> Self { + Self::test_setup_with_spec_and_zkvm(spec, false, None) + } + + fn test_setup_with_spec_and_zkvm( + spec: ChainSpec, + zkvm_dummy_verifiers: bool, + anchor_oldest_slot: Option, + ) -> Self { // Initialise a new beacon chain - let harness = BeaconChainHarness::>::builder(E) + let mut builder = BeaconChainHarness::>::builder(E) .spec(Arc::new(spec)) .deterministic_keypairs(1) .fresh_ephemeral_store() @@ -71,8 +79,23 @@ impl TestRig { Slot::new(0), Duration::from_secs(0), Duration::from_secs(12), - )) - .build(); + )); + + if zkvm_dummy_verifiers { + // TODO(zkproofs): For unit tests, we likely always want dummy verifiers + builder = builder.zkvm_with_dummy_verifiers(); + } + + let harness = builder.build(); + if let Some(oldest_slot) = anchor_oldest_slot { + let store = &harness.chain.store; + let prev_anchor = store.get_anchor_info(); + let mut new_anchor = prev_anchor.clone(); + new_anchor.oldest_block_slot = oldest_slot; + store + .compare_and_set_anchor_info_with_write(prev_anchor, new_anchor) + .expect("anchor info updated"); + } let chain = harness.chain.clone(); let fork_context = Arc::new(ForkContext::new::( @@ -160,7 +183,20 @@ impl TestRig { pub fn test_setup_after_fulu_with_zkvm() -> Option { let mut spec = test_spec::(); spec.zkvm_enabled = true; - let r = Self::test_setup_with_spec(spec); + let r = Self::test_setup_with_spec_and_zkvm(spec, true, None); + if r.fork_name.fulu_enabled() { + Some(r) + } else { + None + } + } + + /// Setup test rig for Fulu with zkvm enabled and backfill required. + pub fn test_setup_after_fulu_with_zkvm_backfill() -> Option { + let mut spec = test_spec::(); + spec.zkvm_enabled = true; + let backfill_start_slot = Slot::new(E::slots_per_epoch()); + let r = Self::test_setup_with_spec_and_zkvm(spec, true, Some(backfill_start_slot)); if r.fork_name.fulu_enabled() { Some(r) } else { diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index cb728a90c1b..f122087ae3c 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -3,26 +3,26 @@ use crate::network_beacon_processor::ChainSegmentProcessId; use crate::status::ToStatusMessage; use crate::sync::SyncMessage; use crate::sync::manager::SLOT_IMPORT_TOLERANCE; -use crate::sync::network_context::RangeRequestId; +use crate::sync::network_context::{MAX_EXECUTION_PROOF_RETRIES, RangeRequestId}; use crate::sync::range_sync::RangeSyncType; use beacon_chain::data_column_verification::CustodyDataColumn; use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; use beacon_chain::{EngineState, NotifyExecutionLayer, block_verification_types::RpcBlock}; use beacon_processor::WorkType; -use lighthouse_network::rpc::RequestType; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, DataColumnsByRangeRequest, OldBlocksByRangeRequest, - OldBlocksByRangeRequestV2, StatusMessageV2, + BlobsByRangeRequest, DataColumnsByRangeRequest, ExecutionProofsByRangeRequest, + OldBlocksByRangeRequest, OldBlocksByRangeRequestV2, StatusMessageV2, }; +use lighthouse_network::rpc::{RPCError, RequestType}; use lighthouse_network::service::api_types::{ AppRequestId, BlobsByRangeRequestId, BlocksByRangeRequestId, DataColumnsByRangeRequestId, - SyncRequestId, + ExecutionProofsByRangeRequestId, SyncRequestId, }; use lighthouse_network::{PeerId, SyncInfo}; use std::time::Duration; use types::{ - BlobSidecarList, BlockImportSource, Epoch, EthSpec, Hash256, MinimalEthSpec as E, - SignedBeaconBlock, SignedBeaconBlockHash, Slot, + BlobSidecarList, BlockImportSource, Epoch, EthSpec, ExecutionBlockHash, ExecutionProof, + ExecutionProofId, Hash256, MinimalEthSpec as E, SignedBeaconBlock, SignedBeaconBlockHash, Slot, }; const D: Duration = Duration::new(0, 0); @@ -38,6 +38,13 @@ enum ByRangeDataRequestIds { PostPeerDAS(Vec<(DataColumnsByRangeRequestId, PeerId)>), } +struct BlocksByRangeRequestMeta { + id: BlocksByRangeRequestId, + peer: PeerId, + start_slot: u64, + count: u64, +} + /// Sync tests are usually written in the form: /// - Do some action /// - Expect a request to be sent @@ -84,6 +91,20 @@ impl TestRig { }) } + fn add_head_zkvm_peer_with_root(&mut self, head_root: Hash256) -> PeerId { + let local_info = self.local_info(); + let peer_id = self.new_connected_zkvm_peer(); + self.send_sync_message(SyncMessage::AddPeer( + peer_id, + SyncInfo { + head_root, + head_slot: local_info.head_slot + 1 + Slot::new(SLOT_IMPORT_TOLERANCE as u64), + ..local_info + }, + )); + peer_id + } + // Produce a finalized peer with an advanced finalized epoch fn add_finalized_peer(&mut self) -> PeerId { self.add_finalized_peer_with_root(Hash256::random()) @@ -155,6 +176,13 @@ impl TestRig { } } + fn add_synced_zkvm_peer(&mut self) -> PeerId { + let peer_id = self.new_connected_zkvm_peer(); + let local_info = self.local_info(); + self.send_sync_message(SyncMessage::AddPeer(peer_id, local_info)); + peer_id + } + fn assert_state(&self, state: RangeSyncType) { assert_eq!( self.sync_manager @@ -200,6 +228,16 @@ impl TestRig { &mut self, request_filter: RequestFilter, ) -> ((BlocksByRangeRequestId, PeerId), ByRangeDataRequestIds) { + let (meta, by_range_data_requests) = + self.find_blocks_by_range_request_with_meta(request_filter); + + ((meta.id, meta.peer), by_range_data_requests) + } + + fn find_blocks_by_range_request_with_meta( + &mut self, + request_filter: RequestFilter, + ) -> (BlocksByRangeRequestMeta, ByRangeDataRequestIds) { let filter_f = |peer: PeerId, start_slot: u64| { if let Some(expected_epoch) = request_filter.epoch { let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); @@ -222,10 +260,17 @@ impl TestRig { peer_id, request: RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( - OldBlocksByRangeRequestV2 { start_slot, .. }, + OldBlocksByRangeRequestV2 { + start_slot, count, .. + }, )), app_request_id: AppRequestId::Sync(SyncRequestId::BlocksByRange(id)), - } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), + } if filter_f(*peer_id, *start_slot) => Some(BlocksByRangeRequestMeta { + id: *id, + peer: *peer_id, + start_slot: *start_slot, + count: *count, + }), _ => None, }) .unwrap_or_else(|e| { @@ -272,6 +317,45 @@ impl TestRig { (block_req, by_range_data_requests) } + fn find_execution_proofs_by_range_request( + &mut self, + request_filter: RequestFilter, + ) -> (ExecutionProofsByRangeRequestId, PeerId, u64, u64) { + let filter_f = |peer: PeerId, start_slot: u64| { + if let Some(expected_epoch) = request_filter.epoch { + let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); + if epoch != expected_epoch { + return false; + } + } + if let Some(expected_peer) = request_filter.peer + && peer != expected_peer + { + return false; + } + + true + }; + + self.pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + peer_id, + request: + RequestType::ExecutionProofsByRange(ExecutionProofsByRangeRequest { + start_slot, + count, + }), + app_request_id: AppRequestId::Sync(SyncRequestId::ExecutionProofsByRange(id)), + } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id, *start_slot, *count)), + _ => None, + }) + .unwrap_or_else(|e| { + panic!( + "Should have an ExecutionProofsByRange request, filter {request_filter:?}: {e:?}" + ) + }) + } + fn find_and_complete_blocks_by_range_request( &mut self, request_filter: RequestFilter, @@ -290,6 +374,15 @@ impl TestRig { seen_timestamp: D, }); + self.complete_by_range_data_requests(by_range_data_request_ids); + + blocks_req_id.parent_request_id.requester + } + + fn complete_by_range_data_requests( + &mut self, + by_range_data_request_ids: ByRangeDataRequestIds, + ) { match by_range_data_request_ids { ByRangeDataRequestIds::PreDeneb => {} ByRangeDataRequestIds::PrePeerDAS(id, peer_id) => { @@ -319,8 +412,6 @@ impl TestRig { } } } - - blocks_req_id.parent_request_id.requester } fn find_and_complete_processing_chain_segment(&mut self, id: ChainSegmentProcessId) { @@ -601,3 +692,611 @@ fn finalized_sync_not_enough_custody_peers_on_start() { let last_epoch = advanced_epochs + EXTRA_SYNCED_EPOCHS; r.complete_and_process_range_sync_until(last_epoch, filter()); } + +#[test] +fn range_sync_requests_execution_proofs_for_zkvm() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + rig.assert_state(RangeSyncType::Head); + rig.expect_empty_network(); + + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _ = rig.find_blocks_by_range_request(filter()); + let (_, proof_peer, _, _) = + rig.find_execution_proofs_by_range_request(filter().peer(zkvm_peer)); + assert_eq!(proof_peer, zkvm_peer); +} + +#[test] +fn range_sync_uses_cached_execution_proofs() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let (proof_req_id, proof_peer, proof_start_slot, proof_count) = + rig.find_execution_proofs_by_range_request(filter().peer(zkvm_peer)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert_eq!(proof_count, block_meta.count); + assert_eq!(proof_peer, zkvm_peer); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let min_proofs = rig + .harness + .chain + .spec + .zkvm_min_proofs_required() + .expect("zkvm enabled"); + + let proofs = (0..min_proofs) + .map(|i| { + ExecutionProof::new( + ExecutionProofId::new(u8::try_from(i).expect("proof id fits")).unwrap(), + block.slot(), + block_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap() + }) + .collect::>(); + + rig.harness + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, proofs) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + let request_id = block_meta.id.parent_request_id.requester; + let process_id = match request_id { + RangeRequestId::RangeSync { chain_id, batch_id } => { + ChainSegmentProcessId::RangeBatchId(chain_id, batch_id) + } + RangeRequestId::BackfillSync { batch_id } => { + ChainSegmentProcessId::BackSyncBatchId(batch_id) + } + }; + rig.find_and_complete_processing_chain_segment(process_id); +} + +#[test] +fn range_sync_retries_execution_proofs_without_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry for execution proof failure"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn backfill_retries_execution_proofs_without_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm_backfill() else { + return; + }; + + let zkvm_peer_1 = rig.add_synced_zkvm_peer(); + let zkvm_peer_2 = rig.add_synced_zkvm_peer(); + let local_info = rig.local_info(); + let _supernode_peer = rig.add_supernode_peer(local_info); + + let backfill_epoch = Slot::new(E::slots_per_epoch()) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (block_meta, by_range_data) = + rig.find_blocks_by_range_request_with_meta(filter().epoch(backfill_epoch)); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(backfill_epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry for execution proof failure"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(backfill_epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn range_sync_execution_proof_retries_exhaust_then_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_3 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (mut proof_req_id, mut proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2 || proof_peer == zkvm_peer_3); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + rig.complete_by_range_data_requests(by_range_data); + + for attempt in 1..=MAX_EXECUTION_PROOF_RETRIES { + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if attempt < MAX_EXECUTION_PROOF_RETRIES { + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry before proof retries are exhausted"); + } + + let (next_req_id, next_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(next_peer, proof_peer); + proof_req_id = next_req_id; + proof_peer = next_peer; + } + } + + rig.pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2(OldBlocksByRangeRequestV2 { + start_slot, + .. + })), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .unwrap_or_else(|e| panic!("Expected BlocksByRange retry after exhausted proofs: {e}")); +} + +#[test] +fn range_sync_proof_retry_on_unsupported_protocol() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + rig.complete_by_range_data_requests(by_range_data); + + rig.send_sync_message(SyncMessage::RpcError { + peer_id: proof_peer, + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + error: RPCError::UnsupportedProtocol, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry on unsupported protocol"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn range_sync_ignores_bad_proofs_when_cached() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert_eq!(proof_peer, zkvm_peer); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let min_proofs = rig + .harness + .chain + .spec + .zkvm_min_proofs_required() + .expect("zkvm enabled"); + + let proofs = (0..min_proofs) + .map(|i| { + ExecutionProof::new( + ExecutionProofId::new(u8::try_from(i).expect("proof id fits")).unwrap(), + block.slot(), + block_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap() + }) + .collect::>(); + + rig.harness + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, proofs) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::ExecutionProofsByRange(ExecutionProofsByRangeRequest { + start_slot, + .. + }), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected execution proof retry when cache already satisfies requirement"); + } +} diff --git a/beacon_node/store/src/errors.rs b/beacon_node/store/src/errors.rs index a07cc838863..9cb6620816e 100644 --- a/beacon_node/store/src/errors.rs +++ b/beacon_node/store/src/errors.rs @@ -32,6 +32,8 @@ pub enum Error { BlobInfoConcurrentMutation, /// The store's `data_column_info` was mutated concurrently, the latest modification wasn't applied. DataColumnInfoConcurrentMutation, + /// The store's `execution_proof_info` was mutated concurrently, the latest modification wasn't applied. + ExecutionProofInfoConcurrentMutation, /// The block or state is unavailable due to weak subjectivity sync. HistoryUnavailable, /// State reconstruction cannot commence because not all historic blocks are known. @@ -92,6 +94,7 @@ pub enum Error { LoadSplit(Box), LoadBlobInfo(Box), LoadDataColumnInfo(Box), + LoadExecutionProofInfo(Box), LoadConfig(Box), LoadHotStateSummary(Hash256, Box), LoadHotStateSummaryForSplit(Box), diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index c4137191744..a05d915795f 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -9,12 +9,13 @@ use crate::metadata::{ ANCHOR_INFO_KEY, ANCHOR_UNINITIALIZED, AnchorInfo, BLOB_INFO_KEY, BlobInfo, COMPACTION_TIMESTAMP_KEY, CONFIG_KEY, CURRENT_SCHEMA_VERSION, CompactionTimestamp, DATA_COLUMN_CUSTODY_INFO_KEY, DATA_COLUMN_INFO_KEY, DataColumnCustodyInfo, DataColumnInfo, - SCHEMA_VERSION_KEY, SPLIT_KEY, STATE_UPPER_LIMIT_NO_RETAIN, SchemaVersion, + EXECUTION_PROOF_INFO_KEY, ExecutionProofInfo, SCHEMA_VERSION_KEY, SPLIT_KEY, + STATE_UPPER_LIMIT_NO_RETAIN, SchemaVersion, }; use crate::state_cache::{PutStateOutcome, StateCache}; use crate::{ BlobSidecarListFromRoot, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp, StoreItem, - StoreOp, get_data_column_key, + StoreOp, get_data_column_key, get_execution_proof_key, metrics::{self, COLD_METRIC, HOT_METRIC}, parse_data_column_key, }; @@ -61,6 +62,8 @@ pub struct HotColdDB, Cold: ItemStore> { blob_info: RwLock, /// The starting slots for the range of data columns stored in the database. data_column_info: RwLock, + /// The starting slots for the range of execution proofs stored in the database. + execution_proof_info: RwLock, pub(crate) config: StoreConfig, pub hierarchy: HierarchyModuli, /// Cold database containing compact historical data. @@ -93,6 +96,7 @@ struct BlockCache { block_cache: LruCache>, blob_cache: LruCache>, data_column_cache: LruCache>>>, + execution_proof_cache: LruCache>>, data_column_custody_info_cache: Option, } @@ -102,6 +106,7 @@ impl BlockCache { block_cache: LruCache::new(size), blob_cache: LruCache::new(size), data_column_cache: LruCache::new(size), + execution_proof_cache: LruCache::new(size), data_column_custody_info_cache: None, } } @@ -116,6 +121,9 @@ impl BlockCache { .get_or_insert_mut(block_root, Default::default) .insert(data_column.index, data_column); } + pub fn put_execution_proofs(&mut self, block_root: Hash256, proofs: Vec>) { + self.execution_proof_cache.put(block_root, proofs); + } pub fn put_data_column_custody_info( &mut self, data_column_custody_info: Option, @@ -139,6 +147,12 @@ impl BlockCache { .get(block_root) .and_then(|map| map.get(column_index).cloned()) } + pub fn get_execution_proofs( + &mut self, + block_root: &Hash256, + ) -> Option>> { + self.execution_proof_cache.get(block_root).cloned() + } pub fn get_data_column_custody_info(&self) -> Option { self.data_column_custody_info_cache.clone() } @@ -151,10 +165,14 @@ impl BlockCache { pub fn delete_data_columns(&mut self, block_root: &Hash256) { let _ = self.data_column_cache.pop(block_root); } + pub fn delete_execution_proofs(&mut self, block_root: &Hash256) { + let _ = self.execution_proof_cache.pop(block_root); + } pub fn delete(&mut self, block_root: &Hash256) { self.delete_block(block_root); self.delete_blobs(block_root); self.delete_data_columns(block_root); + self.delete_execution_proofs(block_root); } } @@ -232,6 +250,7 @@ impl HotColdDB, MemoryStore> { anchor_info: RwLock::new(ANCHOR_UNINITIALIZED), blob_info: RwLock::new(BlobInfo::default()), data_column_info: RwLock::new(DataColumnInfo::default()), + execution_proof_info: RwLock::new(ExecutionProofInfo::default()), cold_db: MemoryStore::open(), blobs_db: MemoryStore::open(), hot_db: MemoryStore::open(), @@ -286,6 +305,7 @@ impl HotColdDB, BeaconNodeBackend> { anchor_info, blob_info: RwLock::new(BlobInfo::default()), data_column_info: RwLock::new(DataColumnInfo::default()), + execution_proof_info: RwLock::new(ExecutionProofInfo::default()), blobs_db: BeaconNodeBackend::open(&config, blobs_db_path)?, cold_db: BeaconNodeBackend::open(&config, cold_path)?, hot_db, @@ -395,10 +415,38 @@ impl HotColdDB, BeaconNodeBackend> { new_data_column_info.clone(), )?; + // Initialize execution proof info + let execution_proof_info = db.load_execution_proof_info()?; + let zkvm_fork_slot = db + .spec + .zkvm_fork_epoch() + .map(|epoch| epoch.start_slot(E::slots_per_epoch())); + let new_execution_proof_info = match &execution_proof_info { + Some(execution_proof_info) => { + // Set the oldest execution proof slot to the fork slot if it is not yet set. + let oldest_execution_proof_slot = execution_proof_info + .oldest_execution_proof_slot + .or(zkvm_fork_slot); + ExecutionProofInfo { + oldest_execution_proof_slot, + } + } + // First start. + None => ExecutionProofInfo { + // Set the oldest execution proof slot to the fork slot if it is not yet set. + oldest_execution_proof_slot: zkvm_fork_slot, + }, + }; + db.compare_and_set_execution_proof_info_with_write( + <_>::default(), + new_execution_proof_info.clone(), + )?; + info!( path = ?blobs_db_path, oldest_blob_slot = ?new_blob_info.oldest_blob_slot, oldest_data_column_slot = ?new_data_column_info.oldest_data_column_slot, + oldest_execution_proof_slot = ?new_execution_proof_info.oldest_execution_proof_slot, "Blob DB initialized" ); @@ -1027,6 +1075,47 @@ impl, Cold: ItemStore> HotColdDB } } + /// Store execution proofs for a block. + pub fn put_execution_proofs( + &self, + block_root: &Hash256, + proofs: &[ExecutionProof], + ) -> Result<(), Error> { + for proof in proofs { + self.blobs_db.put_bytes( + DBColumn::BeaconExecutionProof, + &get_execution_proof_key(block_root, proof.proof_id.as_u8()), + &proof.as_ssz_bytes(), + )?; + } + if !proofs.is_empty() { + let cached = proofs + .iter() + .map(|proof| Arc::new(proof.clone())) + .collect::>(); + self.block_cache + .as_ref() + .inspect(|cache| cache.lock().put_execution_proofs(*block_root, cached)); + } + Ok(()) + } + + /// Create key-value store operations for storing execution proofs. + pub fn execution_proofs_as_kv_store_ops( + &self, + block_root: &Hash256, + proofs: &[ExecutionProof], + ops: &mut Vec, + ) { + for proof in proofs { + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconExecutionProof, + get_execution_proof_key(block_root, proof.proof_id.as_u8()), + proof.as_ssz_bytes(), + )); + } + } + /// Store a state in the store. pub fn put_state(&self, state_root: &Hash256, state: &BeaconState) -> Result<(), Error> { let mut ops: Vec = Vec::new(); @@ -2558,6 +2647,47 @@ impl, Cold: ItemStore> HotColdDB } } + /// Fetch all execution proofs for a given block from the store. + pub fn get_execution_proofs( + &self, + block_root: &Hash256, + ) -> Result>, Error> { + if let Some(proofs) = self + .block_cache + .as_ref() + .and_then(|cache| cache.lock().get_execution_proofs(block_root)) + { + return Ok(proofs); + } + + let mut proofs = Vec::new(); + let prefix = block_root.as_slice(); + + for result in self + .blobs_db + .iter_column_from::>(DBColumn::BeaconExecutionProof, prefix) + { + let (key, value) = result?; + // Check if key starts with our block_root prefix + if !key.starts_with(prefix) { + // We've moved past this block's proofs + break; + } + let proof = Arc::new(ExecutionProof::from_ssz_bytes(&value)?); + proofs.push(proof); + } + + if !proofs.is_empty() { + self.block_cache.as_ref().inspect(|cache| { + cache + .lock() + .put_execution_proofs(*block_root, proofs.clone()) + }); + } + + Ok(proofs) + } + /// Fetch all keys in the data_column column with prefix `block_root` pub fn get_data_column_keys(&self, block_root: Hash256) -> Result, Error> { self.blobs_db @@ -2877,6 +3007,77 @@ impl, Cold: ItemStore> HotColdDB data_column_info.as_kv_store_op(DATA_COLUMN_INFO_KEY) } + /// Get a clone of the store's execution proof info. + /// + /// To do mutations, use `compare_and_set_execution_proof_info`. + pub fn get_execution_proof_info(&self) -> ExecutionProofInfo { + self.execution_proof_info.read_recursive().clone() + } + + /// Initialize the `ExecutionProofInfo` when starting from genesis or a checkpoint. + pub fn init_execution_proof_info(&self, anchor_slot: Slot) -> Result { + let oldest_execution_proof_slot = self.spec.zkvm_fork_epoch().map(|fork_epoch| { + std::cmp::max(anchor_slot, fork_epoch.start_slot(E::slots_per_epoch())) + }); + let execution_proof_info = ExecutionProofInfo { + oldest_execution_proof_slot, + }; + self.compare_and_set_execution_proof_info( + self.get_execution_proof_info(), + execution_proof_info, + ) + } + + /// Atomically update the execution proof info from `prev_value` to `new_value`. + /// + /// Return a `KeyValueStoreOp` which should be written to disk, possibly atomically with other + /// values. + /// + /// Return an `ExecutionProofInfoConcurrentMutation` error if the `prev_value` provided + /// is not correct. + pub fn compare_and_set_execution_proof_info( + &self, + prev_value: ExecutionProofInfo, + new_value: ExecutionProofInfo, + ) -> Result { + let mut execution_proof_info = self.execution_proof_info.write(); + if *execution_proof_info == prev_value { + let kv_op = self.store_execution_proof_info_in_batch(&new_value); + *execution_proof_info = new_value; + Ok(kv_op) + } else { + Err(Error::ExecutionProofInfoConcurrentMutation) + } + } + + /// As for `compare_and_set_execution_proof_info`, but also writes to disk immediately. + pub fn compare_and_set_execution_proof_info_with_write( + &self, + prev_value: ExecutionProofInfo, + new_value: ExecutionProofInfo, + ) -> Result<(), Error> { + let kv_store_op = self.compare_and_set_execution_proof_info(prev_value, new_value)?; + self.hot_db.do_atomically(vec![kv_store_op]) + } + + /// Load the execution proof info from disk, but do not set `self.execution_proof_info`. + fn load_execution_proof_info(&self) -> Result, Error> { + self.hot_db + .get(&EXECUTION_PROOF_INFO_KEY) + .map_err(|e| Error::LoadExecutionProofInfo(e.into())) + } + + /// Store the given `execution_proof_info` to disk. + /// + /// The argument is intended to be `self.execution_proof_info`, but is passed manually to avoid + /// issues with recursive locking. + fn store_execution_proof_info_in_batch( + &self, + execution_proof_info: &ExecutionProofInfo, + ) -> KeyValueStoreOp { + execution_proof_info.as_kv_store_op(EXECUTION_PROOF_INFO_KEY) + } + /// Return the slot-window describing the available historic states. /// /// Returns `(lower_limit, upper_limit)`. @@ -3395,6 +3596,178 @@ impl, Cold: ItemStore> HotColdDB Ok(()) } + /// Try to prune execution proofs older than the execution proof boundary. + /// + /// Proofs from the epoch `execution_proof_boundary` are retained. + /// This epoch is an _exclusive_ endpoint for the pruning process. + /// + /// This function only supports pruning execution proofs older than the split point, + /// which is older than (or equal to) finalization. + pub fn try_prune_execution_proofs( + &self, + force: bool, + execution_proof_boundary: Epoch, + ) -> Result<(), Error> { + // Check if zkvm fork is enabled + if self.spec.zkvm_fork_epoch().is_none() { + debug!("ZKVM fork is disabled"); + return Ok(()); + } + + let pruning_enabled = self.get_config().prune_blobs; // Use same config as blobs for now + if !force && !pruning_enabled { + debug!( + prune_blobs = pruning_enabled, + "Execution proof pruning is disabled" + ); + return Ok(()); + } + + let execution_proof_info = self.get_execution_proof_info(); + let Some(oldest_execution_proof_slot) = execution_proof_info.oldest_execution_proof_slot + else { + debug!("No execution proofs stored yet"); + return Ok(()); + }; + + let start_epoch = oldest_execution_proof_slot.epoch(E::slots_per_epoch()); + + // Prune execution proofs up until the `execution_proof_boundary - 1` or the split + // slot's epoch, whichever is older. + let split = self.get_split_info(); + let end_epoch = std::cmp::min( + execution_proof_boundary.saturating_sub(1u64), + split.slot.epoch(E::slots_per_epoch()).saturating_sub(1u64), + ); + let end_slot = end_epoch.end_slot(E::slots_per_epoch()); + + let can_prune = end_epoch != Epoch::new(0) && start_epoch <= end_epoch; + if !can_prune { + debug!( + %oldest_execution_proof_slot, + %execution_proof_boundary, + %split.slot, + %end_epoch, + %start_epoch, + "Execution proofs are pruned" + ); + return Ok(()); + } + + debug!( + %end_epoch, + %execution_proof_boundary, + "Pruning execution proofs" + ); + + // Iterate blocks backwards from the `end_epoch`. + let Some((end_block_root, _)) = self + .forwards_block_roots_iterator_until(end_slot, end_slot, || { + self.get_hot_state(&split.state_root, true)? + .ok_or(HotColdDBError::MissingSplitState( + split.state_root, + split.slot, + )) + .map(|state| (state, split.state_root)) + .map_err(Into::into) + })? + .next() + .transpose()? + else { + debug!( + %end_epoch, + %execution_proof_boundary, + "No execution proofs to prune" + ); + return Ok(()); + }; + + let mut db_ops = vec![]; + let mut removed_block_roots = vec![]; + let mut new_oldest_slot: Option = None; + + // Iterate blocks backwards until we reach blocks older than the boundary. + for tuple in ParentRootBlockIterator::new(self, end_block_root) { + let (block_root, blinded_block) = tuple?; + let slot = blinded_block.slot(); + + // Get all execution proof keys for this block + let keys = self.get_all_execution_proof_keys(&block_root); + + // Check if any proofs exist for this block + let mut block_has_proofs = false; + for key in keys { + if self + .blobs_db + .key_exists(DBColumn::BeaconExecutionProof, &key)? + { + block_has_proofs = true; + db_ops.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconExecutionProof, + key, + )); + } + } + + if block_has_proofs { + debug!( + ?block_root, + %slot, + "Pruning execution proofs for block" + ); + removed_block_roots.push(block_root); + new_oldest_slot = Some(slot); + } + // Continue iterating even if this block has no proofs - proofs may be sparse + } + + // Commit deletions + if !db_ops.is_empty() { + debug!( + num_deleted = db_ops.len(), + "Deleting execution proofs from disk" + ); + self.blobs_db.do_atomically(db_ops)?; + } + + // TODO(zkproofs): Fix this to make it more readable + if !removed_block_roots.is_empty() + && let Some(mut block_cache) = self.block_cache.as_ref().map(|cache| cache.lock()) + { + for block_root in removed_block_roots { + block_cache.delete_execution_proofs(&block_root); + } + } + + // Update the execution proof info with the new oldest slot + if let Some(new_slot) = new_oldest_slot { + let new_oldest = end_slot + 1; + self.compare_and_set_execution_proof_info_with_write( + execution_proof_info.clone(), + ExecutionProofInfo { + oldest_execution_proof_slot: Some(new_oldest), + }, + )?; + debug!( + old_oldest = %new_slot, + new_oldest = %new_oldest, + "Updated execution proof info" + ); + } + + debug!("Execution proof pruning complete"); + + Ok(()) + } + + /// Get all possible execution proof keys for a given block root. + /// Returns keys for proof_ids 0 to MAX_PROOFS-1. + fn get_all_execution_proof_keys(&self, block_root: &Hash256) -> Vec> { + (0..types::MAX_PROOFS as u8) + .map(|proof_id| get_execution_proof_key(block_root, proof_id)) + .collect() + } + /// Delete *all* states from the freezer database and update the anchor accordingly. /// /// WARNING: this method deletes the genesis state and replaces it with the provided diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index ae5b2e1e571..516e858e581 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -40,6 +40,7 @@ use strum::{EnumIter, EnumString, IntoStaticStr}; pub use types::*; const DATA_COLUMN_DB_KEY_SIZE: usize = 32 + 8; +const EXECUTION_PROOF_DB_KEY_SIZE: usize = 32 + 1; // block_root + proof_id pub type ColumnIter<'a, K> = Box), Error>> + 'a>; pub type ColumnKeyIter<'a, K> = Box> + 'a>; @@ -171,6 +172,25 @@ pub fn parse_data_column_key(data: Vec) -> Result<(Hash256, ColumnIndex), Er Ok((block_root, column_index)) } +pub fn get_execution_proof_key(block_root: &Hash256, proof_id: u8) -> Vec { + let mut result = block_root.as_slice().to_vec(); + result.push(proof_id); + result +} + +pub fn parse_execution_proof_key(data: Vec) -> Result<(Hash256, u8), Error> { + if data.len() != EXECUTION_PROOF_DB_KEY_SIZE { + return Err(Error::InvalidKey(format!( + "Unexpected BeaconExecutionProof key len {}", + data.len() + ))); + } + let (block_root_bytes, proof_id_bytes) = data.split_at(32); + let block_root = Hash256::from_slice(block_root_bytes); + let proof_id = proof_id_bytes[0]; + Ok((block_root, proof_id)) +} + #[must_use] #[derive(Clone)] pub enum KeyValueStoreOp { @@ -263,6 +283,12 @@ pub enum DBColumn { BeaconDataColumn, #[strum(serialize = "bdi")] BeaconDataColumnCustodyInfo, + /// For storing execution proofs (zkVM proofs) in the blob database. + /// + /// - Key: `Hash256` block root + `u8` proof_id (33 bytes total). + /// - Value: SSZ-encoded ExecutionProof. + #[strum(serialize = "bep")] + BeaconExecutionProof, /// For full `BeaconState`s in the hot database (finalized or fork-boundary states). /// /// DEPRECATED. @@ -437,6 +463,7 @@ impl DBColumn { | Self::LightClientUpdate | Self::Dummy => 8, Self::BeaconDataColumn => DATA_COLUMN_DB_KEY_SIZE, + Self::BeaconExecutionProof => EXECUTION_PROOF_DB_KEY_SIZE, } } } diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index cf494684515..7a5979481fe 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -19,6 +19,7 @@ pub const ANCHOR_INFO_KEY: Hash256 = Hash256::repeat_byte(5); pub const BLOB_INFO_KEY: Hash256 = Hash256::repeat_byte(6); pub const DATA_COLUMN_INFO_KEY: Hash256 = Hash256::repeat_byte(7); pub const DATA_COLUMN_CUSTODY_INFO_KEY: Hash256 = Hash256::repeat_byte(8); +pub const EXECUTION_PROOF_INFO_KEY: Hash256 = Hash256::repeat_byte(9); /// State upper limit value used to indicate that a node is not storing historic states. pub const STATE_UPPER_LIMIT_NO_RETAIN: Slot = Slot::new(u64::MAX); @@ -255,3 +256,30 @@ impl StoreItem for DataColumnInfo { Ok(Self::from_ssz_bytes(bytes)?) } } + +/// Database parameters relevant to execution proof sync. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, Serialize, Deserialize, Default)] +pub struct ExecutionProofInfo { + /// The slot after which execution proofs are or *will be* available (>=). + /// + /// If this slot is in the future, then it is the first slot of the ZKVM fork, from which + /// execution proofs will be available. + /// + /// If the `oldest_execution_proof_slot` is `None` then this means that the ZKVM fork epoch + /// is not yet known. + pub oldest_execution_proof_slot: Option, +} + +impl StoreItem for ExecutionProofInfo { + fn db_column() -> DBColumn { + DBColumn::BeaconMeta + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &[u8]) -> Result { + Ok(Self::from_ssz_bytes(bytes)?) + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 1bab464b689..21eb5e1b8c7 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -61,7 +61,7 @@ pub mod chain_spec { } // Re-export execution_proof types for backwards compatibility -pub use crate::execution_proof::{ExecutionProof, MAX_PROOF_DATA_BYTES}; +pub use crate::execution_proof::{ExecutionProof, MAX_PROOF_DATA_BYTES, MAX_PROOFS}; pub use crate::execution_proof_id::{EXECUTION_PROOF_TYPE_COUNT, ExecutionProofId}; pub mod beacon_block { diff --git a/dummy_el/geth-wrapper.sh b/dummy_el/geth-wrapper.sh index 8112bb44e9c..5705888b49c 100644 --- a/dummy_el/geth-wrapper.sh +++ b/dummy_el/geth-wrapper.sh @@ -2,28 +2,106 @@ set -e # This is a wrapper that pretends to be geth but actually runs dummy_el -# Kurtosis calls: geth init ... && geth --authrpc.port=8551 ... -# We ignore the init, and when we see the actual geth command with authrpc.port, we start dummy_el +# Kurtosis may call various geth commands - we handle them all appropriately echo "[dummy_el geth-wrapper] Called with: $@" -# Check if this is the "geth init" command and ignore it +# Check if this is the "geth init" command - ignore it if echo "$@" | grep -q "init"; then echo "[dummy_el geth-wrapper] Ignoring 'geth init' command" exit 0 fi -# If we're here, it's the actual geth run command -# Kurtosis mounts JWT secret at /jwt/jwtsecret -JWT_PATH="/jwt/jwtsecret" +# Check for version/help commands +if echo "$@" | grep -qE "^(version|--version|-v|help|--help|-h)$"; then + echo "Dummy-EL/v0.1.0 (geth-compatible wrapper)" + exit 0 +fi + +# Filter out flags that we don't need for dummy_el +# These are geth-specific flags that kurtosis may pass +FILTERED_ARGS="" +for arg in "$@"; do + case "$arg" in + --override.*|--override*|-override.*|-override*) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --datadir=*|--datadir) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --syncmode=*|--syncmode) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --gcmode=*|--gcmode) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --networkid=*|--networkid) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + *) + FILTERED_ARGS="$FILTERED_ARGS $arg" + ;; + esac +done + +# For any other command, we start dummy_el +# Parse geth arguments to extract what we need + +JWT_PATH="" +ENGINE_PORT="8551" +RPC_PORT="8545" +WS_PORT="8546" +METRICS_PORT="9001" +P2P_PORT="30303" +HOST="0.0.0.0" + +# Parse arguments to find JWT secret and ports +for arg in "$@"; do + case "$arg" in + --authrpc.jwtsecret=*) + JWT_PATH="${arg#*=}" + ;; + --authrpc.port=*) + ENGINE_PORT="${arg#*=}" + ;; + --http.port=*) + RPC_PORT="${arg#*=}" + ;; + --ws.port=*) + WS_PORT="${arg#*=}" + ;; + --metrics.port=*) + METRICS_PORT="${arg#*=}" + ;; + --port=*) + P2P_PORT="${arg#*=}" + ;; + --discovery.port=*) + # Use discovery port for P2P if specified + P2P_PORT="${arg#*=}" + ;; + esac +done + +# Fallback to default JWT location if not parsed +if [ -z "$JWT_PATH" ] && [ -f "/jwt/jwtsecret" ]; then + JWT_PATH="/jwt/jwtsecret" +fi echo "[dummy_el geth-wrapper] Starting dummy_el instead of geth" +echo "[dummy_el geth-wrapper] Engine port: $ENGINE_PORT, RPC port: $RPC_PORT, WS port: $WS_PORT" +echo "[dummy_el geth-wrapper] Metrics port: $METRICS_PORT, P2P port: $P2P_PORT" -# Run dummy_el with JWT if available, otherwise without -if [ -f "$JWT_PATH" ]; then +# Build dummy_el command +DUMMY_EL_CMD="/usr/local/bin/dummy_el --host $HOST --port $ENGINE_PORT --rpc-port $RPC_PORT --ws-port $WS_PORT --metrics-port $METRICS_PORT --p2p-port $P2P_PORT" + +# Add JWT if available +if [ -n "$JWT_PATH" ] && [ -f "$JWT_PATH" ]; then echo "[dummy_el geth-wrapper] Using JWT from $JWT_PATH" - exec /usr/local/bin/dummy_el --host 0.0.0.0 --port 8551 --jwt-secret "$JWT_PATH" + DUMMY_EL_CMD="$DUMMY_EL_CMD --jwt-secret $JWT_PATH" else - echo "[dummy_el geth-wrapper] WARNING: No JWT file found at $JWT_PATH" - exec /usr/local/bin/dummy_el --host 0.0.0.0 --port 8551 + echo "[dummy_el geth-wrapper] WARNING: No JWT file found" fi + +echo "[dummy_el geth-wrapper] Executing: $DUMMY_EL_CMD" +exec $DUMMY_EL_CMD diff --git a/execution-witness-sentry/Cargo.toml b/execution-witness-sentry/Cargo.toml new file mode 100644 index 00000000000..ceee0f20bed --- /dev/null +++ b/execution-witness-sentry/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "execution-witness-sentry" +version = "0.1.0" +edition = { workspace = true } +description = "Monitors execution layer nodes and fetches execution witnesses" + +[dependencies] +alloy-provider = { version = "1", features = ["ws"] } +alloy-rpc-types-eth = "1" +anyhow = "1" +clap = { version = "4", features = ["derive"] } +discv5 = { workspace = true } +eventsource-client = "0.13" +flate2 = "1.1" +futures = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = "1" +tokio = { workspace = true, features = ["sync", "rt-multi-thread", "macros"] } +toml = "0.8" +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +url = { workspace = true } diff --git a/execution-witness-sentry/config.toml b/execution-witness-sentry/config.toml new file mode 100644 index 00000000000..b35c2ca262e --- /dev/null +++ b/execution-witness-sentry/config.toml @@ -0,0 +1,27 @@ +output_dir = "." +chain = "local" +retain = 10 +num_proofs = 2 + +[[endpoints]] +name = "el-1-reth-lighthouse" +el_url = "http://127.0.0.1:32003" +el_ws_url = "ws://127.0.0.1:32004" + +# Non-zkvm CL for head event subscription (to know when new blocks arrive) +[[cl_endpoints]] +name = "cl-1-lighthouse-reth" +url = "http://127.0.0.1:33001/" + +# zkvm-enabled CLs for proof submission +[[cl_endpoints]] +name = "cl-4-lighthouse-geth" +url = "http://127.0.0.1:33022/" + +[[cl_endpoints]] +name = "cl-5-lighthouse-geth" +url = "http://127.0.0.1:33029/" + +[[cl_endpoints]] +name = "cl-6-lighthouse-geth" +url = "http://127.0.0.1:33036/" diff --git a/execution-witness-sentry/src/cl_subscription.rs b/execution-witness-sentry/src/cl_subscription.rs new file mode 100644 index 00000000000..040e01cc438 --- /dev/null +++ b/execution-witness-sentry/src/cl_subscription.rs @@ -0,0 +1,128 @@ +//! SSE subscription for CL head events. + +use std::pin::Pin; +use std::task::{Context, Poll}; + +use eventsource_client::{Client, SSE}; +use futures::Stream; +use serde::Deserialize; +use url::Url; + +use crate::error::{Error, Result}; + +/// Head event from the CL. +#[derive(Debug, Clone, Deserialize)] +pub struct HeadEvent { + pub slot: String, + pub block: String, + pub state: String, + pub epoch_transition: bool, + pub execution_optimistic: bool, +} + +/// Block event from the CL. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockEvent { + pub slot: String, + pub block: String, + pub execution_optimistic: bool, +} + +/// Unified CL event. +#[derive(Debug, Clone)] +pub enum ClEvent { + Head(HeadEvent), + Block(BlockEvent), +} + +impl ClEvent { + pub fn slot(&self) -> &str { + match self { + ClEvent::Head(e) => &e.slot, + ClEvent::Block(e) => &e.slot, + } + } + + pub fn block_root(&self) -> &str { + match self { + ClEvent::Head(e) => &e.block, + ClEvent::Block(e) => &e.block, + } + } +} + +/// Stream of CL events. +pub struct ClEventStream { + client: Pin> + Send>>, +} + +impl Stream for ClEventStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + match self.client.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(SSE::Event(event)))) => { + let result = match event.event_type.as_str() { + "head" => serde_json::from_str::(&event.data) + .map(ClEvent::Head) + .map_err(Error::Parse), + "block" => serde_json::from_str::(&event.data) + .map(ClEvent::Block) + .map_err(Error::Parse), + _ => continue, + }; + return Poll::Ready(Some(result)); + } + Poll::Ready(Some(Ok(SSE::Comment(_)))) => continue, + Poll::Ready(Some(Ok(SSE::Connected(_)))) => continue, + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(Err(Error::Sse(format!("{:?}", e))))); + } + Poll::Ready(None) => return Poll::Ready(None), + Poll::Pending => return Poll::Pending, + } + } + } +} + +/// Subscribe to CL head events via SSE. +pub fn subscribe_cl_events(base_url: &str) -> Result { + let url = build_events_url(base_url)?; + + let client = eventsource_client::ClientBuilder::for_url(url.as_str()) + .map_err(|e| Error::Config(format!("Invalid SSE URL: {}", e)))? + .build(); + + Ok(ClEventStream { + client: Box::pin(client.stream()), + }) +} + +fn build_events_url(base_url: &str) -> Result { + let base = Url::parse(base_url)?; + Ok(base.join("/eth/v1/events?topics=head,block")?) +} + +#[cfg(test)] +mod tests { + use super::build_events_url; + + #[test] + fn build_events_url_adds_path_without_trailing_slash() { + let url = build_events_url("http://localhost:5052").unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:5052/eth/v1/events?topics=head,block" + ); + } + + #[test] + fn build_events_url_adds_path_with_trailing_slash() { + let url = build_events_url("http://localhost:5052/").unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:5052/eth/v1/events?topics=head,block" + ); + } +} diff --git a/execution-witness-sentry/src/config.rs b/execution-witness-sentry/src/config.rs new file mode 100644 index 00000000000..50d64969e72 --- /dev/null +++ b/execution-witness-sentry/src/config.rs @@ -0,0 +1,58 @@ +//! Configuration types for the execution witness sentry. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::error::{Error, Result}; + +/// Sentry configuration. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Config { + /// Execution layer endpoints to monitor. + pub endpoints: Vec, + /// Consensus layer endpoints to submit proofs to. + pub cl_endpoints: Option>, + /// Directory to save block and witness data. + pub output_dir: Option, + /// Chain identifier (used in output path). + pub chain: Option, + /// Number of recent blocks to retain (older blocks are deleted). + pub retain: Option, + /// Number of proofs to submit per block. + pub num_proofs: Option, +} + +/// Execution layer endpoint configuration. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Endpoint { + /// Human-readable name for this endpoint. + pub name: String, + /// HTTP JSON-RPC URL. + pub el_url: String, + /// WebSocket URL for subscriptions. + pub el_ws_url: String, +} + +/// Consensus layer endpoint configuration. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ClEndpoint { + /// Human-readable name for this endpoint. + pub name: String, + /// HTTP API URL. + pub url: String, +} + +impl Config { + /// Load configuration from a TOML file. + pub fn load(path: impl AsRef) -> Result { + let content = std::fs::read_to_string(path.as_ref()).map_err(|e| { + Error::Config(format!( + "failed to read config file '{}': {}", + path.as_ref().display(), + e + )) + })?; + Ok(toml::from_str(&content)?) + } +} diff --git a/execution-witness-sentry/src/error.rs b/execution-witness-sentry/src/error.rs new file mode 100644 index 00000000000..91f0a7b72e1 --- /dev/null +++ b/execution-witness-sentry/src/error.rs @@ -0,0 +1,53 @@ +//! Error types for the execution witness sentry. + +use std::io; + +use thiserror::Error; + +/// Errors that can occur in the execution witness sentry. +#[derive(Debug, Error)] +pub enum Error { + /// Failed to load or parse configuration. + #[error("config error: {0}")] + Config(String), + + /// HTTP request failed. + #[error("HTTP error: {0}")] + Http(#[from] reqwest::Error), + + /// JSON-RPC error returned by the node. + #[error("RPC error {code}: {message}")] + Rpc { + /// Error code. + code: i64, + /// Error message. + message: String, + }, + + /// Failed to parse response. + #[error("parse error: {0}")] + Parse(#[from] serde_json::Error), + + /// WebSocket connection or subscription failed. + #[error("WebSocket error: {0}")] + WebSocket(String), + + /// URL parsing failed. + #[error("invalid URL: {0}")] + InvalidUrl(#[from] url::ParseError), + + /// I/O error (file operations, compression). + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + /// TOML parsing error. + #[error("TOML parse error: {0}")] + Toml(#[from] toml::de::Error), + + /// SSE connection error. + #[error("SSE error: {0}")] + Sse(String), +} + +/// Result type alias using our Error type. +pub type Result = std::result::Result; diff --git a/execution-witness-sentry/src/lib.rs b/execution-witness-sentry/src/lib.rs new file mode 100644 index 00000000000..fcb9b077cc8 --- /dev/null +++ b/execution-witness-sentry/src/lib.rs @@ -0,0 +1,45 @@ +//! Execution witness sentry - monitors execution layer nodes for new blocks +//! and fetches their execution witnesses. +//! +//! This crate provides functionality to: +//! - Subscribe to new block headers via WebSocket +//! - Fetch blocks and execution witnesses via JSON-RPC +//! - Store block data and witnesses to disk +//! - Submit execution proofs to consensus layer nodes +//! +//! ## Example +//! +//! ```ignore +//! use execution_witness_sentry::{Config, ElClient, subscribe_blocks}; +//! +//! let config = Config::load("config.toml")?; +//! let client = ElClient::new(url); +//! +//! // Subscribe to new blocks +//! let mut stream = subscribe_blocks(&ws_url).await?; +//! +//! while let Some(header) = stream.next().await { +//! let witness = client.get_execution_witness(header.number).await?; +//! // Process witness... +//! } +//! ``` + +pub mod cl_subscription; +pub mod config; +pub mod error; +pub mod rpc; +pub mod storage; +pub mod subscription; + +// Re-export main types at crate root for convenience. +pub use cl_subscription::{BlockEvent, ClEvent, ClEventStream, HeadEvent, subscribe_cl_events}; +pub use config::{ClEndpoint, Config, Endpoint}; +pub use error::{Error, Result}; +pub use rpc::{BlockInfo, ClClient, ElClient, ExecutionProof, generate_random_proof}; +pub use storage::{ + BlockMetadata, BlockStorage, SavedProof, compress_gzip, decompress_gzip, load_block_data, +}; +pub use subscription::subscribe_blocks; + +// Re-export alloy types that appear in our public API. +pub use alloy_rpc_types_eth::{Block, Header}; diff --git a/execution-witness-sentry/src/main.rs b/execution-witness-sentry/src/main.rs new file mode 100644 index 00000000000..8170bcda024 --- /dev/null +++ b/execution-witness-sentry/src/main.rs @@ -0,0 +1,731 @@ +//! Execution witness sentry CLI. +//! +//! Monitors execution layer nodes for new blocks and fetches their execution witnesses. +//! Subscribes to CL head events to correlate EL blocks with beacon slots. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::pin::pin; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clap::Parser; +use futures::StreamExt; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; +use url::Url; + +use execution_witness_sentry::{ + BlockStorage, ClClient, ClEvent, Config, ElClient, ExecutionProof, SavedProof, + generate_random_proof, subscribe_blocks, subscribe_cl_events, +}; + +/// Execution witness sentry - monitors EL nodes and fetches witnesses. +#[derive(Parser, Debug)] +#[command(name = "execution-witness-sentry")] +#[command(about = "Monitor execution layer nodes and fetch execution witnesses")] +struct Cli { + /// Path to configuration file. + #[arg(long, short, default_value = "config.toml")] + config: PathBuf, +} + +/// Cached EL block data waiting for CL correlation. +struct CachedElBlock { + block_number: u64, + timestamp: Instant, +} + +/// Cache for EL blocks keyed by block_hash. +struct ElBlockCache { + blocks: HashMap, + max_age: Duration, +} + +impl ElBlockCache { + fn new(max_age: Duration) -> Self { + Self { + blocks: HashMap::new(), + max_age, + } + } + + fn insert(&mut self, block_hash: String, block_number: u64, _endpoint_name: String) { + self.blocks.insert( + block_hash, + CachedElBlock { + block_number, + timestamp: Instant::now(), + }, + ); + self.cleanup(); + } + + fn get(&self, block_hash: &str) -> Option<&CachedElBlock> { + self.blocks.get(block_hash) + } + + fn remove(&mut self, block_hash: &str) -> Option { + self.blocks.remove(block_hash) + } + + fn cleanup(&mut self) { + let now = Instant::now(); + self.blocks + .retain(|_, v| now.duration_since(v.timestamp) < self.max_age); + } +} + +/// EL event for the channel. +struct ElBlockEvent { + endpoint_name: String, + block_number: u64, + block_hash: String, +} + +/// CL event for the channel. +struct ClBlockEvent { + cl_name: String, + slot: u64, + block_root: String, + execution_block_hash: String, +} + +/// Status of a zkvm CL node. +#[derive(Debug, Clone)] +struct ZkvmClStatus { + name: String, + head_slot: u64, + gap: i64, // Negative means behind source CL +} + +/// Monitor zkvm CL nodes and report their sync status. +async fn monitor_zkvm_status( + source_client: &ClClient, + zkvm_clients: &[(String, ClClient)], +) -> Vec { + let source_head = match source_client.get_head_slot().await { + Ok(slot) => slot, + Err(e) => { + warn!(error = %e, "Failed to get source CL head"); + return vec![]; + } + }; + + let mut statuses = Vec::new(); + for (name, client) in zkvm_clients { + match client.get_head_slot().await { + Ok(head_slot) => { + let gap = head_slot as i64 - source_head as i64; + statuses.push(ZkvmClStatus { + name: name.clone(), + head_slot, + gap, + }); + } + Err(e) => { + warn!(name = %name, error = %e, "Failed to get zkvm CL head"); + } + } + } + + statuses +} + +/// Backfill proofs for a zkvm CL that is behind. +/// First tries to use saved proofs from disk, falls back to generating new ones. +/// Returns the number of proofs submitted. +async fn backfill_proofs( + source_client: &ClClient, + zkvm_client: &ClClient, + zkvm_name: &str, + num_proofs: usize, + max_slots: u64, + storage: Option<&BlockStorage>, +) -> usize { + // Get the zkvm CL's current head + let zkvm_head = match zkvm_client.get_head_slot().await { + Ok(slot) => slot, + Err(e) => { + warn!(name = %zkvm_name, error = %e, "Failed to get zkvm CL head for backfill"); + return 0; + } + }; + + // Get source CL head + let source_head = match source_client.get_head_slot().await { + Ok(slot) => slot, + Err(e) => { + warn!(error = %e, "Failed to get source CL head for backfill"); + return 0; + } + }; + + if zkvm_head >= source_head { + return 0; // Already caught up + } + + let gap = source_head - zkvm_head; + let slots_to_check = gap.min(max_slots); + + info!( + name = %zkvm_name, + zkvm_head = zkvm_head, + source_head = source_head, + gap = gap, + checking = slots_to_check, + "Backfilling proofs" + ); + + let mut proofs_submitted = 0; + + // Iterate through slots from zkvm_head + 1 to zkvm_head + slots_to_check + for slot in (zkvm_head + 1)..=(zkvm_head + slots_to_check) { + // First try to load saved proofs from disk + if let Some(storage) = storage + && let Ok(Some((_metadata, saved_proofs))) = storage.load_proofs_by_slot(slot) + && !saved_proofs.is_empty() + { + debug!( + slot = slot, + num_proofs = saved_proofs.len(), + "Using saved proofs from disk" + ); + + for saved_proof in &saved_proofs { + let proof = ExecutionProof { + proof_id: saved_proof.proof_id, + slot: saved_proof.slot, + block_hash: saved_proof.block_hash.clone(), + block_root: saved_proof.block_root.clone(), + proof_data: saved_proof.proof_data.clone(), + }; + + match zkvm_client.submit_execution_proof(&proof).await { + Ok(()) => { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = saved_proof.proof_id, + "Backfill proof submitted (from disk)" + ); + proofs_submitted += 1; + } + Err(e) => { + let msg = e.to_string(); + if !msg.contains("already known") { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = saved_proof.proof_id, + error = %e, + "Backfill proof failed" + ); + } + } + } + } + continue; // Move to next slot + } + + // No saved proofs, fetch block info and generate new proofs + let block_info = match source_client.get_block_info(slot).await { + Ok(Some(info)) => info, + Ok(None) => { + debug!(slot = slot, "Empty slot, skipping"); + continue; + } + Err(e) => { + debug!(slot = slot, error = %e, "Failed to get block info"); + continue; + } + }; + + // Only submit proofs for blocks with execution payloads + let Some(exec_hash) = block_info.execution_block_hash else { + debug!(slot = slot, "No execution payload, skipping"); + continue; + }; + + // Generate and submit proofs + for proof_id in 0..num_proofs { + let proof = ExecutionProof { + proof_id: proof_id as u8, + slot, + block_hash: exec_hash.clone(), + block_root: block_info.block_root.clone(), + proof_data: generate_random_proof(proof_id as u32), + }; + + match zkvm_client.submit_execution_proof(&proof).await { + Ok(()) => { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = proof_id, + "Backfill proof submitted (generated)" + ); + proofs_submitted += 1; + } + Err(e) => { + let msg = e.to_string(); + if !msg.contains("already known") { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = proof_id, + error = %e, + "Backfill proof failed" + ); + } + } + } + } + } + + if proofs_submitted > 0 { + info!( + name = %zkvm_name, + proofs_submitted = proofs_submitted, + "Backfill complete" + ); + } + + proofs_submitted +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("execution_witness_sentry=info".parse()?), + ) + .init(); + + let cli = Cli::parse(); + let config = Config::load(&cli.config)?; + + info!(endpoints = config.endpoints.len(), "Loaded configuration"); + for endpoint in &config.endpoints { + info!( + name = %endpoint.name, + el_url = %endpoint.el_url, + el_ws_url = %endpoint.el_ws_url, + "EL endpoint configured" + ); + } + + // Set up CL clients - separate zkvm targets from event sources + let mut zkvm_clients: Vec<(String, ClClient)> = Vec::new(); // zkvm-enabled nodes for proof submission + let mut event_source_client: Option<(String, String, ClClient)> = None; // First available CL for events + + if let Some(endpoints) = config.cl_endpoints.as_ref() { + for endpoint in endpoints { + let url = match Url::parse(&endpoint.url) { + Ok(u) => u, + Err(e) => { + warn!(name = %endpoint.name, error = %e, "Invalid CL endpoint URL"); + continue; + } + }; + let client = ClClient::new(url); + + match client.is_zkvm_enabled().await { + Ok(true) => { + info!(name = %endpoint.name, "CL endpoint has zkvm enabled (proof target)"); + zkvm_clients.push((endpoint.name.clone(), client)); + } + Ok(false) => { + info!(name = %endpoint.name, "CL endpoint does not have zkvm enabled"); + // Use first non-zkvm CL as event source + if event_source_client.is_none() { + info!(name = %endpoint.name, "Using as event source"); + event_source_client = + Some((endpoint.name.clone(), endpoint.url.clone(), client)); + } + } + Err(e) => { + warn!(name = %endpoint.name, error = %e, "Failed to check zkvm status"); + } + } + } + } + + info!( + zkvm_targets = zkvm_clients.len(), + "zkvm-enabled CL endpoints configured" + ); + + let Some(event_source) = event_source_client else { + error!("No non-zkvm CL endpoint available for event source"); + return Ok(()); + }; + info!(name = %event_source.0, "CL event source configured"); + + let num_proofs = config.num_proofs.unwrap_or(2) as usize; + + // Set up block storage + let storage = config.output_dir.as_ref().map(|dir| { + BlockStorage::new( + dir, + config.chain.as_deref().unwrap_or("unknown"), + config.retain, + ) + }); + + // Cache for EL blocks (keyed by block_hash) + let el_cache = Arc::new(Mutex::new(ElBlockCache::new(Duration::from_secs(60)))); + + // Channels for events + let (el_tx, mut el_rx) = tokio::sync::mpsc::channel::(100); + let (cl_tx, mut cl_rx) = tokio::sync::mpsc::channel::(100); + + // Spawn EL subscription tasks + for endpoint in config.endpoints.clone() { + let tx = el_tx.clone(); + let name = endpoint.name.clone(); + let ws_url = endpoint.el_ws_url.clone(); + + tokio::spawn(async move { + info!(name = %name, "Connecting to EL WebSocket"); + + let stream = match subscribe_blocks(&ws_url).await { + Ok(s) => s, + Err(e) => { + error!(name = %name, error = %e, "Failed to subscribe to EL"); + return; + } + }; + + info!(name = %name, "Subscribed to EL newHeads"); + let mut stream = pin!(stream); + + while let Some(result) = stream.next().await { + match result { + Ok(header) => { + let event = ElBlockEvent { + endpoint_name: name.clone(), + block_number: header.number, + block_hash: format!("{:?}", header.hash), + }; + if tx.send(event).await.is_err() { + break; + } + } + Err(e) => { + error!(name = %name, error = %e, "EL stream error"); + } + } + } + warn!(name = %name, "EL WebSocket stream ended"); + }); + } + + let (es_name, es_url, es_client) = event_source; + let source_client_for_monitor = es_client.clone(); + + // Spawn CL subscription task for the event source (non-zkvm CL) + { + let tx = cl_tx.clone(); + + tokio::spawn(async move { + info!(name = %es_name, "Connecting to CL SSE"); + + let stream = match subscribe_cl_events(&es_url) { + Ok(s) => s, + Err(e) => { + error!(name = %es_name, error = %e, "Failed to subscribe to CL events"); + return; + } + }; + + info!(name = %es_name, "Subscribed to CL head events"); + let mut stream = pin!(stream); + + while let Some(result) = stream.next().await { + match result { + Ok(ClEvent::Head(head)) => { + let slot: u64 = match head.slot.parse() { + Ok(slot) => slot, + Err(e) => { + warn!( + name = %es_name, + error = %e, + slot = %head.slot, + "Invalid head slot value" + ); + continue; + } + }; + let block_root = head.block.clone(); + + // Fetch the execution block hash for this beacon block + let exec_hash = match es_client.get_block_execution_hash(&block_root).await + { + Ok(Some(hash)) => hash, + Ok(None) => { + debug!(name = %es_name, slot = slot, "No execution hash for block"); + continue; + } + Err(e) => { + debug!(name = %es_name, error = %e, "Failed to get execution hash"); + continue; + } + }; + + let event = ClBlockEvent { + cl_name: es_name.clone(), + slot, + block_root, + execution_block_hash: exec_hash, + }; + if tx.send(event).await.is_err() { + break; + } + } + Ok(ClEvent::Block(_)) => { + // We use head events primarily + } + Err(e) => { + error!(name = %es_name, error = %e, "CL stream error"); + } + } + } + warn!(name = %es_name, "CL SSE stream ended"); + }); + } + + drop(el_tx); + drop(cl_tx); + + // Create a timer for periodic monitoring and backfill (500ms for fast catch-up) + let mut monitor_interval = tokio::time::interval(Duration::from_millis(500)); + monitor_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + info!("Waiting for events (with monitoring every 500ms)"); + + // Process events from both EL and CL + loop { + tokio::select! { + // Periodic monitoring and backfill + _ = monitor_interval.tick() => { + // Monitor zkvm CL status + let statuses = monitor_zkvm_status(&source_client_for_monitor, &zkvm_clients).await; + + for status in &statuses { + if status.gap < -5 { + // More than 5 slots behind - log warning and backfill + warn!( + name = %status.name, + head_slot = status.head_slot, + gap = status.gap, + "zkvm CL is behind, starting backfill" + ); + + // Find the client and backfill + if let Some((_, client)) = zkvm_clients.iter().find(|(n, _)| n == &status.name) { + backfill_proofs( + &source_client_for_monitor, + client, + &status.name, + num_proofs, + 20, // Max 20 slots per backfill cycle + storage.as_ref(), + ).await; + } + } else if status.gap < 0 { + // Slightly behind - just log + debug!( + name = %status.name, + head_slot = status.head_slot, + gap = status.gap, + "zkvm CL slightly behind" + ); + } else { + // In sync or ahead + debug!( + name = %status.name, + head_slot = status.head_slot, + gap = status.gap, + "zkvm CL in sync" + ); + } + } + } + + Some(el_event) = el_rx.recv() => { + info!( + name = %el_event.endpoint_name, + number = el_event.block_number, + hash = %el_event.block_hash, + "EL block received" + ); + + // Find the endpoint and fetch block + witness + let Some(endpoint) = config.endpoints.iter().find(|e| e.name == el_event.endpoint_name) else { + continue; + }; + + let Ok(el_url) = Url::parse(&endpoint.el_url) else { + continue; + }; + let el_client = ElClient::new(el_url); + + // Fetch block and witness + let (block, gzipped_block) = match el_client.get_block_by_hash(&el_event.block_hash).await { + Ok(Some(data)) => data, + Ok(None) => { + warn!(number = el_event.block_number, "Block not found"); + continue; + } + Err(e) => { + error!(number = el_event.block_number, error = %e, "Failed to fetch block"); + continue; + } + }; + + let (witness, gzipped_witness) = match el_client.get_execution_witness(el_event.block_number).await { + Ok(Some(data)) => data, + Ok(None) => { + warn!(number = el_event.block_number, "Witness not found"); + continue; + } + Err(e) => { + error!(number = el_event.block_number, error = %e, "Failed to fetch witness"); + continue; + } + }; + + info!( + number = el_event.block_number, + block_gzipped = gzipped_block.len(), + witness_gzipped = gzipped_witness.len(), + "Fetched block and witness" + ); + + // Save to disk if storage is configured + if let Some(ref storage) = storage { + let combined = serde_json::json!({ + "block": block, + "witness": witness, + }); + let combined_bytes = serde_json::to_vec(&combined)?; + let gzipped_combined = execution_witness_sentry::compress_gzip(&combined_bytes)?; + + if let Err(e) = storage.save_block(&block, &gzipped_combined) { + error!(error = %e, "Failed to save block"); + } else { + info!( + number = el_event.block_number, + separate = gzipped_block.len() + gzipped_witness.len(), + combined = gzipped_combined.len(), + "Saved" + ); + } + } + + // Cache the EL block for correlation with CL events + let mut cache = el_cache.lock().await; + cache.insert( + el_event.block_hash.clone(), + el_event.block_number, + el_event.endpoint_name.clone(), + ); + } + + Some(cl_event) = cl_rx.recv() => { + info!( + source = %cl_event.cl_name, + slot = cl_event.slot, + block_root = %cl_event.block_root, + exec_hash = %cl_event.execution_block_hash, + "CL head event received" + ); + + // Check if we have the EL block cached + let cached_block_number = { + let cache = el_cache.lock().await; + cache.get(&cl_event.execution_block_hash).map(|c| c.block_number) + }; + + if cached_block_number.is_none() { + debug!( + exec_hash = %cl_event.execution_block_hash, + "EL block not in cache, skipping proof submission" + ); + continue; + } + let block_number = cached_block_number.unwrap(); + + // Generate proofs once (for all CLs and for saving) + let mut generated_proofs: Vec = Vec::new(); + for proof_id in 0..num_proofs { + generated_proofs.push(SavedProof { + proof_id: proof_id as u8, + slot: cl_event.slot, + block_hash: cl_event.execution_block_hash.clone(), + block_root: cl_event.block_root.clone(), + proof_data: generate_random_proof(proof_id as u32), + }); + } + + // Save proofs to disk for backfill + if let Some(ref storage) = storage { + if let Err(e) = storage.save_proofs( + block_number, + cl_event.slot, + &cl_event.block_root, + &cl_event.execution_block_hash, + &generated_proofs, + ) { + warn!(slot = cl_event.slot, error = %e, "Failed to save proofs to disk"); + } else { + debug!(slot = cl_event.slot, block_number = block_number, "Saved proofs to disk"); + } + } + + // Submit proofs to ALL zkvm-enabled CL clients + for (cl_name, cl_client) in &zkvm_clients { + for saved_proof in &generated_proofs { + let proof = ExecutionProof { + proof_id: saved_proof.proof_id, + slot: saved_proof.slot, + block_hash: saved_proof.block_hash.clone(), + block_root: saved_proof.block_root.clone(), + proof_data: saved_proof.proof_data.clone(), + }; + + match cl_client.submit_execution_proof(&proof).await { + Ok(()) => { + info!( + cl = %cl_name, + slot = cl_event.slot, + proof_id = saved_proof.proof_id, + "Proof submitted" + ); + } + Err(e) => { + debug!( + cl = %cl_name, + slot = cl_event.slot, + proof_id = saved_proof.proof_id, + error = %e, + "Proof submission failed" + ); + } + } + } + } + + // Remove from cache after submission + let mut cache = el_cache.lock().await; + cache.remove(&cl_event.execution_block_hash); + } + + else => break, + } + } + + Ok(()) +} diff --git a/execution-witness-sentry/src/rpc.rs b/execution-witness-sentry/src/rpc.rs new file mode 100644 index 00000000000..9722ee67085 --- /dev/null +++ b/execution-witness-sentry/src/rpc.rs @@ -0,0 +1,393 @@ +//! JSON-RPC client for execution layer nodes. + +use alloy_rpc_types_eth::Block; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::error::{Error, Result}; +use crate::storage::compress_gzip; + +/// JSON-RPC request structure. +#[derive(Debug, Clone, Serialize)] +struct JsonRpcRequest { + jsonrpc: &'static str, + method: &'static str, + params: T, + id: u64, +} + +/// JSON-RPC response structure. +#[derive(Debug, Clone, Deserialize)] +pub struct JsonRpcResponse { + pub result: Option, + pub error: Option, +} + +/// JSON-RPC error structure. +#[derive(Debug, Clone, Deserialize)] +pub struct JsonRpcError { + pub code: i64, + pub message: String, +} + +/// Execution layer JSON-RPC client. +pub struct ElClient { + url: Url, + http_client: reqwest::Client, +} + +impl ElClient { + /// Create a new EL client. + pub fn new(url: Url) -> Self { + Self { + url, + http_client: reqwest::Client::new(), + } + } + + /// Fetch a block by hash. Returns the block and its gzipped JSON. + pub async fn get_block_by_hash(&self, block_hash: &str) -> Result)>> { + let request = JsonRpcRequest { + jsonrpc: "2.0", + method: "eth_getBlockByHash", + params: (block_hash, false), + id: 1, + }; + + let response = self + .http_client + .post(self.url.clone()) + .json(&request) + .send() + .await?; + + if !response.status().is_success() { + return Err(Error::Rpc { + code: response.status().as_u16() as i64, + message: response.text().await.unwrap_or_default(), + }); + } + + let rpc_response: JsonRpcResponse = response.json().await?; + + if let Some(error) = rpc_response.error { + return Err(Error::Rpc { + code: error.code, + message: error.message, + }); + } + + match rpc_response.result { + Some(block) => { + let json_bytes = serde_json::to_vec(&block)?; + let gzipped = compress_gzip(&json_bytes)?; + Ok(Some((block, gzipped))) + } + None => Ok(None), + } + } + + /// Fetch execution witness for a block. Returns the witness and its gzipped JSON. + pub async fn get_execution_witness( + &self, + block_number: u64, + ) -> Result)>> { + let block_num_hex = format!("0x{:x}", block_number); + let request = JsonRpcRequest { + jsonrpc: "2.0", + method: "debug_executionWitness", + params: (block_num_hex,), + id: 1, + }; + + let response = self + .http_client + .post(self.url.clone()) + .json(&request) + .send() + .await?; + + if !response.status().is_success() { + return Err(Error::Rpc { + code: response.status().as_u16() as i64, + message: response.text().await.unwrap_or_default(), + }); + } + + let rpc_response: JsonRpcResponse = response.json().await?; + + if let Some(error) = rpc_response.error { + return Err(Error::Rpc { + code: error.code, + message: error.message, + }); + } + + match rpc_response.result { + Some(witness) => { + let json_bytes = serde_json::to_vec(&witness)?; + let gzipped = compress_gzip(&json_bytes)?; + Ok(Some((witness, gzipped))) + } + None => Ok(None), + } + } +} + +/// Execution proof to submit to CL nodes. +#[derive(Debug, Clone, Serialize)] +pub struct ExecutionProof { + pub proof_id: u8, + pub slot: u64, + pub block_hash: String, + pub block_root: String, + pub proof_data: Vec, +} + +/// Consensus layer HTTP API client. +#[derive(Clone)] +pub struct ClClient { + url: Url, + http_client: reqwest::Client, +} + +/// Block response with execution payload. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockResponse { + pub data: BlockData, +} + +/// Block data. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockData { + pub message: BlockMessage, +} + +/// Block message. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockMessage { + pub body: BlockBody, +} + +/// Block body. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockBody { + pub execution_payload: Option, +} + +/// Execution payload (minimal fields). +#[derive(Debug, Clone, Deserialize)] +pub struct ExecutionPayload { + pub block_hash: String, +} + +/// Syncing status response. +#[derive(Debug, Clone, Deserialize)] +pub struct SyncingResponse { + pub data: SyncingData, +} + +/// Syncing status data. +#[derive(Debug, Clone, Deserialize)] +pub struct SyncingData { + pub head_slot: String, + pub is_syncing: bool, + pub is_optimistic: Option, +} + +/// Block header response. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockHeaderResponse { + pub data: BlockHeaderData, +} + +/// Block header data. +#[derive(Debug, Clone, Deserialize)] +pub struct BlockHeaderData { + pub root: String, +} + +/// Node identity response. +#[derive(Debug, Clone, Deserialize)] +pub struct IdentityResponse { + pub data: IdentityData, +} + +/// Node identity data. +#[derive(Debug, Clone, Deserialize)] +pub struct IdentityData { + pub enr: String, +} + +impl ClClient { + /// Create a new CL client. + pub fn new(url: Url) -> Self { + Self { + url, + http_client: reqwest::Client::new(), + } + } + + /// Get node syncing status. + pub async fn get_syncing(&self) -> Result { + let url = self.url.join("eth/v1/node/syncing")?; + let response = self.http_client.get(url).send().await?; + Ok(response.json().await?) + } + + /// Get block header for a slot. + pub async fn get_block_header(&self, slot: u64) -> Result> { + let url = self.url.join(&format!("eth/v1/beacon/headers/{}", slot))?; + let response = self.http_client.get(url).send().await?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + + Ok(Some(response.json().await?)) + } + + /// Submit an execution proof. + pub async fn submit_execution_proof(&self, proof: &ExecutionProof) -> Result<()> { + let url = self.url.join("eth/v1/beacon/pool/execution_proofs")?; + + let response = self.http_client.post(url).json(proof).send().await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::Rpc { + code: status.as_u16() as i64, + message: body, + }); + } + + Ok(()) + } + + /// Get node identity (including ENR). + pub async fn get_identity(&self) -> Result { + let url = self.url.join("eth/v1/node/identity")?; + let response = self.http_client.get(url).send().await?; + Ok(response.json().await?) + } + + /// Check if the node has zkvm enabled by inspecting its ENR. + pub async fn is_zkvm_enabled(&self) -> Result { + let identity = self.get_identity().await?; + Ok(enr_has_zkvm(&identity.data.enr)) + } + + /// Get the execution block hash for a beacon block. + pub async fn get_block_execution_hash(&self, block_root: &str) -> Result> { + let url = self + .url + .join(&format!("eth/v2/beacon/blocks/{}", block_root))?; + let response = self.http_client.get(url).send().await?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + + let block_response: BlockResponse = response.json().await?; + Ok(block_response + .data + .message + .body + .execution_payload + .map(|p| p.block_hash)) + } + + /// Get the current head slot. + pub async fn get_head_slot(&self) -> Result { + let syncing = self.get_syncing().await?; + syncing + .data + .head_slot + .parse() + .map_err(|e| Error::Config(format!("Invalid head slot: {}", e))) + } + + /// Get block info (slot, block_root, execution_block_hash) for a given slot. + /// Returns None if the slot is empty (no block). + pub async fn get_block_info(&self, slot: u64) -> Result> { + let url = self.url.join(&format!("eth/v2/beacon/blocks/{}", slot))?; + let response = self.http_client.get(url).send().await?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + + if !response.status().is_success() { + return Err(Error::Rpc { + code: response.status().as_u16() as i64, + message: response.text().await.unwrap_or_default(), + }); + } + + let block_response: BlockResponse = response.json().await?; + let execution_block_hash = block_response + .data + .message + .body + .execution_payload + .map(|p| p.block_hash); + + // Get the block root from headers endpoint + let header_url = self.url.join(&format!("eth/v1/beacon/headers/{}", slot))?; + let header_response = self.http_client.get(header_url).send().await?; + + if header_response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + + let header: BlockHeaderResponse = header_response.json().await?; + + Ok(Some(BlockInfo { + slot, + block_root: header.data.root, + execution_block_hash, + })) + } +} + +/// Block info for backfill. +#[derive(Debug, Clone)] +pub struct BlockInfo { + pub slot: u64, + pub block_root: String, + pub execution_block_hash: Option, +} + +/// The ENR field specifying whether zkVM execution proofs are enabled. +const ZKVM_ENABLED_ENR_KEY: &str = "zkvm"; + +/// Check if an ENR string contains the zkvm flag. +fn enr_has_zkvm(enr_str: &str) -> bool { + use discv5::enr::{CombinedKey, Enr}; + use std::str::FromStr; + + match Enr::::from_str(enr_str) { + Ok(enr) => enr + .get_decodable::(ZKVM_ENABLED_ENR_KEY) + .and_then(|result| result.ok()) + .unwrap_or(false), + Err(_) => false, + } +} + +/// Generate random proof bytes. +pub fn generate_random_proof(proof_id: u32) -> Vec { + use std::time::{SystemTime, UNIX_EPOCH}; + let seed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + + let mut proof = vec![0u8; 32]; + for (i, byte) in proof.iter_mut().enumerate() { + *byte = ((seed >> (i % 8)) ^ (i as u64)) as u8; + } + proof[31] = proof_id as u8; + proof +} diff --git a/execution-witness-sentry/src/storage.rs b/execution-witness-sentry/src/storage.rs new file mode 100644 index 00000000000..58a80d449d4 --- /dev/null +++ b/execution-witness-sentry/src/storage.rs @@ -0,0 +1,248 @@ +//! Block data storage utilities. + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use alloy_rpc_types_eth::Block; +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +/// Metadata stored alongside block data. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockMetadata { + /// EL block hash + pub block_hash: String, + /// EL block number + pub block_number: u64, + /// Gas used in the block + pub gas_used: u64, + /// CL slot number (if known) + #[serde(skip_serializing_if = "Option::is_none")] + pub slot: Option, + /// CL beacon block root (if known) + #[serde(skip_serializing_if = "Option::is_none")] + pub block_root: Option, + /// Number of proofs stored + #[serde(default)] + pub num_proofs: usize, +} + +/// A saved proof that can be loaded for backfill. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SavedProof { + pub proof_id: u8, + pub slot: u64, + pub block_hash: String, + pub block_root: String, + pub proof_data: Vec, +} + +/// Compress data using gzip. +pub fn compress_gzip(data: &[u8]) -> Result> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data)?; + Ok(encoder.finish()?) +} + +/// Decompress gzip data. +pub fn decompress_gzip(data: &[u8]) -> Result> { + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(decompressed) +} + +/// Load block data from a gzipped JSON file. +pub fn load_block_data(path: impl AsRef) -> Result { + let compressed = std::fs::read(path)?; + let decompressed = decompress_gzip(&compressed)?; + Ok(serde_json::from_slice(&decompressed)?) +} + +/// Manages block data storage on disk. +pub struct BlockStorage { + output_dir: PathBuf, + chain: String, + retain: Option, +} + +impl BlockStorage { + /// Create a new block storage manager. + pub fn new( + output_dir: impl Into, + chain: impl Into, + retain: Option, + ) -> Self { + Self { + output_dir: output_dir.into(), + chain: chain.into(), + retain, + } + } + + /// Get the directory path for a block number. + pub fn block_dir(&self, block_number: u64) -> PathBuf { + self.output_dir + .join(&self.chain) + .join(block_number.to_string()) + } + + /// Save block data to disk (without CL info - will be updated later). + pub fn save_block(&self, block: &Block, combined_data: &[u8]) -> Result<()> { + let block_number = block.header.number; + let block_hash = format!("{:?}", block.header.hash); + let gas_used = block.header.gas_used; + + let block_dir = self.block_dir(block_number); + std::fs::create_dir_all(&block_dir)?; + + // Write metadata (without CL info initially) + let metadata = BlockMetadata { + block_hash, + block_number, + gas_used, + slot: None, + block_root: None, + num_proofs: 0, + }; + let metadata_path = block_dir.join("metadata.json"); + std::fs::write(metadata_path, serde_json::to_string_pretty(&metadata)?)?; + + // Write combined block + witness data + let data_path = block_dir.join("data.json.gz"); + std::fs::write(data_path, combined_data)?; + + // Clean up old blocks if retention is configured + if let Some(retain) = self.retain + && block_number > retain + { + self.delete_old_block(block_number - retain)?; + } + + Ok(()) + } + + /// Save proofs and update metadata with CL info. + /// This is called when we receive CL head event with slot/block_root. + pub fn save_proofs( + &self, + block_number: u64, + slot: u64, + block_root: &str, + block_hash: &str, + proofs: &[SavedProof], + ) -> Result<()> { + let block_dir = self.block_dir(block_number); + + // Create dir if it doesn't exist (in case block wasn't saved yet) + std::fs::create_dir_all(&block_dir)?; + + // Load existing metadata or create new + let metadata_path = block_dir.join("metadata.json"); + let mut metadata = if metadata_path.exists() { + let content = std::fs::read_to_string(&metadata_path)?; + serde_json::from_str(&content)? + } else { + BlockMetadata { + block_hash: block_hash.to_string(), + block_number, + gas_used: 0, + slot: None, + block_root: None, + num_proofs: 0, + } + }; + + // Update with CL info + metadata.slot = Some(slot); + metadata.block_root = Some(block_root.to_string()); + metadata.num_proofs = proofs.len(); + + // Save updated metadata + std::fs::write(&metadata_path, serde_json::to_string_pretty(&metadata)?)?; + + // Save proofs + let proofs_path = block_dir.join("proofs.json"); + std::fs::write(&proofs_path, serde_json::to_string_pretty(&proofs)?)?; + + Ok(()) + } + + /// Load proofs for a given slot. + /// Searches for a block directory that has matching slot in metadata. + pub fn load_proofs_by_slot( + &self, + slot: u64, + ) -> Result)>> { + let chain_dir = self.output_dir.join(&self.chain); + if !chain_dir.exists() { + return Ok(None); + } + + // Iterate through block directories to find one with matching slot + for entry in std::fs::read_dir(&chain_dir)? { + let entry = entry?; + let block_dir = entry.path(); + + if !block_dir.is_dir() { + continue; + } + + let metadata_path = block_dir.join("metadata.json"); + if !metadata_path.exists() { + continue; + } + + let content = std::fs::read_to_string(&metadata_path)?; + let metadata: BlockMetadata = match serde_json::from_str(&content) { + Ok(m) => m, + Err(_) => continue, + }; + + if metadata.slot == Some(slot) { + // Found matching slot, load proofs + let proofs_path = block_dir.join("proofs.json"); + if proofs_path.exists() { + let proofs_content = std::fs::read_to_string(&proofs_path)?; + let proofs: Vec = serde_json::from_str(&proofs_content)?; + return Ok(Some((metadata, proofs))); + } else { + return Ok(Some((metadata, vec![]))); + } + } + } + + Ok(None) + } + + /// Load metadata for a given block number. + pub fn load_metadata(&self, block_number: u64) -> Result> { + let block_dir = self.block_dir(block_number); + let metadata_path = block_dir.join("metadata.json"); + + if !metadata_path.exists() { + return Ok(None); + } + + let content = std::fs::read_to_string(&metadata_path)?; + Ok(Some(serde_json::from_str(&content)?)) + } + + /// Delete an old block directory. + fn delete_old_block(&self, block_number: u64) -> Result<()> { + let old_dir = self.block_dir(block_number); + if old_dir.exists() { + std::fs::remove_dir_all(old_dir)?; + } + Ok(()) + } + + /// Get the chain directory path. + pub fn chain_dir(&self) -> PathBuf { + self.output_dir.join(&self.chain) + } +} diff --git a/execution-witness-sentry/src/subscription.rs b/execution-witness-sentry/src/subscription.rs new file mode 100644 index 00000000000..3882561590b --- /dev/null +++ b/execution-witness-sentry/src/subscription.rs @@ -0,0 +1,45 @@ +//! WebSocket subscription for new block headers. + +use std::pin::Pin; +use std::task::{Context, Poll}; + +use alloy_provider::{Provider, ProviderBuilder, WsConnect}; +use alloy_rpc_types_eth::Header; +use futures::Stream; + +use crate::error::{Error, Result}; + +/// Subscription stream that keeps the provider alive. +pub struct BlockSubscription

{ + #[allow(dead_code)] + provider: P, + stream: Pin + Send>>, +} + +impl

Unpin for BlockSubscription

{} + +impl Stream for BlockSubscription

{ + type Item = Result

; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.stream.as_mut().poll_next(cx).map(|opt| opt.map(Ok)) + } +} + +/// Subscribe to new block headers via WebSocket. +pub async fn subscribe_blocks(ws_url: &str) -> Result> + Send> { + let ws = WsConnect::new(ws_url); + let provider = ProviderBuilder::new() + .connect_ws(ws) + .await + .map_err(|e| Error::WebSocket(format!("WebSocket connection failed: {}", e)))?; + + let subscription = provider + .subscribe_blocks() + .await + .map_err(|e| Error::WebSocket(format!("Block subscription failed: {}", e)))?; + + let stream = Box::pin(subscription.into_stream()); + + Ok(BlockSubscription { provider, stream }) +} diff --git a/execution-witness-sentry/tests/cl_subscription.rs b/execution-witness-sentry/tests/cl_subscription.rs new file mode 100644 index 00000000000..40d86890eb0 --- /dev/null +++ b/execution-witness-sentry/tests/cl_subscription.rs @@ -0,0 +1,11 @@ +use execution_witness_sentry::subscribe_cl_events; + +#[test] +fn subscribe_cl_events_accepts_base_url_without_trailing_slash() { + assert!(subscribe_cl_events("http://localhost:5052").is_ok()); +} + +#[test] +fn subscribe_cl_events_accepts_base_url_with_trailing_slash() { + assert!(subscribe_cl_events("http://localhost:5052/").is_ok()); +} diff --git a/scripts/local_testnet/network_params.yaml b/scripts/local_testnet/network_params.yaml index a048674e630..79ff7df07b6 100644 --- a/scripts/local_testnet/network_params.yaml +++ b/scripts/local_testnet/network_params.yaml @@ -21,11 +21,7 @@ network_params: seconds_per_slot: 6 snooper_enabled: false global_log_level: debug -additional_services: - - dora - - spamoor - - prometheus_grafana - - tempo +additional_services: [] spamoor_params: image: ethpandaops/spamoor:master spammers: @@ -34,4 +30,4 @@ spamoor_params: throughput: 200 - scenario: blobs config: - throughput: 20 \ No newline at end of file + throughput: 20 diff --git a/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml b/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml index 11439e6d0eb..b7d7e7f62ea 100644 --- a/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml +++ b/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml @@ -1,25 +1,23 @@ -# Mixed configuration: 3 normal nodes, 1 node with dummy EL +# Mixed configuration: 3 normal nodes with reth, 3 zkvm nodes with dummy_el participants: - # Nodes with real execution layer (nodes 1-3) - - el_type: geth - el_image: ethereum/client-go:latest + # Normal nodes with real EL (nodes 1-3) + - el_type: reth + el_image: ghcr.io/paradigmxyz/reth:latest cl_type: lighthouse cl_image: lighthouse:local cl_extra_params: - - --activate-zkvm - - --target-peers=3 + - --target-peers=5 count: 3 - # Node with dummy execution layer (node 4) - # TODO(zkproofs): Currently there is no way to add no client here - # We likely want to use our dummy zkvm EL here + # ZKVM nodes with dummy EL (nodes 4-6) + # Uses dummy_el wrapped as geth - returns SYNCING for all engine calls - el_type: geth el_image: dummy_el:local cl_type: lighthouse cl_image: lighthouse:local cl_extra_params: - --activate-zkvm - - --target-peers=3 - count: 1 + - --target-peers=5 + count: 3 network_params: electra_fork_epoch: 0 fulu_fork_epoch: 1 @@ -29,3 +27,10 @@ snooper_enabled: false additional_services: - dora - prometheus_grafana +port_publisher: + el: + enabled: true + public_port_start: 32000 + cl: + enabled: true + public_port_start: 33000 \ No newline at end of file diff --git a/scripts/local_testnet/network_params_simple.yaml b/scripts/local_testnet/network_params_simple.yaml new file mode 100644 index 00000000000..3f2ca40f371 --- /dev/null +++ b/scripts/local_testnet/network_params_simple.yaml @@ -0,0 +1,19 @@ +# Simple testnet config for testing EL listener with 2 nodes using Reth +participants: + - cl_type: lighthouse + cl_image: lighthouse:local + el_type: reth + count: 2 +network_params: + electra_fork_epoch: 0 + seconds_per_slot: 6 +snooper_enabled: false +global_log_level: info +additional_services: [] +port_publisher: + el: + enabled: true + public_port_start: 32000 + cl: + enabled: true + public_port_start: 33000 \ No newline at end of file