diff --git a/Cargo.lock b/Cargo.lock index 24c5db3f21a..b10d62ef57c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -742,7 +742,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -753,7 +753,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3220,7 +3220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4931,7 +4931,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6524,7 +6524,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7835,9 +7835,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.0" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68df0380e5c9d20ce49534f292a36a7514ae21350726efe1865bdb1fa91d278" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" dependencies = [ "alloy-rlp", "arbitrary", @@ -7981,7 +7981,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9139,7 +9139,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10595,7 +10595,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8861d159374..19158eb29e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ members = [ "beacon_node/store", "beacon_node/timer", "boot_node", - "execution-witness-sentry", "common/account_utils", "common/clap_utils", "common/deposit_contract", @@ -63,6 +62,7 @@ members = [ "crypto/kzg", "database_manager", "dummy_el", + "execution-witness-sentry", "lcli", "lighthouse", "lighthouse/environment", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index dfd28cd9572..d6990894018 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4149,6 +4149,24 @@ impl BeaconChain { // This prevents inconsistency between the two at the expense of concurrency. drop(fork_choice); + // Persist execution proofs to the database if zkvm is enabled and proofs are cached. + // This is done after the block is successfully stored so we don't lose proofs on cache eviction. + if let Some(proofs) = self + .data_availability_checker + .get_execution_proofs(&block_root) + && !proofs.is_empty() + { + let proofs_owned: Vec<_> = proofs.iter().map(|p| (**p).clone()).collect(); + if let Err(e) = self.store.put_execution_proofs(&block_root, &proofs_owned) { + // Log but don't fail block import - proofs can still be served from cache + warn!( + %block_root, + error = ?e, + "Failed to persist execution proofs to database" + ); + } + } + // We're declaring the block "imported" at this point, since fork choice and the DB know // about it. let block_time_imported = timestamp_now(); diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index bc5b41b09e1..feabcd5f44a 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1159,6 +1159,13 @@ where .process_prune_blobs(data_availability_boundary); } + // Prune execution proofs older than the execution proof boundary in the background. + if let Some(execution_proof_boundary) = beacon_chain.execution_proof_boundary() { + beacon_chain + .store_migrator + .process_prune_execution_proofs(execution_proof_boundary); + } + Ok(beacon_chain) } } diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 228e5eb2d27..17dc227430b 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1034,6 +1034,12 @@ impl BeaconChain { .process_prune_blobs(data_availability_boundary); } + // Prune execution proofs in the background. + if let Some(execution_proof_boundary) = self.execution_proof_boundary() { + self.store_migrator + .process_prune_execution_proofs(execution_proof_boundary); + } + // Take a write-lock on the canonical head and signal for it to prune. self.canonical_head.fork_choice_write_lock().prune()?; diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index bd232f2e8a2..e290cf510f0 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -120,6 +120,7 @@ pub enum Notification { Finalization(FinalizationNotification), Reconstruction, PruneBlobs(Epoch), + PruneExecutionProofs(Epoch), ManualFinalization(ManualFinalizationNotification), ManualCompaction, } @@ -251,6 +252,28 @@ impl, Cold: ItemStore> BackgroundMigrator>, + execution_proof_boundary: Epoch, + ) { + if let Err(e) = db.try_prune_execution_proofs(false, execution_proof_boundary) { + error!( + error = ?e, + "Execution proof pruning failed" + ); + } + } + /// If configured to run in the background, send `notif` to the background thread. /// /// Return `None` if the message was sent to the background thread, `Some(notif)` otherwise. @@ -440,11 +463,15 @@ impl, Cold: ItemStore> BackgroundMigrator reconstruction_notif = Some(notif), Notification::Finalization(fin) => finalization_notif = Some(fin), Notification::ManualFinalization(fin) => manual_finalization_notif = Some(fin), Notification::PruneBlobs(dab) => prune_blobs_notif = Some(dab), + Notification::PruneExecutionProofs(epb) => { + prune_execution_proofs_notif = Some(epb) + } Notification::ManualCompaction => manual_compaction_notif = Some(notif), } // Read the rest of the messages in the channel, taking the best of each type. @@ -475,6 +502,10 @@ impl, Cold: ItemStore> BackgroundMigrator { prune_blobs_notif = std::cmp::max(prune_blobs_notif, Some(dab)); } + Notification::PruneExecutionProofs(epb) => { + prune_execution_proofs_notif = + std::cmp::max(prune_execution_proofs_notif, Some(epb)); + } } } // Run finalization and blob pruning migrations first, then a reconstruction batch. @@ -489,6 +520,9 @@ impl, Cold: ItemStore> BackgroundMigrator = DBColumn::iter().map(|c| c.as_str()).collect(); let expected_columns = vec![ - "bma", "blk", "blb", "bdc", "bdi", "ste", "hsd", "hsn", "bsn", "bsd", "bss", "bs3", "bcs", - "bst", "exp", "bch", "opo", "etc", "frk", "pkc", "brp", "bsx", "bsr", "bbx", "bbr", "bhr", - "brm", "dht", "cus", "otb", "bhs", "olc", "lcu", "scb", "scm", "dmy", + "bma", "blk", "blb", "bdc", "bdi", "bep", "ste", "hsd", "hsn", "bsn", "bsd", "bss", "bs3", + "bcs", "bst", "exp", "bch", "opo", "etc", "frk", "pkc", "brp", "bsx", "bsr", "bbx", "bbr", + "bhr", "brm", "dht", "cus", "otb", "bhs", "olc", "lcu", "scb", "scm", "dmy", ]; assert_eq!(expected_columns, current_columns); } diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 6f5170be300..f98d57e5cb6 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -129,6 +129,7 @@ pub struct BeaconProcessorQueueLengths { block_broots_queue: usize, blob_broots_queue: usize, execution_proof_broots_queue: usize, + execution_proof_brange_queue: usize, blob_brange_queue: usize, dcbroots_queue: usize, dcbrange_queue: usize, @@ -198,6 +199,7 @@ impl BeaconProcessorQueueLengths { block_broots_queue: 1024, blob_broots_queue: 1024, execution_proof_broots_queue: 1024, + execution_proof_brange_queue: 1024, blob_brange_queue: 1024, dcbroots_queue: 1024, dcbrange_queue: 1024, @@ -620,6 +622,7 @@ pub enum Work { BlobsByRangeRequest(BlockingFn), BlobsByRootsRequest(BlockingFn), ExecutionProofsByRootsRequest(BlockingFn), + ExecutionProofsByRangeRequest(BlockingFn), DataColumnsByRootsRequest(BlockingFn), DataColumnsByRangeRequest(BlockingFn), GossipBlsToExecutionChange(BlockingFn), @@ -675,6 +678,7 @@ pub enum WorkType { BlobsByRangeRequest, BlobsByRootsRequest, ExecutionProofsByRootsRequest, + ExecutionProofsByRangeRequest, DataColumnsByRootsRequest, DataColumnsByRangeRequest, GossipBlsToExecutionChange, @@ -728,6 +732,7 @@ impl Work { Work::BlobsByRangeRequest(_) => WorkType::BlobsByRangeRequest, Work::BlobsByRootsRequest(_) => WorkType::BlobsByRootsRequest, Work::ExecutionProofsByRootsRequest(_) => WorkType::ExecutionProofsByRootsRequest, + Work::ExecutionProofsByRangeRequest(_) => WorkType::ExecutionProofsByRangeRequest, Work::DataColumnsByRootsRequest(_) => WorkType::DataColumnsByRootsRequest, Work::DataColumnsByRangeRequest(_) => WorkType::DataColumnsByRangeRequest, Work::LightClientBootstrapRequest(_) => WorkType::LightClientBootstrapRequest, @@ -901,6 +906,8 @@ impl BeaconProcessor { let mut blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); let mut execution_proof_broots_queue = FifoQueue::new(queue_lengths.execution_proof_broots_queue); + let mut execution_proof_brange_queue = + FifoQueue::new(queue_lengths.execution_proof_brange_queue); let mut blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); let mut dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); let mut dcbrange_queue = FifoQueue::new(queue_lengths.dcbrange_queue); @@ -1226,6 +1233,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = execution_proof_broots_queue.pop() { Some(item) + } else if let Some(item) = execution_proof_brange_queue.pop() { + Some(item) } else if let Some(item) = dcbroots_queue.pop() { Some(item) } else if let Some(item) = dcbrange_queue.pop() { @@ -1430,6 +1439,9 @@ impl BeaconProcessor { Work::ExecutionProofsByRootsRequest { .. } => { execution_proof_broots_queue.push(work, work_id) } + Work::ExecutionProofsByRangeRequest { .. } => { + execution_proof_brange_queue.push(work, work_id) + } Work::DataColumnsByRootsRequest { .. } => { dcbroots_queue.push(work, work_id) } @@ -1489,6 +1501,9 @@ impl BeaconProcessor { WorkType::ExecutionProofsByRootsRequest => { execution_proof_broots_queue.len() } + WorkType::ExecutionProofsByRangeRequest => { + execution_proof_brange_queue.len() + } WorkType::DataColumnsByRootsRequest => dcbroots_queue.len(), WorkType::DataColumnsByRangeRequest => dcbrange_queue.len(), WorkType::GossipBlsToExecutionChange => { @@ -1649,6 +1664,7 @@ impl BeaconProcessor { Work::BlobsByRangeRequest(process_fn) | Work::BlobsByRootsRequest(process_fn) | Work::ExecutionProofsByRootsRequest(process_fn) + | Work::ExecutionProofsByRangeRequest(process_fn) | Work::DataColumnsByRootsRequest(process_fn) | Work::DataColumnsByRangeRequest(process_fn) => { task_spawner.spawn_blocking(process_fn) diff --git a/beacon_node/http_api/src/beacon/pool.rs b/beacon_node/http_api/src/beacon/pool.rs index 63b1a95b2ed..50a257db01b 100644 --- a/beacon_node/http_api/src/beacon/pool.rs +++ b/beacon_node/http_api/src/beacon/pool.rs @@ -530,6 +530,7 @@ pub fn post_beacon_pool_attestations_v2( /// Submits an execution proof to the beacon node. /// The proof will be validated and stored in the data availability checker. /// If valid, the proof will be published to the gossip network. +/// If the proof makes a block available, the block will be imported. pub fn post_beacon_pool_execution_proofs( network_tx_filter: &NetworkTxFilter, beacon_pool_path: &BeaconPoolPathFilter, @@ -541,81 +542,92 @@ pub fn post_beacon_pool_execution_proofs( .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |task_spawner: TaskSpawner, + |_task_spawner: TaskSpawner, chain: Arc>, proof: ExecutionProof, - network_tx: UnboundedSender>| { - task_spawner.blocking_json_task(Priority::P0, move || { - let proof = Arc::new(proof); - - // Validate the proof using the same logic as gossip validation - let verified_proof: GossipVerifiedExecutionProof = - GossipVerifiedExecutionProof::new(proof.clone(), &chain).map_err(|e| { - match e { - GossipExecutionProofError::PriorKnown { - slot, - block_root, - proof_id, - } => { - debug!( - %slot, - %block_root, - %proof_id, - "Execution proof already known" - ); - warp_utils::reject::custom_bad_request(format!( - "proof already known for slot {} block_root {} proof_id {}", - slot, block_root, proof_id - )) - } - GossipExecutionProofError::PriorKnownUnpublished => { - // Proof is valid but was received via non-gossip source - // It's in the DA checker, so we should publish it to gossip - warp_utils::reject::custom_bad_request( - "proof already received but not yet published".to_string(), - ) - } - _ => warp_utils::reject::object_invalid(format!( - "proof verification failed: {:?}", - e - )), - } - })?; - - let slot = verified_proof.slot(); - let block_root = verified_proof.block_root(); - let proof_id = verified_proof.subnet_id(); - - // Publish the proof to the gossip network - utils::publish_pubsub_message( - &network_tx, - PubsubMessage::ExecutionProof(verified_proof.clone().into_inner()), - )?; - - // Store the proof in the data availability checker - if let Err(e) = chain - .data_availability_checker - .put_rpc_execution_proofs(block_root, vec![verified_proof.into_inner()]) - { - warn!( - %slot, - %block_root, - %proof_id, - error = ?e, - "Failed to store execution proof in DA checker" - ); - } - - info!( - %slot, - %block_root, - %proof_id, - "Execution proof submitted and published" - ); - - Ok(()) - }) + network_tx: UnboundedSender>| async move { + let result = publish_execution_proof(chain, proof, network_tx).await; + convert_rejection(result.map(|()| warp::reply::json(&()))).await }, ) .boxed() } + +/// Validate, publish, and process an execution proof. +async fn publish_execution_proof( + chain: Arc>, + proof: ExecutionProof, + network_tx: UnboundedSender>, +) -> Result<(), warp::Rejection> { + let proof = Arc::new(proof); + + // Validate the proof using the same logic as gossip validation + let verified_proof: GossipVerifiedExecutionProof = + GossipVerifiedExecutionProof::new(proof.clone(), &chain).map_err(|e| match e { + GossipExecutionProofError::PriorKnown { + slot, + block_root, + proof_id, + } => { + debug!( + %slot, + %block_root, + %proof_id, + "Execution proof already known" + ); + warp_utils::reject::custom_bad_request(format!( + "proof already known for slot {} block_root {} proof_id {}", + slot, block_root, proof_id + )) + } + GossipExecutionProofError::PriorKnownUnpublished => { + // Proof is valid but was received via non-gossip source + // It's in the DA checker, so we should publish it to gossip + warp_utils::reject::custom_bad_request( + "proof already received but not yet published".to_string(), + ) + } + _ => warp_utils::reject::object_invalid(format!("proof verification failed: {:?}", e)), + })?; + + let slot = verified_proof.slot(); + let block_root = verified_proof.block_root(); + let proof_id = verified_proof.subnet_id(); + + // Publish the proof to the gossip network + utils::publish_pubsub_message( + &network_tx, + PubsubMessage::ExecutionProof(verified_proof.clone().into_inner()), + )?; + + // Store the proof in the data availability checker and check if block is now available. + // This properly triggers block import if all components are now available. + match chain + .process_rpc_execution_proofs(slot, block_root, vec![verified_proof.into_inner()]) + .await + { + Ok(status) => { + info!( + %slot, + %block_root, + %proof_id, + ?status, + "Execution proof submitted and published" + ); + } + Err(e) => { + // Log the error but don't fail the request - the proof was already + // published to gossip and stored in the DA checker. The error is + // likely due to the block already being imported or similar. + debug!( + %slot, + %block_root, + %proof_id, + error = ?e, + "Error processing execution proof availability (proof was still published)" + ); + } + } + + Ok(()) +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 1b280d54035..52b98d4d3c7 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -606,6 +606,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => PeerAction::MidToleranceError, Protocol::DataColumnsByRange => PeerAction::MidToleranceError, Protocol::ExecutionProofsByRoot => PeerAction::MidToleranceError, + Protocol::ExecutionProofsByRange => PeerAction::MidToleranceError, Protocol::Goodbye => PeerAction::LowToleranceError, Protocol::MetaData => PeerAction::LowToleranceError, Protocol::Status => PeerAction::LowToleranceError, @@ -627,6 +628,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => return, Protocol::DataColumnsByRange => return, Protocol::ExecutionProofsByRoot => return, + Protocol::ExecutionProofsByRange => return, Protocol::Goodbye => return, Protocol::LightClientBootstrap => return, Protocol::LightClientOptimisticUpdate => return, @@ -651,6 +653,7 @@ impl PeerManager { Protocol::DataColumnsByRoot => PeerAction::MidToleranceError, Protocol::DataColumnsByRange => PeerAction::MidToleranceError, Protocol::ExecutionProofsByRoot => PeerAction::MidToleranceError, + Protocol::ExecutionProofsByRange => PeerAction::MidToleranceError, Protocol::LightClientBootstrap => return, Protocol::LightClientOptimisticUpdate => return, Protocol::LightClientFinalityUpdate => return, diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index b3401038df8..aa0fe8a3d9d 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -81,6 +81,7 @@ impl SSZSnappyInboundCodec { RpcSuccessResponse::DataColumnsByRoot(res) => res.as_ssz_bytes(), RpcSuccessResponse::DataColumnsByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::ExecutionProofsByRoot(res) => res.as_ssz_bytes(), + RpcSuccessResponse::ExecutionProofsByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientBootstrap(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientOptimisticUpdate(res) => res.as_ssz_bytes(), RpcSuccessResponse::LightClientFinalityUpdate(res) => res.as_ssz_bytes(), @@ -362,6 +363,7 @@ impl Encoder> for SSZSnappyOutboundCodec { RequestType::DataColumnsByRange(req) => req.as_ssz_bytes(), RequestType::DataColumnsByRoot(req) => req.data_column_ids.as_ssz_bytes(), RequestType::ExecutionProofsByRoot(req) => req.as_ssz_bytes(), + RequestType::ExecutionProofsByRange(req) => req.as_ssz_bytes(), RequestType::Ping(req) => req.as_ssz_bytes(), RequestType::LightClientBootstrap(req) => req.as_ssz_bytes(), RequestType::LightClientUpdatesByRange(req) => req.as_ssz_bytes(), @@ -578,6 +580,11 @@ fn handle_rpc_request( Ok(Some(RequestType::ExecutionProofsByRoot(request))) } + SupportedProtocol::ExecutionProofsByRangeV1 => { + Ok(Some(RequestType::ExecutionProofsByRange( + ExecutionProofsByRangeRequest::from_ssz_bytes(decoded_buffer)?, + ))) + } SupportedProtocol::PingV1 => Ok(Some(RequestType::Ping(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -746,6 +753,11 @@ fn handle_rpc_response( ExecutionProof::from_ssz_bytes(decoded_buffer)?, )))) } + SupportedProtocol::ExecutionProofsByRangeV1 => { + Ok(Some(RpcSuccessResponse::ExecutionProofsByRange(Arc::new( + ExecutionProof::from_ssz_bytes(decoded_buffer)?, + )))) + } SupportedProtocol::PingV1 => Ok(Some(RpcSuccessResponse::Pong(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -1295,6 +1307,12 @@ mod tests { RequestType::ExecutionProofsByRoot(exec_proofs) => { assert_eq!(decoded, RequestType::ExecutionProofsByRoot(exec_proofs)) } + RequestType::ExecutionProofsByRange(exec_proofs_range) => { + assert_eq!( + decoded, + RequestType::ExecutionProofsByRange(exec_proofs_range) + ) + } RequestType::Ping(ping) => { assert_eq!(decoded, RequestType::Ping(ping)) } diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index d23c16f8fa1..99c0f33da31 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -94,6 +94,7 @@ pub struct RateLimiterConfig { pub(super) data_columns_by_root_quota: Quota, pub(super) data_columns_by_range_quota: Quota, pub(super) execution_proofs_by_root_quota: Quota, + pub(super) execution_proofs_by_range_quota: Quota, pub(super) light_client_bootstrap_quota: Quota, pub(super) light_client_optimistic_update_quota: Quota, pub(super) light_client_finality_update_quota: Quota, @@ -126,6 +127,9 @@ impl RateLimiterConfig { // TODO(zkproofs): Configure this to be less arbitrary pub const DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); + // TODO(zkproofs): Configure this to be less arbitrary + pub const DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA: Quota = + Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_FINALITY_UPDATE_QUOTA: Quota = Quota::one_every(10); @@ -146,6 +150,7 @@ impl Default for RateLimiterConfig { data_columns_by_root_quota: Self::DEFAULT_DATA_COLUMNS_BY_ROOT_QUOTA, data_columns_by_range_quota: Self::DEFAULT_DATA_COLUMNS_BY_RANGE_QUOTA, execution_proofs_by_root_quota: Self::DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA, + execution_proofs_by_range_quota: Self::DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA, light_client_bootstrap_quota: Self::DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA, light_client_optimistic_update_quota: Self::DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA, @@ -184,6 +189,14 @@ impl Debug for RateLimiterConfig { "data_columns_by_root", fmt_q!(&self.data_columns_by_root_quota), ) + .field( + "execution_proofs_by_root", + fmt_q!(&self.execution_proofs_by_root_quota), + ) + .field( + "execution_proofs_by_range", + fmt_q!(&self.execution_proofs_by_range_quota), + ) .finish() } } @@ -207,6 +220,7 @@ impl FromStr for RateLimiterConfig { let mut data_columns_by_root_quota = None; let mut data_columns_by_range_quota = None; let mut execution_proofs_by_root_quota = None; + let mut execution_proofs_by_range_quota = None; let mut light_client_bootstrap_quota = None; let mut light_client_optimistic_update_quota = None; let mut light_client_finality_update_quota = None; @@ -231,6 +245,9 @@ impl FromStr for RateLimiterConfig { Protocol::ExecutionProofsByRoot => { execution_proofs_by_root_quota = execution_proofs_by_root_quota.or(quota) } + Protocol::ExecutionProofsByRange => { + execution_proofs_by_range_quota = execution_proofs_by_range_quota.or(quota) + } Protocol::Ping => ping_quota = ping_quota.or(quota), Protocol::MetaData => meta_data_quota = meta_data_quota.or(quota), Protocol::LightClientBootstrap => { @@ -268,6 +285,8 @@ impl FromStr for RateLimiterConfig { .unwrap_or(Self::DEFAULT_DATA_COLUMNS_BY_RANGE_QUOTA), execution_proofs_by_root_quota: execution_proofs_by_root_quota .unwrap_or(Self::DEFAULT_EXECUTION_PROOFS_BY_ROOT_QUOTA), + execution_proofs_by_range_quota: execution_proofs_by_range_quota + .unwrap_or(Self::DEFAULT_EXECUTION_PROOFS_BY_RANGE_QUOTA), light_client_bootstrap_quota: light_client_bootstrap_quota .unwrap_or(Self::DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA), light_client_optimistic_update_quota: light_client_optimistic_update_quota diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 9ba8f66dafa..966106b6f69 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -603,6 +603,36 @@ impl ExecutionProofsByRootRequest { } } +/// Request execution proofs for a range of slots. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct ExecutionProofsByRangeRequest { + /// The starting slot to request execution proofs. + pub start_slot: u64, + /// The number of slots from the start slot. + pub count: u64, +} + +impl ExecutionProofsByRangeRequest { + pub fn max_proofs_requested(&self) -> u64 { + // Each slot could have up to MAX_PROOFS execution proofs + self.count + .saturating_mul(types::execution_proof::MAX_PROOFS as u64) + } + + pub fn ssz_min_len() -> usize { + ExecutionProofsByRangeRequest { + start_slot: 0, + count: 0, + } + .as_ssz_bytes() + .len() + } + + pub fn ssz_max_len() -> usize { + Self::ssz_min_len() + } +} + /// Request a number of beacon data columns from a peer. #[derive(Encode, Decode, Clone, Debug, PartialEq)] pub struct LightClientUpdatesByRangeRequest { @@ -673,6 +703,9 @@ pub enum RpcSuccessResponse { /// A response to a get EXECUTION_PROOFS_BY_ROOT request. ExecutionProofsByRoot(Arc), + /// A response to a get EXECUTION_PROOFS_BY_RANGE request. + ExecutionProofsByRange(Arc), + /// A PONG response to a PING request. Pong(Ping), @@ -704,6 +737,9 @@ pub enum ResponseTermination { /// Execution proofs by root stream termination. ExecutionProofsByRoot, + /// Execution proofs by range stream termination. + ExecutionProofsByRange, + /// Light client updates by range stream termination. LightClientUpdatesByRange, } @@ -718,6 +754,7 @@ impl ResponseTermination { ResponseTermination::DataColumnsByRoot => Protocol::DataColumnsByRoot, ResponseTermination::DataColumnsByRange => Protocol::DataColumnsByRange, ResponseTermination::ExecutionProofsByRoot => Protocol::ExecutionProofsByRoot, + ResponseTermination::ExecutionProofsByRange => Protocol::ExecutionProofsByRange, ResponseTermination::LightClientUpdatesByRange => Protocol::LightClientUpdatesByRange, } } @@ -814,6 +851,7 @@ impl RpcSuccessResponse { RpcSuccessResponse::DataColumnsByRoot(_) => Protocol::DataColumnsByRoot, RpcSuccessResponse::DataColumnsByRange(_) => Protocol::DataColumnsByRange, RpcSuccessResponse::ExecutionProofsByRoot(_) => Protocol::ExecutionProofsByRoot, + RpcSuccessResponse::ExecutionProofsByRange(_) => Protocol::ExecutionProofsByRange, RpcSuccessResponse::Pong(_) => Protocol::Ping, RpcSuccessResponse::MetaData(_) => Protocol::MetaData, RpcSuccessResponse::LightClientBootstrap(_) => Protocol::LightClientBootstrap, @@ -840,6 +878,7 @@ impl RpcSuccessResponse { Self::LightClientUpdatesByRange(r) => Some(r.attested_header_slot()), // TODO(zkproofs): Change this when we add Slot to ExecutionProof Self::ExecutionProofsByRoot(_) + | Self::ExecutionProofsByRange(_) | Self::MetaData(_) | Self::Status(_) | Self::Pong(_) => None, @@ -905,6 +944,13 @@ impl std::fmt::Display for RpcSuccessResponse { RpcSuccessResponse::ExecutionProofsByRoot(proof) => { write!(f, "ExecutionProofsByRoot: Block root: {}", proof.block_root) } + RpcSuccessResponse::ExecutionProofsByRange(proof) => { + write!( + f, + "ExecutionProofsByRange: Block root: {}", + proof.block_root + ) + } RpcSuccessResponse::Pong(ping) => write!(f, "Pong: {}", ping.data), RpcSuccessResponse::MetaData(metadata) => { write!(f, "Metadata: {}", metadata.seq_number()) @@ -1027,3 +1073,13 @@ impl std::fmt::Display for ExecutionProofsByRootRequest { ) } } + +impl std::fmt::Display for ExecutionProofsByRangeRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Request: ExecutionProofsByRange: Start Slot: {}, Count: {}", + self.start_slot, self.count + ) + } +} diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index dfa44976390..0a37db0d210 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -254,6 +254,9 @@ pub enum Protocol { /// The `ExecutionProofsByRoot` protocol name. #[strum(serialize = "execution_proofs_by_root")] ExecutionProofsByRoot, + /// The `ExecutionProofsByRange` protocol name. + #[strum(serialize = "execution_proofs_by_range")] + ExecutionProofsByRange, /// The `Ping` protocol name. Ping, /// The `MetaData` protocol name. @@ -285,6 +288,7 @@ impl Protocol { Protocol::DataColumnsByRoot => Some(ResponseTermination::DataColumnsByRoot), Protocol::DataColumnsByRange => Some(ResponseTermination::DataColumnsByRange), Protocol::ExecutionProofsByRoot => Some(ResponseTermination::ExecutionProofsByRoot), + Protocol::ExecutionProofsByRange => Some(ResponseTermination::ExecutionProofsByRange), Protocol::Ping => None, Protocol::MetaData => None, Protocol::LightClientBootstrap => None, @@ -316,6 +320,7 @@ pub enum SupportedProtocol { DataColumnsByRootV1, DataColumnsByRangeV1, ExecutionProofsByRootV1, + ExecutionProofsByRangeV1, PingV1, MetaDataV1, MetaDataV2, @@ -341,6 +346,7 @@ impl SupportedProtocol { SupportedProtocol::DataColumnsByRootV1 => "1", SupportedProtocol::DataColumnsByRangeV1 => "1", SupportedProtocol::ExecutionProofsByRootV1 => "1", + SupportedProtocol::ExecutionProofsByRangeV1 => "1", SupportedProtocol::PingV1 => "1", SupportedProtocol::MetaDataV1 => "1", SupportedProtocol::MetaDataV2 => "2", @@ -366,6 +372,7 @@ impl SupportedProtocol { SupportedProtocol::DataColumnsByRootV1 => Protocol::DataColumnsByRoot, SupportedProtocol::DataColumnsByRangeV1 => Protocol::DataColumnsByRange, SupportedProtocol::ExecutionProofsByRootV1 => Protocol::ExecutionProofsByRoot, + SupportedProtocol::ExecutionProofsByRangeV1 => Protocol::ExecutionProofsByRange, SupportedProtocol::PingV1 => Protocol::Ping, SupportedProtocol::MetaDataV1 => Protocol::MetaData, SupportedProtocol::MetaDataV2 => Protocol::MetaData, @@ -417,10 +424,16 @@ impl SupportedProtocol { ]); } if fork_context.spec.is_zkvm_enabled() { - supported.push(ProtocolId::new( - SupportedProtocol::ExecutionProofsByRootV1, - Encoding::SSZSnappy, - )); + supported.extend_from_slice(&[ + ProtocolId::new( + SupportedProtocol::ExecutionProofsByRootV1, + Encoding::SSZSnappy, + ), + ProtocolId::new( + SupportedProtocol::ExecutionProofsByRangeV1, + Encoding::SSZSnappy, + ), + ]); } supported } @@ -535,6 +548,10 @@ impl ProtocolId { DataColumnsByRangeRequest::ssz_max_len::(), ), Protocol::ExecutionProofsByRoot => RpcLimits::new(0, spec.max_blocks_by_root_request), + Protocol::ExecutionProofsByRange => RpcLimits::new( + ExecutionProofsByRangeRequest::ssz_min_len(), + ExecutionProofsByRangeRequest::ssz_max_len(), + ), Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -572,6 +589,7 @@ impl ProtocolId { rpc_data_column_limits::(fork_context.current_fork_epoch(), &fork_context.spec) } Protocol::ExecutionProofsByRoot => rpc_execution_proof_limits(), + Protocol::ExecutionProofsByRange => rpc_execution_proof_limits(), Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -614,6 +632,7 @@ impl ProtocolId { | SupportedProtocol::BlocksByRootV1 | SupportedProtocol::BlocksByRangeV1 | SupportedProtocol::ExecutionProofsByRootV1 + | SupportedProtocol::ExecutionProofsByRangeV1 | SupportedProtocol::PingV1 | SupportedProtocol::MetaDataV1 | SupportedProtocol::MetaDataV2 @@ -748,6 +767,7 @@ pub enum RequestType { DataColumnsByRoot(DataColumnsByRootRequest), DataColumnsByRange(DataColumnsByRangeRequest), ExecutionProofsByRoot(ExecutionProofsByRootRequest), + ExecutionProofsByRange(ExecutionProofsByRangeRequest), LightClientBootstrap(LightClientBootstrapRequest), LightClientOptimisticUpdate, LightClientFinalityUpdate, @@ -772,6 +792,7 @@ impl RequestType { RequestType::DataColumnsByRoot(req) => req.max_requested() as u64, RequestType::DataColumnsByRange(req) => req.max_requested::(), RequestType::ExecutionProofsByRoot(req) => req.max_requested() as u64, + RequestType::ExecutionProofsByRange(req) => req.max_proofs_requested(), RequestType::Ping(_) => 1, RequestType::MetaData(_) => 1, RequestType::LightClientBootstrap(_) => 1, @@ -802,6 +823,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => SupportedProtocol::DataColumnsByRootV1, RequestType::DataColumnsByRange(_) => SupportedProtocol::DataColumnsByRangeV1, RequestType::ExecutionProofsByRoot(_) => SupportedProtocol::ExecutionProofsByRootV1, + RequestType::ExecutionProofsByRange(_) => SupportedProtocol::ExecutionProofsByRangeV1, RequestType::Ping(_) => SupportedProtocol::PingV1, RequestType::MetaData(req) => match req { MetadataRequest::V1(_) => SupportedProtocol::MetaDataV1, @@ -834,6 +856,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => ResponseTermination::DataColumnsByRoot, RequestType::DataColumnsByRange(_) => ResponseTermination::DataColumnsByRange, RequestType::ExecutionProofsByRoot(_) => ResponseTermination::ExecutionProofsByRoot, + RequestType::ExecutionProofsByRange(_) => ResponseTermination::ExecutionProofsByRange, RequestType::Status(_) => unreachable!(), RequestType::Goodbye(_) => unreachable!(), RequestType::Ping(_) => unreachable!(), @@ -884,6 +907,10 @@ impl RequestType { SupportedProtocol::ExecutionProofsByRootV1, Encoding::SSZSnappy, )], + RequestType::ExecutionProofsByRange(_) => vec![ProtocolId::new( + SupportedProtocol::ExecutionProofsByRangeV1, + Encoding::SSZSnappy, + )], RequestType::Ping(_) => vec![ProtocolId::new( SupportedProtocol::PingV1, Encoding::SSZSnappy, @@ -923,6 +950,7 @@ impl RequestType { RequestType::DataColumnsByRoot(_) => false, RequestType::DataColumnsByRange(_) => false, RequestType::ExecutionProofsByRoot(_) => false, + RequestType::ExecutionProofsByRange(_) => false, RequestType::Ping(_) => true, RequestType::MetaData(_) => true, RequestType::LightClientBootstrap(_) => true, @@ -1039,6 +1067,9 @@ impl std::fmt::Display for RequestType { RequestType::ExecutionProofsByRoot(req) => { write!(f, "Execution proofs by root: {:?}", req) } + RequestType::ExecutionProofsByRange(req) => { + write!(f, "Execution proofs by range: {:?}", req) + } RequestType::Ping(ping) => write!(f, "Ping: {}", ping.data), RequestType::MetaData(_) => write!(f, "MetaData request"), RequestType::LightClientBootstrap(bootstrap) => { diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index f70b29cfe45..9dfbc668c89 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -107,6 +107,8 @@ pub struct RPCRateLimiter { dcbrange_rl: Limiter, /// ExecutionProofsByRoot rate limiter. execution_proofs_by_root_rl: Limiter, + /// ExecutionProofsByRange rate limiter. + execution_proofs_by_range_rl: Limiter, /// LightClientBootstrap rate limiter. lc_bootstrap_rl: Limiter, /// LightClientOptimisticUpdate rate limiter. @@ -152,6 +154,8 @@ pub struct RPCRateLimiterBuilder { dcbrange_quota: Option, /// Quota for the ExecutionProofsByRoot protocol. execution_proofs_by_root_quota: Option, + /// Quota for the ExecutionProofsByRange protocol. + execution_proofs_by_range_quota: Option, /// Quota for the LightClientBootstrap protocol. lcbootstrap_quota: Option, /// Quota for the LightClientOptimisticUpdate protocol. @@ -178,6 +182,7 @@ impl RPCRateLimiterBuilder { Protocol::DataColumnsByRoot => self.dcbroot_quota = q, Protocol::DataColumnsByRange => self.dcbrange_quota = q, Protocol::ExecutionProofsByRoot => self.execution_proofs_by_root_quota = q, + Protocol::ExecutionProofsByRange => self.execution_proofs_by_range_quota = q, Protocol::LightClientBootstrap => self.lcbootstrap_quota = q, Protocol::LightClientOptimisticUpdate => self.lc_optimistic_update_quota = q, Protocol::LightClientFinalityUpdate => self.lc_finality_update_quota = q, @@ -230,6 +235,10 @@ impl RPCRateLimiterBuilder { .execution_proofs_by_root_quota .ok_or("ExecutionProofsByRoot quota not specified")?; + let execution_proofs_by_range_quota = self + .execution_proofs_by_range_quota + .ok_or("ExecutionProofsByRange quota not specified")?; + // create the rate limiters let ping_rl = Limiter::from_quota(ping_quota)?; let metadata_rl = Limiter::from_quota(metadata_quota)?; @@ -242,6 +251,7 @@ impl RPCRateLimiterBuilder { let dcbroot_rl = Limiter::from_quota(dcbroot_quota)?; let dcbrange_rl = Limiter::from_quota(dcbrange_quota)?; let execution_proofs_by_root_rl = Limiter::from_quota(execution_proofs_by_root_quota)?; + let execution_proofs_by_range_rl = Limiter::from_quota(execution_proofs_by_range_quota)?; let lc_bootstrap_rl = Limiter::from_quota(lc_bootstrap_quota)?; let lc_optimistic_update_rl = Limiter::from_quota(lc_optimistic_update_quota)?; let lc_finality_update_rl = Limiter::from_quota(lc_finality_update_quota)?; @@ -266,6 +276,7 @@ impl RPCRateLimiterBuilder { dcbroot_rl, dcbrange_rl, execution_proofs_by_root_rl, + execution_proofs_by_range_rl, lc_bootstrap_rl, lc_optimistic_update_rl, lc_finality_update_rl, @@ -320,6 +331,7 @@ impl RPCRateLimiter { data_columns_by_root_quota, data_columns_by_range_quota, execution_proofs_by_root_quota, + execution_proofs_by_range_quota, light_client_bootstrap_quota, light_client_optimistic_update_quota, light_client_finality_update_quota, @@ -341,6 +353,10 @@ impl RPCRateLimiter { Protocol::ExecutionProofsByRoot, execution_proofs_by_root_quota, ) + .set_quota( + Protocol::ExecutionProofsByRange, + execution_proofs_by_range_quota, + ) .set_quota(Protocol::LightClientBootstrap, light_client_bootstrap_quota) .set_quota( Protocol::LightClientOptimisticUpdate, @@ -389,6 +405,7 @@ impl RPCRateLimiter { Protocol::DataColumnsByRoot => &mut self.dcbroot_rl, Protocol::DataColumnsByRange => &mut self.dcbrange_rl, Protocol::ExecutionProofsByRoot => &mut self.execution_proofs_by_root_rl, + Protocol::ExecutionProofsByRange => &mut self.execution_proofs_by_range_rl, Protocol::LightClientBootstrap => &mut self.lc_bootstrap_rl, Protocol::LightClientOptimisticUpdate => &mut self.lc_optimistic_update_rl, Protocol::LightClientFinalityUpdate => &mut self.lc_finality_update_rl, @@ -414,6 +431,7 @@ impl RPCRateLimiter { dcbroot_rl, dcbrange_rl, execution_proofs_by_root_rl, + execution_proofs_by_range_rl, lc_bootstrap_rl, lc_optimistic_update_rl, lc_finality_update_rl, @@ -432,6 +450,7 @@ impl RPCRateLimiter { dcbrange_rl.prune(time_since_start); dcbroot_rl.prune(time_since_start); execution_proofs_by_root_rl.prune(time_since_start); + execution_proofs_by_range_rl.prune(time_since_start); lc_bootstrap_rl.prune(time_since_start); lc_optimistic_update_rl.prune(time_since_start); lc_finality_update_rl.prune(time_since_start); diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index d97506653b5..ca3a5b78bd9 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -32,6 +32,8 @@ pub enum SyncRequestId { BlobsByRange(BlobsByRangeRequestId), /// Data columns by range request DataColumnsByRange(DataColumnsByRangeRequestId), + /// Execution proofs by range request + ExecutionProofsByRange(ExecutionProofsByRangeRequestId), } /// Request ID for data_columns_by_root requests. Block lookups do not issue this request directly. @@ -77,6 +79,17 @@ pub enum DataColumnsByRangeRequester { CustodyBackfillSync(CustodyBackFillBatchRequestId), } +/// Request ID for execution_proofs_by_range requests during range sync. +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +pub struct ExecutionProofsByRangeRequestId { + /// Id to identify this attempt at an execution_proofs_by_range request for `parent_request_id` + pub id: Id, + /// The Id of the overall By Range request. + pub parent_request_id: ComponentsByRangeRequestId, + /// The peer id associated with the request. + pub peer: PeerId, +} + /// Block components by range request for range sync. Includes an ID for downstream consumers to /// handle retries and tie all their sub requests together. #[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] @@ -168,6 +181,8 @@ pub enum Response { DataColumnsByRoot(Option>>), /// A response to a get EXECUTION_PROOFS_BY_ROOT request. ExecutionProofsByRoot(Option>), + /// A response to a get EXECUTION_PROOFS_BY_RANGE request. + ExecutionProofsByRange(Option>), /// A response to a LightClientUpdate request. LightClientBootstrap(Arc>), /// A response to a LightClientOptimisticUpdate request. @@ -209,6 +224,10 @@ impl std::convert::From> for RpcResponse { Some(p) => RpcResponse::Success(RpcSuccessResponse::ExecutionProofsByRoot(p)), None => RpcResponse::StreamTermination(ResponseTermination::ExecutionProofsByRoot), }, + Response::ExecutionProofsByRange(r) => match r { + Some(p) => RpcResponse::Success(RpcSuccessResponse::ExecutionProofsByRange(p)), + None => RpcResponse::StreamTermination(ResponseTermination::ExecutionProofsByRange), + }, Response::Status(s) => RpcResponse::Success(RpcSuccessResponse::Status(s)), Response::LightClientBootstrap(b) => { RpcResponse::Success(RpcSuccessResponse::LightClientBootstrap(b)) @@ -245,6 +264,12 @@ macro_rules! impl_display { impl_display!(BlocksByRangeRequestId, "{}/{}", id, parent_request_id); impl_display!(BlobsByRangeRequestId, "{}/{}", id, parent_request_id); impl_display!(DataColumnsByRangeRequestId, "{}/{}", id, parent_request_id); +impl_display!( + ExecutionProofsByRangeRequestId, + "{}/{}", + id, + parent_request_id +); impl_display!(ComponentsByRangeRequestId, "{}/{}", id, requester); impl_display!(DataColumnsByRootRequestId, "{}/{}", id, requester); impl_display!(SingleLookupReqId, "{}/Lookup/{}", req_id, lookup_id); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9f1530ec732..4e8be98a509 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -411,7 +411,7 @@ impl Network { }; let peer_manager = { - let peer_manager_cfg = PeerManagerCfg { + let mut peer_manager_cfg = PeerManagerCfg { discovery_enabled: !config.disable_discovery, quic_enabled: !config.disable_quic_support, metrics_enabled: config.metrics_enabled, @@ -419,6 +419,15 @@ impl Network { execution_proof_enabled: ctx.chain_spec.is_zkvm_enabled(), ..Default::default() }; + // TODO(zkproofs): We decrease the slot time, so we want to + // correspondingly decrease the status interval at which a node will + // check if it needs to sync with others. + let epoch_secs = ctx + .chain_spec + .seconds_per_slot + .saturating_mul(E::slots_per_epoch()) + .max(1); + peer_manager_cfg.status_interval = peer_manager_cfg.status_interval.min(epoch_secs); PeerManager::new(peer_manager_cfg, network_globals.clone())? }; @@ -1580,6 +1589,17 @@ impl Network { request_type, }) } + RequestType::ExecutionProofsByRange(_) => { + metrics::inc_counter_vec( + &metrics::TOTAL_RPC_REQUESTS, + &["execution_proofs_by_range"], + ); + Some(NetworkEvent::RequestReceived { + peer_id, + inbound_request_id, + request_type, + }) + } RequestType::LightClientBootstrap(_) => { metrics::inc_counter_vec( &metrics::TOTAL_RPC_REQUESTS, @@ -1670,6 +1690,11 @@ impl Network { peer_id, Response::ExecutionProofsByRoot(Some(resp)), ), + RpcSuccessResponse::ExecutionProofsByRange(resp) => self.build_response( + id, + peer_id, + Response::ExecutionProofsByRange(Some(resp)), + ), // Should never be reached RpcSuccessResponse::LightClientBootstrap(bootstrap) => { self.build_response(id, peer_id, Response::LightClientBootstrap(bootstrap)) @@ -1702,6 +1727,9 @@ impl Network { ResponseTermination::ExecutionProofsByRoot => { Response::ExecutionProofsByRoot(None) } + ResponseTermination::ExecutionProofsByRange => { + Response::ExecutionProofsByRange(None) + } ResponseTermination::LightClientUpdatesByRange => { Response::LightClientUpdatesByRange(None) } diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs index dd9e9f1ebb2..9ca5afbcf9c 100644 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ b/beacon_node/lighthouse_tracing/src/lib.rs @@ -41,6 +41,8 @@ pub const SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST: &str = "handle_blocks_by_root_requ pub const SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST: &str = "handle_blobs_by_root_request"; pub const SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST: &str = "handle_execution_proofs_by_root_request"; +pub const SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST: &str = + "handle_execution_proofs_by_range_request"; pub const SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST: &str = "handle_data_columns_by_root_request"; pub const SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE: &str = "handle_light_client_updates_by_range"; pub const SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP: &str = "handle_light_client_bootstrap"; @@ -73,6 +75,7 @@ pub const LH_BN_ROOT_SPAN_NAMES: &[&str] = &[ SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, + SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index c8440a6bbf4..5d2203ee380 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -820,13 +820,29 @@ impl NetworkBeaconProcessor { debug!( %block_root, %proof_id, - "Gossip execution proof already processed via the EL. Accepting the proof without re-processing." + "Gossip execution proof already processed via the EL. Checking availability." ); self.propagate_validation_result( message_id, peer_id, MessageAcceptance::Accept, ); + + // The proof is already in the DA checker (from HTTP API). + // Check if this makes any pending blocks complete and import them. + let slot = execution_proof.slot; + if let Err(e) = self + .chain + .process_rpc_execution_proofs(slot, block_root, vec![execution_proof]) + .await + { + debug!( + %block_root, + %proof_id, + error = ?e, + "Failed to process availability for prior known execution proof" + ); + } } GossipExecutionProofError::PriorKnown { block_root, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7db2790920e..ffac53e522a 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -14,7 +14,7 @@ use beacon_processor::{ use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - ExecutionProofsByRootRequest, LightClientUpdatesByRangeRequest, + ExecutionProofsByRangeRequest, ExecutionProofsByRootRequest, LightClientUpdatesByRangeRequest, }; use lighthouse_network::service::api_types::CustodyBackfillBatchId; use lighthouse_network::{ @@ -699,6 +699,24 @@ impl NetworkBeaconProcessor { }) } + /// Create a new work event to process `ExecutionProofsByRangeRequest`s from the RPC network. + pub fn send_execution_proofs_by_range_request( + self: &Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: ExecutionProofsByRangeRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.handle_execution_proofs_by_range_request(peer_id, inbound_request_id, request) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::ExecutionProofsByRangeRequest(Box::new(process_fn)), + }) + } + /// Create a new work event to process `DataColumnsByRootRequest`s from the RPC network. pub fn send_data_columns_by_roots_request( self: &Arc, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index f063d7e8380..17ee4076731 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -7,7 +7,7 @@ use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessStatus, WhenS use itertools::{Itertools, process_results}; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - ExecutionProofsByRootRequest, + ExecutionProofsByRangeRequest, ExecutionProofsByRootRequest, }; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, ReportSource, Response, SyncInfo}; @@ -15,9 +15,9 @@ use lighthouse_tracing::{ SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, - SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, - SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, - SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, + SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, SPAN_HANDLE_EXECUTION_PROOFS_BY_ROOT_REQUEST, + SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, + SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, }; use methods::LightClientUpdatesByRangeRequest; use slot_clock::SlotClock; @@ -436,19 +436,39 @@ impl NetworkBeaconProcessor { request.already_have.iter().copied().collect(); let count_needed = request.count_needed as usize; - // Get all execution proofs we have for this block from the DA checker - let Some(available_proofs) = self + // Get all execution proofs we have for this block from the DA checker, falling back to the + // store (which checks the store cache/DB). + let available_proofs = match self .chain .data_availability_checker .get_execution_proofs(&block_root) - else { - // No proofs available for this block - debug!( - %peer_id, - %block_root, - "No execution proofs available for peer" - ); - return Ok(()); + { + Some(proofs) => proofs, + None => match self.chain.store.get_execution_proofs(&block_root) { + Ok(proofs) => { + if proofs.is_empty() { + debug!( + %peer_id, + %block_root, + "No execution proofs available for peer" + ); + return Ok(()); + } + proofs + } + Err(e) => { + error!( + %peer_id, + %block_root, + error = ?e, + "Error fetching execution proofs for block root" + ); + return Err(( + RpcErrorResponse::ServerError, + "Error fetching execution proofs", + )); + } + }, }; // Filter out proofs the peer already has and send up to count_needed @@ -486,6 +506,137 @@ impl NetworkBeaconProcessor { Ok(()) } + /// Handle an `ExecutionProofsByRange` request from the peer. + #[instrument( + name = SPAN_HANDLE_EXECUTION_PROOFS_BY_RANGE_REQUEST, + parent = None, + level = "debug", + skip_all, + fields(peer_id = %peer_id, client = tracing::field::Empty) + )] + pub fn handle_execution_proofs_by_range_request( + &self, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + req: ExecutionProofsByRangeRequest, + ) { + let client = self.network_globals.client(&peer_id); + Span::current().record("client", field::display(client.kind)); + + self.terminate_response_stream( + peer_id, + inbound_request_id, + self.handle_execution_proofs_by_range_request_inner(peer_id, inbound_request_id, req), + Response::ExecutionProofsByRange, + ); + } + + /// Handle an `ExecutionProofsByRange` request from the peer. + fn handle_execution_proofs_by_range_request_inner( + &self, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + req: ExecutionProofsByRangeRequest, + ) -> Result<(), (RpcErrorResponse, &'static str)> { + debug!( + %peer_id, + count = req.count, + start_slot = req.start_slot, + "Received ExecutionProofsByRange Request" + ); + + let request_start_slot = Slot::from(req.start_slot); + + // Check if zkvm is enabled and get the execution proof boundary + let execution_proof_boundary_slot = match self.chain.execution_proof_boundary() { + Some(boundary) => boundary.start_slot(T::EthSpec::slots_per_epoch()), + None => { + debug!("ZKVM fork is disabled"); + return Err((RpcErrorResponse::InvalidRequest, "ZKVM fork is disabled")); + } + }; + + // Get the oldest execution proof slot from the store + let oldest_execution_proof_slot = self + .chain + .store + .get_execution_proof_info() + .oldest_execution_proof_slot + .unwrap_or(execution_proof_boundary_slot); + + if request_start_slot < oldest_execution_proof_slot { + debug!( + %request_start_slot, + %oldest_execution_proof_slot, + %execution_proof_boundary_slot, + "Range request start slot is older than the oldest execution proof slot." + ); + + return if execution_proof_boundary_slot < oldest_execution_proof_slot { + Err(( + RpcErrorResponse::ResourceUnavailable, + "execution proofs pruned within boundary", + )) + } else { + Err(( + RpcErrorResponse::InvalidRequest, + "Req outside availability period", + )) + }; + } + + let block_roots = self.get_block_roots_for_slot_range( + req.start_slot, + req.count, + "ExecutionProofsByRange", + )?; + let mut proofs_sent = 0; + + for root in block_roots { + // Get execution proofs from the database (like BlobsByRange does for blobs) + match self.chain.store.get_execution_proofs(&root) { + Ok(proofs) => { + for proof in proofs { + // Due to skip slots, proofs could be out of the range + if proof.slot >= request_start_slot + && proof.slot < request_start_slot + req.count + { + proofs_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + inbound_request_id, + response: Response::ExecutionProofsByRange(Some(proof)), + }); + } + } + } + Err(e) => { + error!( + request = ?req, + %peer_id, + block_root = ?root, + error = ?e, + "Error fetching execution proofs for block root" + ); + return Err(( + RpcErrorResponse::ServerError, + "Failed fetching execution proofs from database", + )); + } + } + } + + debug!( + %peer_id, + start_slot = req.start_slot, + count = req.count, + sent = proofs_sent, + "ExecutionProofsByRange outgoing response processed" + ); + + Ok(()) + } + /// Handle a `DataColumnsByRoot` request from the peer. #[instrument( name = SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index eb02ddad921..f5bf65c9777 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -283,6 +283,15 @@ impl Router { request, ), ), + RequestType::ExecutionProofsByRange(request) => self + .handle_beacon_processor_send_result( + self.network_beacon_processor + .send_execution_proofs_by_range_request( + peer_id, + inbound_request_id, + request, + ), + ), _ => {} } } @@ -323,6 +332,13 @@ impl Router { Response::ExecutionProofsByRoot(execution_proof) => { self.on_execution_proofs_by_root_response(peer_id, app_request_id, execution_proof); } + Response::ExecutionProofsByRange(execution_proof) => { + self.on_execution_proofs_by_range_response( + peer_id, + app_request_id, + execution_proof, + ); + } // Light client responses should not be received Response::LightClientBootstrap(_) | Response::LightClientOptimisticUpdate(_) @@ -727,6 +743,30 @@ impl Router { }); } + /// Handle an `ExecutionProofsByRange` response from the peer. + pub fn on_execution_proofs_by_range_response( + &mut self, + peer_id: PeerId, + app_request_id: AppRequestId, + execution_proof: Option>, + ) { + trace!( + %peer_id, + "Received ExecutionProofsByRange Response" + ); + + if let AppRequestId::Sync(sync_request_id) = app_request_id { + self.send_to_sync(SyncMessage::RpcExecutionProof { + peer_id, + sync_request_id, + execution_proof, + seen_timestamp: timestamp_now(), + }); + } else { + crit!("All execution proofs by range responses should belong to sync"); + } + } + /// Handle a `DataColumnsByRoot` response from the peer. pub fn on_data_columns_by_root_response( &mut self, diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 6c0cbd7e554..441e9b0a6d9 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -348,6 +348,23 @@ impl BackFillSync { CouplingError::BlobPeerFailure(msg) => { tracing::debug!(?batch_id, msg, "Blob peer failure"); } + CouplingError::ExecutionProofPeerFailure { + error, + peer, + exceeded_retries, + } => { + tracing::debug!(?batch_id, ?peer, error, "Execution proof peer failure"); + if !*exceeded_retries { + let mut failed_peers = HashSet::new(); + failed_peers.insert(*peer); + return self.retry_execution_proof_batch( + network, + batch_id, + request_id, + failed_peers, + ); + } + } CouplingError::InternalError(msg) => { error!(?batch_id, msg, "Block components coupling internal error"); } @@ -1001,6 +1018,46 @@ impl BackFillSync { Ok(()) } + /// Retries execution proof requests within the batch by creating a new proofs request. + pub fn retry_execution_proof_batch( + &mut self, + network: &mut SyncNetworkContext, + batch_id: BatchId, + id: Id, + mut failed_peers: HashSet, + ) -> Result<(), BackFillError> { + if let Some(batch) = self.batches.get_mut(&batch_id) { + failed_peers.extend(&batch.failed_peers()); + let req = batch.to_blocks_by_range_request().0; + + let synced_peers = network + .network_globals() + .peers + .read() + .synced_peers_for_epoch(batch_id) + .cloned() + .collect::>(); + + match network.retry_execution_proofs_by_range(id, &synced_peers, &failed_peers, req) { + Ok(()) => { + debug!( + ?batch_id, + id, "Retried execution proof requests from different peers" + ); + return Ok(()); + } + Err(e) => { + debug!(?batch_id, id, e, "Failed to retry execution proof batch"); + } + } + } else { + return Err(BackFillError::InvalidSyncState( + "Batch should exist to be retried".to_string(), + )); + } + Ok(()) + } + /// When resuming a chain, this function searches for batches that need to be re-downloaded and /// transitions their state to redownload the batch. fn resume_batches(&mut self, network: &mut SyncNetworkContext) -> Result<(), BackFillError> { diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index ed9a11a03de..faa2fac949c 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -5,6 +5,7 @@ use lighthouse_network::{ PeerId, service::api_types::{ BlobsByRangeRequestId, BlocksByRangeRequestId, DataColumnsByRangeRequestId, + ExecutionProofsByRangeRequestId, }, }; use ssz_types::RuntimeVariableList; @@ -12,7 +13,7 @@ use std::{collections::HashMap, sync::Arc}; use tracing::{Span, debug}; use types::{ BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, - Hash256, SignedBeaconBlock, + ExecutionProof, Hash256, SignedBeaconBlock, }; use crate::sync::network_context::MAX_COLUMN_RETRIES; @@ -24,6 +25,7 @@ use crate::sync::network_context::MAX_COLUMN_RETRIES; /// - Blocks themselves (always required) /// - Blob sidecars (pre-Fulu fork) /// - Data columns (Fulu fork and later) +/// - Execution proofs (for zkvm-enabled nodes) /// /// It accumulates responses until all expected components are received, then couples /// them together and returns complete `RpcBlock`s ready for processing. Handles validation @@ -33,10 +35,25 @@ pub struct RangeBlockComponentsRequest { blocks_request: ByRangeRequest>>>, /// Sidecars we have received awaiting for their corresponding block. block_data_request: RangeBlockDataRequest, + /// Execution proofs request (for zkvm-enabled nodes). + execution_proofs_request: Option>, /// Span to track the range request and all children range requests. pub(crate) request_span: Span, } +/// Tracks execution proofs requests during range sync. +struct ExecutionProofsRequest { + /// The request tracking state. + request: ByRangeRequest>>, + /// The peer we requested proofs from. + peer: PeerId, + /// Number of proofs required per block. + min_proofs_required: usize, + /// Number of proof attempts completed for this batch. + attempt: usize, + _phantom: std::marker::PhantomData, +} + pub enum ByRangeRequest { Active(I), Complete(T), @@ -67,6 +84,12 @@ pub(crate) enum CouplingError { exceeded_retries: bool, }, BlobPeerFailure(String), + /// The peer we requested execution proofs from was faulty/malicious + ExecutionProofPeerFailure { + error: String, + peer: PeerId, + exceeded_retries: bool, + }, } impl RangeBlockComponentsRequest { @@ -76,6 +99,7 @@ impl RangeBlockComponentsRequest { /// * `blocks_req_id` - Request ID for the blocks /// * `blobs_req_id` - Optional request ID for blobs (pre-Fulu fork) /// * `data_columns` - Optional tuple of (request_id->column_indices pairs, expected_custody_columns) for Fulu fork + /// * `execution_proofs` - Optional tuple of (request_id, peer, min_proofs_required) for zkvm-enabled nodes #[allow(clippy::type_complexity)] pub fn new( blocks_req_id: BlocksByRangeRequestId, @@ -84,6 +108,7 @@ impl RangeBlockComponentsRequest { Vec<(DataColumnsByRangeRequestId, Vec)>, Vec, )>, + execution_proofs: Option<(ExecutionProofsByRangeRequestId, usize)>, request_span: Span, ) -> Self { let block_data_request = if let Some(blobs_req_id) = blobs_req_id { @@ -103,9 +128,19 @@ impl RangeBlockComponentsRequest { RangeBlockDataRequest::NoData }; + let execution_proofs_request = + execution_proofs.map(|(req_id, min_proofs_required)| ExecutionProofsRequest { + request: ByRangeRequest::Active(req_id), + peer: req_id.peer, + min_proofs_required, + attempt: 0, + _phantom: std::marker::PhantomData, + }); + Self { blocks_request: ByRangeRequest::Active(blocks_req_id), block_data_request, + execution_proofs_request, request_span, } } @@ -187,6 +222,30 @@ impl RangeBlockComponentsRequest { } } + /// Adds received execution proofs to the request. + /// + /// Returns an error if this request doesn't expect execution proofs, + /// or if the request ID doesn't match. + pub fn add_execution_proofs( + &mut self, + req_id: ExecutionProofsByRangeRequestId, + proofs: Vec>, + ) -> Result<(), String> { + match &mut self.execution_proofs_request { + Some(exec_proofs_req) => { + exec_proofs_req.request.finish(req_id, proofs)?; + exec_proofs_req.attempt += 1; + Ok(()) + } + None => Err("received execution proofs but none were expected".to_owned()), + } + } + + /// Returns true if this request expects execution proofs. + pub fn expects_execution_proofs(&self) -> bool { + self.execution_proofs_request.is_some() + } + /// Attempts to construct RPC blocks from all received components. /// /// Returns `None` if not all expected requests have completed. @@ -200,6 +259,13 @@ impl RangeBlockComponentsRequest { return None; }; + // Check if execution proofs are required but not yet complete + if let Some(exec_proofs_req) = &self.execution_proofs_request + && exec_proofs_req.request.to_finished().is_none() + { + return None; + } + // Increment the attempt once this function returns the response or errors match &mut self.block_data_request { RangeBlockDataRequest::NoData => { @@ -269,6 +335,50 @@ impl RangeBlockComponentsRequest { } } + /// Returns the collected execution proofs if available. + /// This should be called after `responses()` returns `Some`. + pub fn get_execution_proofs(&self) -> Option>> { + self.execution_proofs_request + .as_ref() + .and_then(|req| req.request.to_finished().cloned()) + } + + /// Returns the peer that was responsible for providing execution proofs. + pub fn execution_proofs_peer(&self) -> Option { + self.execution_proofs_request.as_ref().map(|req| req.peer) + } + + /// Returns the minimum number of execution proofs required per block, if any. + pub fn min_execution_proofs_required(&self) -> Option { + self.execution_proofs_request + .as_ref() + .map(|req| req.min_proofs_required) + } + + /// Returns the number of completed proof attempts for this batch, if any. + pub fn execution_proofs_attempt(&self) -> Option { + self.execution_proofs_request + .as_ref() + .map(|req| req.attempt) + } + + /// Resets the execution proofs request to retry with a new peer. + pub fn reinsert_execution_proofs_request( + &mut self, + req_id: ExecutionProofsByRangeRequestId, + min_proofs_required: usize, + ) -> Result<(), String> { + match &mut self.execution_proofs_request { + Some(exec_proofs_req) => { + exec_proofs_req.request = ByRangeRequest::Active(req_id); + exec_proofs_req.peer = req_id.peer; + exec_proofs_req.min_proofs_required = min_proofs_required; + Ok(()) + } + None => Err("execution proofs request not present".to_owned()), + } + } + fn responses_with_blobs( blocks: Vec>>, blobs: Vec>>, @@ -529,7 +639,7 @@ mod tests { let blocks_req_id = blocks_id(components_id()); let mut info = - RangeBlockComponentsRequest::::new(blocks_req_id, None, None, Span::none()); + RangeBlockComponentsRequest::::new(blocks_req_id, None, None, None, Span::none()); // Send blocks and complete terminate response info.add_blocks(blocks_req_id, blocks).unwrap(); @@ -557,6 +667,7 @@ mod tests { blocks_req_id, Some(blobs_req_id), None, + None, Span::none(), ); @@ -606,6 +717,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expects_custody_columns.clone())), + None, Span::none(), ); // Send blocks and complete terminate response @@ -674,6 +786,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expects_custody_columns.clone())), + None, Span::none(), ); @@ -762,6 +875,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); @@ -848,6 +962,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); @@ -941,6 +1056,7 @@ mod tests { blocks_req_id, None, Some((columns_req_id.clone(), expected_custody_columns.clone())), + None, Span::none(), ); diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index c0af69d7a40..6c41d3d9c75 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -61,7 +61,8 @@ use lighthouse_network::service::api_types::{ BlobsByRangeRequestId, BlocksByRangeRequestId, ComponentsByRangeRequestId, CustodyBackFillBatchRequestId, CustodyBackfillBatchId, CustodyRequester, DataColumnsByRangeRequestId, DataColumnsByRangeRequester, DataColumnsByRootRequestId, - DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, + DataColumnsByRootRequester, ExecutionProofsByRangeRequestId, Id, SingleLookupReqId, + SyncRequestId, }; use lighthouse_network::types::{NetworkGlobals, SyncState}; use lighthouse_network::{PeerAction, PeerId}; @@ -518,6 +519,8 @@ impl SyncManager { SyncRequestId::DataColumnsByRange(req_id) => { self.on_data_columns_by_range_response(req_id, peer_id, RpcEvent::RPCError(error)) } + SyncRequestId::ExecutionProofsByRange(req_id) => self + .on_execution_proofs_by_range_response(req_id, peer_id, RpcEvent::RPCError(error)), } } @@ -1225,6 +1228,12 @@ impl SyncManager { peer_id, RpcEvent::from_chunk(execution_proof, seen_timestamp), ), + SyncRequestId::ExecutionProofsByRange(req_id) => self + .on_execution_proofs_by_range_response( + req_id, + peer_id, + RpcEvent::from_chunk(execution_proof, seen_timestamp), + ), _ => { crit!(%peer_id, "bad request id for execution_proof"); } @@ -1352,6 +1361,28 @@ impl SyncManager { } } + /// Handles a response for an execution proofs by range request. + /// + /// Note: This is currently a stub. Execution proofs by range requests are not yet issued + /// during range sync. + fn on_execution_proofs_by_range_response( + &mut self, + id: ExecutionProofsByRangeRequestId, + peer_id: PeerId, + proof: RpcEvent>, + ) { + if let Some(resp) = self + .network + .on_execution_proofs_by_range_response(id, peer_id, proof) + { + self.on_range_components_response( + id.parent_request_id, + peer_id, + RangeBlockComponent::ExecutionProofs(id, resp), + ); + } + } + fn on_custody_by_root_result( &mut self, requester: CustodyRequester, diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 0943787c925..65ae25bfd3c 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -17,18 +17,21 @@ use crate::sync::block_lookups::SingleLookupId; use crate::sync::block_sidecar_coupling::CouplingError; use crate::sync::network_context::requests::BlobsByRootSingleBlockRequest; use crate::sync::range_data_column_batch_request::RangeDataColumnBatchRequest; -use beacon_chain::block_verification_types::RpcBlock; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessStatus, EngineState}; use custody::CustodyRequestResult; use fnv::FnvHashMap; -use lighthouse_network::rpc::methods::{BlobsByRangeRequest, DataColumnsByRangeRequest}; +use lighthouse_network::rpc::methods::{ + BlobsByRangeRequest, DataColumnsByRangeRequest, ExecutionProofsByRangeRequest, +}; use lighthouse_network::rpc::{BlocksByRangeRequest, GoodbyeReason, RPCError, RequestType}; pub use lighthouse_network::service::api_types::RangeRequestId; use lighthouse_network::service::api_types::{ AppRequestId, BlobsByRangeRequestId, BlocksByRangeRequestId, ComponentsByRangeRequestId, CustodyBackFillBatchRequestId, CustodyBackfillBatchId, CustodyId, CustodyRequester, DataColumnsByRangeRequestId, DataColumnsByRangeRequester, DataColumnsByRootRequestId, - DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, + DataColumnsByRootRequester, ExecutionProofsByRangeRequestId, Id, SingleLookupReqId, + SyncRequestId, }; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Subnet}; use lighthouse_tracing::{SPAN_OUTGOING_BLOCK_BY_ROOT_REQUEST, SPAN_OUTGOING_RANGE_REQUEST}; @@ -37,7 +40,8 @@ pub use requests::LookupVerifyError; use requests::{ ActiveRequests, BlobsByRangeRequestItems, BlobsByRootRequestItems, BlocksByRangeRequestItems, BlocksByRootRequestItems, DataColumnsByRangeRequestItems, DataColumnsByRootRequestItems, - ExecutionProofsByRootRequestItems, ExecutionProofsByRootSingleBlockRequest, + ExecutionProofsByRangeRequestItems, ExecutionProofsByRootRequestItems, + ExecutionProofsByRootSingleBlockRequest, }; #[cfg(test)] use slot_clock::SlotClock; @@ -73,6 +77,8 @@ macro_rules! new_range_request_span { /// Max retries for block components after which we fail the batch. pub const MAX_COLUMN_RETRIES: usize = 3; +/// Max retries for execution proofs after which we fail the batch. +pub const MAX_EXECUTION_PROOF_RETRIES: usize = 3; #[derive(Debug)] pub enum RpcEvent { @@ -118,6 +124,7 @@ pub enum RpcRequestSendError { pub enum NoPeerError { BlockPeer, CustodyPeer(ColumnIndex), + ExecutionProofPeer, } #[derive(Debug, PartialEq, Eq)] @@ -217,6 +224,9 @@ pub struct SyncNetworkContext { /// A mapping of active DataColumnsByRange requests data_columns_by_range_requests: ActiveRequests>, + /// A mapping of active ExecutionProofsByRange requests + execution_proofs_by_range_requests: + ActiveRequests, /// Mapping of active custody column requests for a block root custody_by_root_requests: FnvHashMap>, @@ -254,6 +264,17 @@ pub enum RangeBlockComponent { DataColumnsByRangeRequestId, RpcResponseResult>>>, ), + ExecutionProofs( + ExecutionProofsByRangeRequestId, + RpcResponseResult>>, + ), +} + +struct RangeExecutionProofInputs { + min_proofs_required: usize, + proofs_peer: PeerId, + proofs: Vec>, + attempt: usize, } #[cfg(test)] @@ -303,6 +324,7 @@ impl SyncNetworkContext { blocks_by_range_requests: ActiveRequests::new("blocks_by_range"), blobs_by_range_requests: ActiveRequests::new("blobs_by_range"), data_columns_by_range_requests: ActiveRequests::new("data_columns_by_range"), + execution_proofs_by_range_requests: ActiveRequests::new("execution_proofs_by_range"), custody_by_root_requests: <_>::default(), components_by_range_requests: FnvHashMap::default(), custody_backfill_data_column_batch_requests: FnvHashMap::default(), @@ -332,6 +354,7 @@ impl SyncNetworkContext { blocks_by_range_requests, blobs_by_range_requests, data_columns_by_range_requests, + execution_proofs_by_range_requests, // custody_by_root_requests is a meta request of data_columns_by_root_requests custody_by_root_requests: _, // components_by_range_requests is a meta request of various _by_range requests @@ -371,6 +394,10 @@ impl SyncNetworkContext { .active_requests_of_peer(peer_id) .into_iter() .map(|req_id| SyncRequestId::DataColumnsByRange(*req_id)); + let execution_proofs_by_range_ids = execution_proofs_by_range_requests + .active_requests_of_peer(peer_id) + .into_iter() + .map(|req_id| SyncRequestId::ExecutionProofsByRange(*req_id)); blocks_by_root_ids .chain(blobs_by_root_ids) .chain(data_column_by_root_ids) @@ -378,6 +405,7 @@ impl SyncNetworkContext { .chain(blocks_by_range_ids) .chain(blobs_by_range_ids) .chain(data_column_by_range_ids) + .chain(execution_proofs_by_range_ids) .collect() } @@ -435,6 +463,7 @@ impl SyncNetworkContext { blocks_by_range_requests, blobs_by_range_requests, data_columns_by_range_requests, + execution_proofs_by_range_requests, // custody_by_root_requests is a meta request of data_columns_by_root_requests custody_by_root_requests: _, // components_by_range_requests is a meta request of various _by_range requests @@ -458,6 +487,7 @@ impl SyncNetworkContext { .chain(blocks_by_range_requests.iter_request_peers()) .chain(blobs_by_range_requests.iter_request_peers()) .chain(data_columns_by_range_requests.iter_request_peers()) + .chain(execution_proofs_by_range_requests.iter_request_peers()) { *active_request_count_by_peer.entry(peer_id).or_default() += 1; } @@ -547,6 +577,83 @@ impl SyncNetworkContext { Ok(()) } + /// Retries execution proofs by range by requesting the proofs again from a different peer. + pub fn retry_execution_proofs_by_range( + &mut self, + id: Id, + peers: &HashSet, + peers_to_deprioritize: &HashSet, + request: BlocksByRangeRequest, + ) -> Result<(), String> { + let Some((requester, parent_request_span)) = self + .components_by_range_requests + .iter() + .find_map(|(key, value)| { + if key.id == id { + Some((key.requester, value.request_span.clone())) + } else { + None + } + }) + else { + return Err("request id not present".to_string()); + }; + + let active_request_count_by_peer = self.active_request_count_by_peer(); + + let proof_peer = self + .select_execution_proofs_peer( + peers, + &active_request_count_by_peer, + peers_to_deprioritize, + ) + .ok_or_else(|| "no zkvm-enabled peer available for execution proofs".to_string())?; + + let epoch = Slot::new(*request.start_slot()).epoch(T::EthSpec::slots_per_epoch()); + let min_proofs_required = self.chain.spec.zkvm_min_proofs_required().ok_or_else(|| { + "zkvm enabled but min proofs requirement is not configured".to_string() + })?; + if !self.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + return Err("execution proofs retry requested for pre-zkvm epoch".to_string()); + } + + debug!( + id, + ?requester, + ?proof_peer, + "Retrying execution proofs by range from a different peer" + ); + + let id = ComponentsByRangeRequestId { id, requester }; + let req_id = self + .send_execution_proofs_by_range_request( + proof_peer, + ExecutionProofsByRangeRequest { + start_slot: *request.start_slot(), + count: *request.count(), + }, + id, + new_range_request_span!( + self, + "outgoing_proofs_by_range_retry", + parent_request_span.clone(), + proof_peer + ), + ) + .map_err(|e| format!("{:?}", e))?; + + let Some(range_request) = self.components_by_range_requests.get_mut(&id) else { + return Err( + "retrying execution proofs for range request that does not exist".to_string(), + ); + }; + + range_request + .reinsert_execution_proofs_request(req_id, min_proofs_required) + .map_err(|e| format!("{e:?}"))?; + Ok(()) + } + /// A blocks by range request sent by the range sync algorithm pub fn block_components_by_range_request( &mut self, @@ -672,6 +779,43 @@ impl SyncNetworkContext { .transpose()?; let epoch = Slot::new(*request.start_slot()).epoch(T::EthSpec::slots_per_epoch()); + + // Request execution proofs only if zkvm is enabled for this epoch. + let execution_proofs_request = if self.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + let min_proofs_required = + self.chain.spec.zkvm_min_proofs_required().ok_or_else(|| { + RpcRequestSendError::InternalError( + "zkvm enabled but min proofs requirement is not configured".to_string(), + ) + })?; + + // Find a zkvm-enabled peer from block_peers or column_peers + let zkvm_peer = self.find_zkvm_enabled_peer(block_peers, column_peers); + + if let Some(proofs_peer) = zkvm_peer { + let proofs_request = ExecutionProofsByRangeRequest { + start_slot: *request.start_slot(), + count: *request.count(), + }; + let req_id = self.send_execution_proofs_by_range_request( + proofs_peer, + proofs_request, + id, + new_range_request_span!( + self, + "outgoing_proofs_by_range", + range_request_span.clone(), + proofs_peer + ), + )?; + Some((req_id, min_proofs_required)) + } else { + return Err(RpcRequestSendError::NoPeer(NoPeerError::ExecutionProofPeer)); + } + } else { + None + }; + let info = RangeBlockComponentsRequest::new( blocks_req_id, blobs_req_id, @@ -681,6 +825,7 @@ impl SyncNetworkContext { self.chain.sampling_columns_for_epoch(epoch).to_vec(), ) }), + execution_proofs_request, range_request_span, ); self.components_by_range_requests.insert(id, info); @@ -743,6 +888,33 @@ impl SyncNetworkContext { Ok(columns_to_request_by_peer) } + fn select_execution_proofs_peer( + &self, + peers: &HashSet, + active_request_count_by_peer: &HashMap, + peers_to_deprioritize: &HashSet, + ) -> Option { + let peers_db = self.network_globals().peers.read(); + peers + .iter() + .filter(|peer| { + peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + }) + .map(|peer| { + ( + peers_to_deprioritize.contains(peer), + active_request_count_by_peer.get(peer).copied().unwrap_or(0), + rand::random::(), + peer, + ) + }) + .min() + .map(|(_, _, _, peer)| *peer) + } + /// Received a blocks by range or blobs by range response for a request that couples blocks ' /// and blobs. pub fn range_block_component_response( @@ -750,13 +922,14 @@ impl SyncNetworkContext { id: ComponentsByRangeRequestId, range_block_component: RangeBlockComponent, ) -> Option>, RpcResponseError>> { - let Entry::Occupied(mut entry) = self.components_by_range_requests.entry(id) else { - metrics::inc_counter_vec(&metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, &["range_blocks"]); - return None; - }; - - if let Err(e) = { - let request = entry.get_mut(); + let add_result = { + let Some(request) = self.components_by_range_requests.get_mut(&id) else { + metrics::inc_counter_vec( + &metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, + &["range_blocks"], + ); + return None; + }; match range_block_component { RangeBlockComponent::Block(req_id, resp) => resp.and_then(|(blocks, _)| { request.add_blocks(req_id, blocks).map_err(|e| { @@ -783,14 +956,104 @@ impl SyncNetworkContext { }) }) } + RangeBlockComponent::ExecutionProofs(req_id, resp) => { + let expects_execution_proofs = request.expects_execution_proofs(); + // Handle execution proofs response, treating UnsupportedProtocol as an error + // if proofs are required. + let proofs = match resp { + Ok((proofs, _)) => proofs, + Err(RpcResponseError::RpcError(RPCError::UnsupportedProtocol)) + if expects_execution_proofs => + { + return Some(Err(RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + error: "Peer doesn't support execution_proofs_by_range" + .to_string(), + peer: req_id.peer, + exceeded_retries: false, + }, + ))); + } + Err(RpcResponseError::RpcError(RPCError::UnsupportedProtocol)) => { + debug!( + req_id = ?req_id, + "Peer doesn't support execution_proofs_by_range, treating as empty response" + ); + vec![] + } + Err(e) => return Some(Err(e)), + }; + request.add_execution_proofs(req_id, proofs).map_err(|e| { + RpcResponseError::BlockComponentCouplingError(CouplingError::InternalError( + e, + )) + }) + } } - } { - entry.remove(); + }; + + if let Err(e) = add_result { + self.components_by_range_requests.remove(&id); return Some(Err(e)); } - let range_req = entry.get_mut(); - if let Some(blocks_result) = range_req.responses(&self.chain.spec) { + let (blocks_result, min_proofs_required, proofs_peer, proofs, proofs_attempt) = { + let Some(range_req) = self.components_by_range_requests.get_mut(&id) else { + metrics::inc_counter_vec( + &metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, + &["range_blocks"], + ); + return None; + }; + let blocks_result = range_req.responses(&self.chain.spec); + let min_proofs_required = range_req.min_execution_proofs_required(); + let proofs_peer = range_req.execution_proofs_peer(); + let proofs = range_req.get_execution_proofs().unwrap_or_default(); + let proofs_attempt = range_req.execution_proofs_attempt().unwrap_or(0); + ( + blocks_result, + min_proofs_required, + proofs_peer, + proofs, + proofs_attempt, + ) + }; + + if let Some(Ok(blocks)) = &blocks_result + && let Some(min_proofs_required) = min_proofs_required + { + let Some(proofs_peer) = proofs_peer else { + self.components_by_range_requests.remove(&id); + return Some(Err(RpcResponseError::BlockComponentCouplingError( + CouplingError::InternalError( + "execution proofs request completed without a peer".to_string(), + ), + ))); + }; + let proof_inputs = RangeExecutionProofInputs { + min_proofs_required, + proofs_peer, + proofs, + attempt: proofs_attempt, + }; + if let Err(err) = self.process_range_execution_proofs(proof_inputs, blocks) { + let remove_entry = !matches!( + err, + RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + exceeded_retries: false, + .. + } + ) + ); + if remove_entry { + self.components_by_range_requests.remove(&id); + } + return Some(Err(err)); + } + } + + if let Some(blocks_result) = blocks_result { if let Err(CouplingError::DataColumnPeerFailure { error, faulty_peers: _, @@ -800,16 +1063,16 @@ impl SyncNetworkContext { // Remove the entry if it's a peer failure **and** retry counter is exceeded if *exceeded_retries { debug!( - entry=?entry.key(), + entry = ?id, msg = error, "Request exceeded max retries, failing batch" ); - entry.remove(); - }; + self.components_by_range_requests.remove(&id); + } } else { - // also remove the entry only if it coupled successfully + // Also remove the entry only if it coupled successfully // or if it isn't a column peer failure. - entry.remove(); + self.components_by_range_requests.remove(&id); } // If the request is finished, dequeue everything Some(blocks_result.map_err(RpcResponseError::BlockComponentCouplingError)) @@ -818,6 +1081,138 @@ impl SyncNetworkContext { } } + fn process_range_execution_proofs( + &self, + inputs: RangeExecutionProofInputs, + blocks: &[RpcBlock], + ) -> Result<(), RpcResponseError> { + let RangeExecutionProofInputs { + min_proofs_required, + proofs_peer, + proofs, + attempt, + } = inputs; + let exceeded_retries = attempt >= MAX_EXECUTION_PROOF_RETRIES; + let mut proofs_by_root: HashMap>> = HashMap::new(); + for proof in proofs { + proofs_by_root + .entry(proof.block_root) + .or_default() + .push(proof); + } + + let proof_error = |error: String| { + RpcResponseError::BlockComponentCouplingError( + CouplingError::ExecutionProofPeerFailure { + error, + peer: proofs_peer, + exceeded_retries, + }, + ) + }; + + for block in blocks { + let block_root = block.block_root(); + if !self.chain.spec.is_zkvm_enabled_for_epoch(block.epoch()) { + proofs_by_root.remove(&block_root); + continue; + } + let existing_count = self + .chain + .data_availability_checker + .get_existing_proof_ids(&block_root) + .map(|ids| ids.len()) + .unwrap_or(0); + + let proofs_for_block = proofs_by_root.remove(&block_root).unwrap_or_default(); + if existing_count >= min_proofs_required { + if !proofs_for_block.is_empty() { + debug!( + ?block_root, + existing_count, + min_proofs_required, + "Ignoring execution proofs because cache already satisfies requirement" + ); + } + continue; + } + + let payload_hash = block + .message() + .body() + .execution_payload() + .ok() + .map(|payload| payload.execution_payload_ref().block_hash()) + .ok_or_else(|| { + RpcResponseError::BlockComponentCouplingError(CouplingError::InternalError( + "execution payload missing for zkvm proofs".to_string(), + )) + })?; + + let mut verified_proofs = Vec::new(); + for proof in proofs_for_block { + if proof.block_root != block_root { + return Err(proof_error(format!( + "proof block_root mismatch: expected {block_root:?} got {:?}", + proof.block_root + ))); + } + if proof.block_hash != payload_hash { + return Err(proof_error(format!( + "proof execution payload hash mismatch for {block_root:?}" + ))); + } + match self + .chain + .data_availability_checker + .verify_execution_proof_for_gossip(&proof) + { + Ok(true) => verified_proofs.push((*proof).clone()), + Ok(false) => { + return Err(proof_error(format!( + "execution proof verification failed for {block_root:?}" + ))); + } + Err(e) => { + return Err(proof_error(format!( + "execution proof verification error for {block_root:?}: {e:?}" + ))); + } + } + } + + if !verified_proofs.is_empty() + && let Err(e) = self + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, verified_proofs) + { + return Err(proof_error(format!( + "failed to store execution proofs for {block_root:?}: {e:?}" + ))); + } + + let updated_count = self + .chain + .data_availability_checker + .get_existing_proof_ids(&block_root) + .map(|ids| ids.len()) + .unwrap_or(0); + if updated_count < min_proofs_required { + return Err(proof_error(format!( + "missing execution proofs for {block_root:?}: have {updated_count}, need {min_proofs_required}" + ))); + } + } + + if !proofs_by_root.is_empty() { + let unknown_roots: Vec<_> = proofs_by_root.keys().collect(); + debug!(?unknown_roots, "Execution proofs for unknown block roots"); + } + + Ok(()) + } + /// Request block of `block_root` if necessary by checking: /// - If the da_checker has a pending block from gossip or a previous request /// @@ -1402,6 +1797,75 @@ impl SyncNetworkContext { Ok((id, requested_columns)) } + /// Find a zkvm-enabled peer from the given peer sets. + /// + /// Peers advertise zkvm support via their ENR's zkvm flag. This function + /// checks both block_peers and column_peers to find any peer that supports + /// the execution_proofs_by_range protocol. + fn find_zkvm_enabled_peer( + &self, + block_peers: &HashSet, + column_peers: &HashSet, + ) -> Option { + let peers_db = self.network_globals().peers.read(); + + // First try block_peers, then column_peers + let all_peers = block_peers.iter().chain(column_peers.iter()); + + for peer in all_peers { + if peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + { + return Some(*peer); + } + } + + None + } + + fn send_execution_proofs_by_range_request( + &mut self, + peer_id: PeerId, + request: ExecutionProofsByRangeRequest, + parent_request_id: ComponentsByRangeRequestId, + request_span: Span, + ) -> Result { + let id = ExecutionProofsByRangeRequestId { + id: self.next_id(), + parent_request_id, + peer: peer_id, + }; + + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request: RequestType::ExecutionProofsByRange(request.clone()), + app_request_id: AppRequestId::Sync(SyncRequestId::ExecutionProofsByRange(id)), + }) + .map_err(|_| RpcRequestSendError::InternalError("network send error".to_owned()))?; + + debug!( + method = "ExecutionProofsByRange", + slots = request.count, + epoch = %Slot::new(request.start_slot).epoch(T::EthSpec::slots_per_epoch()), + peer = %peer_id, + %id, + "Sync RPC request sent" + ); + + self.execution_proofs_by_range_requests.insert( + id, + peer_id, + // false = do not enforce max_requests are returned for *_by_range methods. We don't + // know how many proofs to expect per block. + false, + ExecutionProofsByRangeRequestItems::new(request), + request_span, + ); + Ok(id) + } + pub fn is_execution_engine_online(&self) -> bool { self.execution_engine_state == EngineState::Online } @@ -1640,6 +2104,20 @@ impl SyncNetworkContext { self.on_rpc_response_result(id, "DataColumnsByRange", resp, peer_id, |d| d.len()) } + /// Handles a response for an execution proofs by range request. + #[allow(clippy::type_complexity)] + pub(crate) fn on_execution_proofs_by_range_response( + &mut self, + id: ExecutionProofsByRangeRequestId, + peer_id: PeerId, + rpc_event: RpcEvent>, + ) -> Option>>> { + let resp = self + .execution_proofs_by_range_requests + .on_response(id, rpc_event); + self.on_rpc_response_result(id, "ExecutionProofsByRange", resp, peer_id, |p| p.len()) + } + fn on_rpc_response_result usize>( &mut self, id: I, diff --git a/beacon_node/network/src/sync/network_context/requests.rs b/beacon_node/network/src/sync/network_context/requests.rs index 63249ed2a4b..238e551659d 100644 --- a/beacon_node/network/src/sync/network_context/requests.rs +++ b/beacon_node/network/src/sync/network_context/requests.rs @@ -15,6 +15,7 @@ pub use data_columns_by_range::DataColumnsByRangeRequestItems; pub use data_columns_by_root::{ DataColumnsByRootRequestItems, DataColumnsByRootSingleBlockRequest, }; +pub use execution_proofs_by_range::ExecutionProofsByRangeRequestItems; pub use execution_proofs_by_root::{ ExecutionProofsByRootRequestItems, ExecutionProofsByRootSingleBlockRequest, }; @@ -29,6 +30,7 @@ mod blocks_by_range; mod blocks_by_root; mod data_columns_by_range; mod data_columns_by_root; +mod execution_proofs_by_range; mod execution_proofs_by_root; #[derive(Debug, PartialEq, Eq, IntoStaticStr)] diff --git a/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs b/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs new file mode 100644 index 00000000000..179f08a6547 --- /dev/null +++ b/beacon_node/network/src/sync/network_context/requests/execution_proofs_by_range.rs @@ -0,0 +1,54 @@ +use super::{ActiveRequestItems, LookupVerifyError}; +use lighthouse_network::rpc::methods::ExecutionProofsByRangeRequest; +use std::sync::Arc; +use types::{ExecutionProof, Slot}; + +/// Accumulates results of an execution_proofs_by_range request. Only returns items after receiving +/// the stream termination. +pub struct ExecutionProofsByRangeRequestItems { + request: ExecutionProofsByRangeRequest, + items: Vec>, +} + +impl ExecutionProofsByRangeRequestItems { + pub fn new(request: ExecutionProofsByRangeRequest) -> Self { + Self { + request, + items: vec![], + } + } +} + +impl ActiveRequestItems for ExecutionProofsByRangeRequestItems { + type Item = Arc; + + fn add(&mut self, proof: Self::Item) -> Result { + let proof_slot = proof.slot; + + // Verify the proof is within the requested slot range + if proof_slot < Slot::new(self.request.start_slot) + || proof_slot >= Slot::new(self.request.start_slot + self.request.count) + { + return Err(LookupVerifyError::UnrequestedSlot(proof_slot)); + } + + // Check for duplicate proofs (same slot and proof_id) + if self + .items + .iter() + .any(|existing| existing.slot == proof_slot && existing.proof_id == proof.proof_id) + { + return Err(LookupVerifyError::DuplicatedProofIDs(proof.proof_id)); + } + + self.items.push(proof); + + // We don't know exactly how many proofs to expect (depends on block content), + // so we never return true here - rely on stream termination + Ok(false) + } + + fn consume(&mut self) -> Vec { + std::mem::take(&mut self.items) + } +} diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 4ce10e23ca1..251dff9ffb7 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -6,12 +6,14 @@ use crate::sync::batch::{ BatchConfig, BatchInfo, BatchOperationOutcome, BatchProcessingResult, BatchState, }; use crate::sync::block_sidecar_coupling::CouplingError; -use crate::sync::network_context::{RangeRequestId, RpcRequestSendError, RpcResponseError}; +use crate::sync::network_context::{ + NoPeerError, RangeRequestId, RpcRequestSendError, RpcResponseError, +}; use crate::sync::{BatchProcessResult, network_context::SyncNetworkContext}; use beacon_chain::BeaconChainTypes; use beacon_chain::block_verification_types::RpcBlock; use lighthouse_network::service::api_types::Id; -use lighthouse_network::{PeerAction, PeerId}; +use lighthouse_network::{PeerAction, PeerId, Subnet}; use lighthouse_tracing::SPAN_SYNCING_CHAIN; use logging::crit; use std::collections::{BTreeMap, HashSet, btree_map::Entry}; @@ -463,6 +465,12 @@ impl SyncingChain { // target when there is no sampling peers available. This is a valid state and should not // return an error. return Ok(KeepChain); + } else if !self.good_peers_on_execution_proof_subnet(self.processing_target, network) { + debug!( + src = "process_completed_batches", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); } else { // NOTE: It is possible that the batch doesn't exist for the processing id. This can happen // when we complete a batch and attempt to download a new batch but there are: @@ -944,6 +952,31 @@ impl SyncingChain { CouplingError::BlobPeerFailure(msg) => { tracing::debug!(?batch_id, msg, "Blob peer failure"); } + CouplingError::ExecutionProofPeerFailure { + error, + peer, + exceeded_retries, + } => { + tracing::debug!(?batch_id, ?peer, error, "Execution proof peer failure"); + if !*exceeded_retries { + if let BatchOperationOutcome::Failed { blacklist } = + batch.downloading_to_awaiting_download()? + { + return Err(RemoveChain::ChainFailed { + blacklist, + failing_batch: batch_id, + }); + } + let mut failed_peers = HashSet::new(); + failed_peers.insert(*peer); + return self.retry_execution_proof_batch( + network, + batch_id, + request_id, + failed_peers, + ); + } + } CouplingError::InternalError(msg) => { tracing::error!(?batch_id, msg, "Block components coupling internal error"); } @@ -1020,6 +1053,13 @@ impl SyncingChain { for batch_id in awaiting_downloads { if self.good_peers_on_sampling_subnets(batch_id, network) { + if !self.good_peers_on_execution_proof_subnet(batch_id, network) { + debug!( + src = "attempt_send_awaiting_download_batches", + "Waiting for zkvm-enabled peers for execution proofs" + ); + continue; + } self.send_batch(network, batch_id)?; } else { debug!( @@ -1083,6 +1123,13 @@ impl SyncingChain { return Ok(KeepChain); } Err(e) => match e { + RpcRequestSendError::NoPeer(NoPeerError::ExecutionProofPeer) => { + debug!( + %batch_id, + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); + } // TODO(das): Handle the NoPeer case explicitly and don't drop the batch. For // sync to work properly it must be okay to have "stalled" batches in // AwaitingDownload state. Currently it will error with invalid state if @@ -1163,6 +1210,45 @@ impl SyncingChain { Ok(KeepChain) } + /// Retries execution proof requests within the batch by creating a new proofs request. + fn retry_execution_proof_batch( + &mut self, + network: &mut SyncNetworkContext, + batch_id: BatchId, + id: Id, + mut failed_peers: HashSet, + ) -> ProcessingResult { + let _guard = self.span.clone().entered(); + debug!(%batch_id, %id, ?failed_peers, "Retrying execution proof requests"); + if let Some(batch) = self.batches.get_mut(&batch_id) { + failed_peers.extend(&batch.failed_peers()); + let req = batch.to_blocks_by_range_request().0; + + let synced_peers = network + .network_globals() + .peers + .read() + .synced_peers_for_epoch(batch_id) + .cloned() + .collect::>(); + + match network.retry_execution_proofs_by_range(id, &synced_peers, &failed_peers, req) { + Ok(()) => { + batch.start_downloading(id)?; + debug!( + ?batch_id, + id, "Retried execution proof requests from other peers" + ); + return Ok(KeepChain); + } + Err(e) => { + debug!(?batch_id, id, e, "Failed to retry execution proof batch"); + } + } + } + Ok(KeepChain) + } + /// Returns true if this chain is currently syncing. pub fn is_syncing(&self) -> bool { match self.state { @@ -1206,6 +1292,13 @@ impl SyncingChain { ); return Ok(KeepChain); } + if !self.good_peers_on_execution_proof_subnet(epoch, network) { + debug!( + src = "request_batches_optimistic", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return Ok(KeepChain); + } if let Entry::Vacant(entry) = self.batches.entry(epoch) { let batch_type = network.batch_type(epoch); @@ -1252,6 +1345,27 @@ impl SyncingChain { } } + /// Returns true if there is at least one zkvm-enabled peer for execution proofs. + fn good_peers_on_execution_proof_subnet( + &self, + epoch: Epoch, + network: &SyncNetworkContext, + ) -> bool { + if !network.chain.spec.is_zkvm_enabled_for_epoch(epoch) { + return true; + } + + let peers_db = network.network_globals().peers.read(); + let synced_peers: HashSet<_> = peers_db.synced_peers_for_epoch(epoch).cloned().collect(); + + self.peers.iter().chain(synced_peers.iter()).any(|peer| { + peers_db + .peer_info(peer) + .map(|info| info.on_subnet_metadata(&Subnet::ExecutionProof)) + .unwrap_or(false) + }) + } + /// Creates the next required batch from the chain. If there are no more batches required, /// `false` is returned. fn include_next_batch(&mut self, network: &mut SyncNetworkContext) -> Option { @@ -1294,6 +1408,13 @@ impl SyncingChain { ); return None; } + if !self.good_peers_on_execution_proof_subnet(self.to_be_downloaded, network) { + debug!( + src = "include_next_batch", + "Waiting for zkvm-enabled peers for execution proofs" + ); + return None; + } // If no batch needs a retry, attempt to send the batch of the next epoch to download let next_batch_id = self.to_be_downloaded; diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 8e190da2b9d..fb4adbcee65 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -61,8 +61,16 @@ impl TestRig { } fn test_setup_with_spec(spec: ChainSpec) -> Self { + Self::test_setup_with_spec_and_zkvm(spec, false, None) + } + + fn test_setup_with_spec_and_zkvm( + spec: ChainSpec, + zkvm_dummy_verifiers: bool, + anchor_oldest_slot: Option, + ) -> Self { // Initialise a new beacon chain - let harness = BeaconChainHarness::>::builder(E) + let mut builder = BeaconChainHarness::>::builder(E) .spec(Arc::new(spec)) .deterministic_keypairs(1) .fresh_ephemeral_store() @@ -71,8 +79,23 @@ impl TestRig { Slot::new(0), Duration::from_secs(0), Duration::from_secs(12), - )) - .build(); + )); + + if zkvm_dummy_verifiers { + // TODO(zkproofs): For unit tests, we likely always want dummy verifiers + builder = builder.zkvm_with_dummy_verifiers(); + } + + let harness = builder.build(); + if let Some(oldest_slot) = anchor_oldest_slot { + let store = &harness.chain.store; + let prev_anchor = store.get_anchor_info(); + let mut new_anchor = prev_anchor.clone(); + new_anchor.oldest_block_slot = oldest_slot; + store + .compare_and_set_anchor_info_with_write(prev_anchor, new_anchor) + .expect("anchor info updated"); + } let chain = harness.chain.clone(); let fork_context = Arc::new(ForkContext::new::( @@ -160,7 +183,20 @@ impl TestRig { pub fn test_setup_after_fulu_with_zkvm() -> Option { let mut spec = test_spec::(); spec.zkvm_enabled = true; - let r = Self::test_setup_with_spec(spec); + let r = Self::test_setup_with_spec_and_zkvm(spec, true, None); + if r.fork_name.fulu_enabled() { + Some(r) + } else { + None + } + } + + /// Setup test rig for Fulu with zkvm enabled and backfill required. + pub fn test_setup_after_fulu_with_zkvm_backfill() -> Option { + let mut spec = test_spec::(); + spec.zkvm_enabled = true; + let backfill_start_slot = Slot::new(E::slots_per_epoch()); + let r = Self::test_setup_with_spec_and_zkvm(spec, true, Some(backfill_start_slot)); if r.fork_name.fulu_enabled() { Some(r) } else { diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index cb728a90c1b..f122087ae3c 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -3,26 +3,26 @@ use crate::network_beacon_processor::ChainSegmentProcessId; use crate::status::ToStatusMessage; use crate::sync::SyncMessage; use crate::sync::manager::SLOT_IMPORT_TOLERANCE; -use crate::sync::network_context::RangeRequestId; +use crate::sync::network_context::{MAX_EXECUTION_PROOF_RETRIES, RangeRequestId}; use crate::sync::range_sync::RangeSyncType; use beacon_chain::data_column_verification::CustodyDataColumn; use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; use beacon_chain::{EngineState, NotifyExecutionLayer, block_verification_types::RpcBlock}; use beacon_processor::WorkType; -use lighthouse_network::rpc::RequestType; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, DataColumnsByRangeRequest, OldBlocksByRangeRequest, - OldBlocksByRangeRequestV2, StatusMessageV2, + BlobsByRangeRequest, DataColumnsByRangeRequest, ExecutionProofsByRangeRequest, + OldBlocksByRangeRequest, OldBlocksByRangeRequestV2, StatusMessageV2, }; +use lighthouse_network::rpc::{RPCError, RequestType}; use lighthouse_network::service::api_types::{ AppRequestId, BlobsByRangeRequestId, BlocksByRangeRequestId, DataColumnsByRangeRequestId, - SyncRequestId, + ExecutionProofsByRangeRequestId, SyncRequestId, }; use lighthouse_network::{PeerId, SyncInfo}; use std::time::Duration; use types::{ - BlobSidecarList, BlockImportSource, Epoch, EthSpec, Hash256, MinimalEthSpec as E, - SignedBeaconBlock, SignedBeaconBlockHash, Slot, + BlobSidecarList, BlockImportSource, Epoch, EthSpec, ExecutionBlockHash, ExecutionProof, + ExecutionProofId, Hash256, MinimalEthSpec as E, SignedBeaconBlock, SignedBeaconBlockHash, Slot, }; const D: Duration = Duration::new(0, 0); @@ -38,6 +38,13 @@ enum ByRangeDataRequestIds { PostPeerDAS(Vec<(DataColumnsByRangeRequestId, PeerId)>), } +struct BlocksByRangeRequestMeta { + id: BlocksByRangeRequestId, + peer: PeerId, + start_slot: u64, + count: u64, +} + /// Sync tests are usually written in the form: /// - Do some action /// - Expect a request to be sent @@ -84,6 +91,20 @@ impl TestRig { }) } + fn add_head_zkvm_peer_with_root(&mut self, head_root: Hash256) -> PeerId { + let local_info = self.local_info(); + let peer_id = self.new_connected_zkvm_peer(); + self.send_sync_message(SyncMessage::AddPeer( + peer_id, + SyncInfo { + head_root, + head_slot: local_info.head_slot + 1 + Slot::new(SLOT_IMPORT_TOLERANCE as u64), + ..local_info + }, + )); + peer_id + } + // Produce a finalized peer with an advanced finalized epoch fn add_finalized_peer(&mut self) -> PeerId { self.add_finalized_peer_with_root(Hash256::random()) @@ -155,6 +176,13 @@ impl TestRig { } } + fn add_synced_zkvm_peer(&mut self) -> PeerId { + let peer_id = self.new_connected_zkvm_peer(); + let local_info = self.local_info(); + self.send_sync_message(SyncMessage::AddPeer(peer_id, local_info)); + peer_id + } + fn assert_state(&self, state: RangeSyncType) { assert_eq!( self.sync_manager @@ -200,6 +228,16 @@ impl TestRig { &mut self, request_filter: RequestFilter, ) -> ((BlocksByRangeRequestId, PeerId), ByRangeDataRequestIds) { + let (meta, by_range_data_requests) = + self.find_blocks_by_range_request_with_meta(request_filter); + + ((meta.id, meta.peer), by_range_data_requests) + } + + fn find_blocks_by_range_request_with_meta( + &mut self, + request_filter: RequestFilter, + ) -> (BlocksByRangeRequestMeta, ByRangeDataRequestIds) { let filter_f = |peer: PeerId, start_slot: u64| { if let Some(expected_epoch) = request_filter.epoch { let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); @@ -222,10 +260,17 @@ impl TestRig { peer_id, request: RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( - OldBlocksByRangeRequestV2 { start_slot, .. }, + OldBlocksByRangeRequestV2 { + start_slot, count, .. + }, )), app_request_id: AppRequestId::Sync(SyncRequestId::BlocksByRange(id)), - } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), + } if filter_f(*peer_id, *start_slot) => Some(BlocksByRangeRequestMeta { + id: *id, + peer: *peer_id, + start_slot: *start_slot, + count: *count, + }), _ => None, }) .unwrap_or_else(|e| { @@ -272,6 +317,45 @@ impl TestRig { (block_req, by_range_data_requests) } + fn find_execution_proofs_by_range_request( + &mut self, + request_filter: RequestFilter, + ) -> (ExecutionProofsByRangeRequestId, PeerId, u64, u64) { + let filter_f = |peer: PeerId, start_slot: u64| { + if let Some(expected_epoch) = request_filter.epoch { + let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); + if epoch != expected_epoch { + return false; + } + } + if let Some(expected_peer) = request_filter.peer + && peer != expected_peer + { + return false; + } + + true + }; + + self.pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + peer_id, + request: + RequestType::ExecutionProofsByRange(ExecutionProofsByRangeRequest { + start_slot, + count, + }), + app_request_id: AppRequestId::Sync(SyncRequestId::ExecutionProofsByRange(id)), + } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id, *start_slot, *count)), + _ => None, + }) + .unwrap_or_else(|e| { + panic!( + "Should have an ExecutionProofsByRange request, filter {request_filter:?}: {e:?}" + ) + }) + } + fn find_and_complete_blocks_by_range_request( &mut self, request_filter: RequestFilter, @@ -290,6 +374,15 @@ impl TestRig { seen_timestamp: D, }); + self.complete_by_range_data_requests(by_range_data_request_ids); + + blocks_req_id.parent_request_id.requester + } + + fn complete_by_range_data_requests( + &mut self, + by_range_data_request_ids: ByRangeDataRequestIds, + ) { match by_range_data_request_ids { ByRangeDataRequestIds::PreDeneb => {} ByRangeDataRequestIds::PrePeerDAS(id, peer_id) => { @@ -319,8 +412,6 @@ impl TestRig { } } } - - blocks_req_id.parent_request_id.requester } fn find_and_complete_processing_chain_segment(&mut self, id: ChainSegmentProcessId) { @@ -601,3 +692,611 @@ fn finalized_sync_not_enough_custody_peers_on_start() { let last_epoch = advanced_epochs + EXTRA_SYNCED_EPOCHS; r.complete_and_process_range_sync_until(last_epoch, filter()); } + +#[test] +fn range_sync_requests_execution_proofs_for_zkvm() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + rig.assert_state(RangeSyncType::Head); + rig.expect_empty_network(); + + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _ = rig.find_blocks_by_range_request(filter()); + let (_, proof_peer, _, _) = + rig.find_execution_proofs_by_range_request(filter().peer(zkvm_peer)); + assert_eq!(proof_peer, zkvm_peer); +} + +#[test] +fn range_sync_uses_cached_execution_proofs() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let (proof_req_id, proof_peer, proof_start_slot, proof_count) = + rig.find_execution_proofs_by_range_request(filter().peer(zkvm_peer)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert_eq!(proof_count, block_meta.count); + assert_eq!(proof_peer, zkvm_peer); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let min_proofs = rig + .harness + .chain + .spec + .zkvm_min_proofs_required() + .expect("zkvm enabled"); + + let proofs = (0..min_proofs) + .map(|i| { + ExecutionProof::new( + ExecutionProofId::new(u8::try_from(i).expect("proof id fits")).unwrap(), + block.slot(), + block_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap() + }) + .collect::>(); + + rig.harness + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, proofs) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + let request_id = block_meta.id.parent_request_id.requester; + let process_id = match request_id { + RangeRequestId::RangeSync { chain_id, batch_id } => { + ChainSegmentProcessId::RangeBatchId(chain_id, batch_id) + } + RangeRequestId::BackfillSync { batch_id } => { + ChainSegmentProcessId::BackSyncBatchId(batch_id) + } + }; + rig.find_and_complete_processing_chain_segment(process_id); +} + +#[test] +fn range_sync_retries_execution_proofs_without_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry for execution proof failure"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn backfill_retries_execution_proofs_without_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm_backfill() else { + return; + }; + + let zkvm_peer_1 = rig.add_synced_zkvm_peer(); + let zkvm_peer_2 = rig.add_synced_zkvm_peer(); + let local_info = rig.local_info(); + let _supernode_peer = rig.add_supernode_peer(local_info); + + let backfill_epoch = Slot::new(E::slots_per_epoch()) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (block_meta, by_range_data) = + rig.find_blocks_by_range_request_with_meta(filter().epoch(backfill_epoch)); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(backfill_epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry for execution proof failure"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(backfill_epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn range_sync_execution_proof_retries_exhaust_then_block_retry() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_3 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (mut proof_req_id, mut proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2 || proof_peer == zkvm_peer_3); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + rig.complete_by_range_data_requests(by_range_data); + + for attempt in 1..=MAX_EXECUTION_PROOF_RETRIES { + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if attempt < MAX_EXECUTION_PROOF_RETRIES { + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry before proof retries are exhausted"); + } + + let (next_req_id, next_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(next_peer, proof_peer); + proof_req_id = next_req_id; + proof_peer = next_peer; + } + } + + rig.pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2(OldBlocksByRangeRequestV2 { + start_slot, + .. + })), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .unwrap_or_else(|e| panic!("Expected BlocksByRange retry after exhausted proofs: {e}")); +} + +#[test] +fn range_sync_proof_retry_on_unsupported_protocol() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer_1 = rig.add_head_zkvm_peer_with_root(head_root); + let zkvm_peer_2 = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert!(proof_peer == zkvm_peer_1 || proof_peer == zkvm_peer_2); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + rig.complete_by_range_data_requests(by_range_data); + + rig.send_sync_message(SyncMessage::RpcError { + peer_id: proof_peer, + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + error: RPCError::UnsupportedProtocol, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected BlocksByRange retry on unsupported protocol"); + } + + let (_, retry_peer, retry_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + assert_eq!(retry_start_slot, block_meta.start_slot); + assert_ne!(retry_peer, proof_peer); +} + +#[test] +fn range_sync_ignores_bad_proofs_when_cached() { + let Some(mut rig) = TestRig::test_setup_after_fulu_with_zkvm() else { + return; + }; + + let head_root = Hash256::random(); + let zkvm_peer = rig.add_head_zkvm_peer_with_root(head_root); + let _supernode_peer = rig.add_head_peer_with_root(head_root); + + let (block_meta, by_range_data) = rig.find_blocks_by_range_request_with_meta(filter()); + let epoch = Slot::new(block_meta.start_slot) + .epoch(E::slots_per_epoch()) + .as_u64(); + let (proof_req_id, proof_peer, proof_start_slot, _) = + rig.find_execution_proofs_by_range_request(filter().epoch(epoch)); + + assert_eq!(proof_start_slot, block_meta.start_slot); + assert_eq!(proof_peer, zkvm_peer); + + let mut block = rig.rand_block(); + *block.message_mut().slot_mut() = Slot::new(block_meta.start_slot); + + let block_root = block.canonical_root(); + let block_hash = block + .message() + .body() + .execution_payload() + .expect("execution payload should exist") + .execution_payload_ref() + .block_hash(); + + let min_proofs = rig + .harness + .chain + .spec + .zkvm_min_proofs_required() + .expect("zkvm enabled"); + + let proofs = (0..min_proofs) + .map(|i| { + ExecutionProof::new( + ExecutionProofId::new(u8::try_from(i).expect("proof id fits")).unwrap(), + block.slot(), + block_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap() + }) + .collect::>(); + + rig.harness + .chain + .data_availability_checker + .put_verified_execution_proofs(block_root, proofs) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: Some(Arc::new(block)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcBlock { + sync_request_id: SyncRequestId::BlocksByRange(block_meta.id), + peer_id: block_meta.peer, + beacon_block: None, + seen_timestamp: D, + }); + + rig.complete_by_range_data_requests(by_range_data); + + let wrong_hash = if block_hash == ExecutionBlockHash::zero() { + ExecutionBlockHash::repeat_byte(0x11) + } else { + ExecutionBlockHash::zero() + }; + + let bad_proof = ExecutionProof::new( + ExecutionProofId::new(0).unwrap(), + Slot::new(block_meta.start_slot), + wrong_hash, + block_root, + vec![1, 2, 3], + ) + .unwrap(); + + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: Some(Arc::new(bad_proof)), + seen_timestamp: D, + }); + rig.send_sync_message(SyncMessage::RpcExecutionProof { + sync_request_id: SyncRequestId::ExecutionProofsByRange(proof_req_id), + peer_id: proof_peer, + execution_proof: None, + seen_timestamp: D, + }); + + if rig + .pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + request: + RequestType::ExecutionProofsByRange(ExecutionProofsByRangeRequest { + start_slot, + .. + }), + .. + } if *start_slot == block_meta.start_slot => Some(()), + _ => None, + }) + .is_ok() + { + panic!("unexpected execution proof retry when cache already satisfies requirement"); + } +} diff --git a/beacon_node/store/src/errors.rs b/beacon_node/store/src/errors.rs index a07cc838863..9cb6620816e 100644 --- a/beacon_node/store/src/errors.rs +++ b/beacon_node/store/src/errors.rs @@ -32,6 +32,8 @@ pub enum Error { BlobInfoConcurrentMutation, /// The store's `data_column_info` was mutated concurrently, the latest modification wasn't applied. DataColumnInfoConcurrentMutation, + /// The store's `execution_proof_info` was mutated concurrently, the latest modification wasn't applied. + ExecutionProofInfoConcurrentMutation, /// The block or state is unavailable due to weak subjectivity sync. HistoryUnavailable, /// State reconstruction cannot commence because not all historic blocks are known. @@ -92,6 +94,7 @@ pub enum Error { LoadSplit(Box), LoadBlobInfo(Box), LoadDataColumnInfo(Box), + LoadExecutionProofInfo(Box), LoadConfig(Box), LoadHotStateSummary(Hash256, Box), LoadHotStateSummaryForSplit(Box), diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index c4137191744..a05d915795f 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -9,12 +9,13 @@ use crate::metadata::{ ANCHOR_INFO_KEY, ANCHOR_UNINITIALIZED, AnchorInfo, BLOB_INFO_KEY, BlobInfo, COMPACTION_TIMESTAMP_KEY, CONFIG_KEY, CURRENT_SCHEMA_VERSION, CompactionTimestamp, DATA_COLUMN_CUSTODY_INFO_KEY, DATA_COLUMN_INFO_KEY, DataColumnCustodyInfo, DataColumnInfo, - SCHEMA_VERSION_KEY, SPLIT_KEY, STATE_UPPER_LIMIT_NO_RETAIN, SchemaVersion, + EXECUTION_PROOF_INFO_KEY, ExecutionProofInfo, SCHEMA_VERSION_KEY, SPLIT_KEY, + STATE_UPPER_LIMIT_NO_RETAIN, SchemaVersion, }; use crate::state_cache::{PutStateOutcome, StateCache}; use crate::{ BlobSidecarListFromRoot, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp, StoreItem, - StoreOp, get_data_column_key, + StoreOp, get_data_column_key, get_execution_proof_key, metrics::{self, COLD_METRIC, HOT_METRIC}, parse_data_column_key, }; @@ -61,6 +62,8 @@ pub struct HotColdDB, Cold: ItemStore> { blob_info: RwLock, /// The starting slots for the range of data columns stored in the database. data_column_info: RwLock, + /// The starting slots for the range of execution proofs stored in the database. + execution_proof_info: RwLock, pub(crate) config: StoreConfig, pub hierarchy: HierarchyModuli, /// Cold database containing compact historical data. @@ -93,6 +96,7 @@ struct BlockCache { block_cache: LruCache>, blob_cache: LruCache>, data_column_cache: LruCache>>>, + execution_proof_cache: LruCache>>, data_column_custody_info_cache: Option, } @@ -102,6 +106,7 @@ impl BlockCache { block_cache: LruCache::new(size), blob_cache: LruCache::new(size), data_column_cache: LruCache::new(size), + execution_proof_cache: LruCache::new(size), data_column_custody_info_cache: None, } } @@ -116,6 +121,9 @@ impl BlockCache { .get_or_insert_mut(block_root, Default::default) .insert(data_column.index, data_column); } + pub fn put_execution_proofs(&mut self, block_root: Hash256, proofs: Vec>) { + self.execution_proof_cache.put(block_root, proofs); + } pub fn put_data_column_custody_info( &mut self, data_column_custody_info: Option, @@ -139,6 +147,12 @@ impl BlockCache { .get(block_root) .and_then(|map| map.get(column_index).cloned()) } + pub fn get_execution_proofs( + &mut self, + block_root: &Hash256, + ) -> Option>> { + self.execution_proof_cache.get(block_root).cloned() + } pub fn get_data_column_custody_info(&self) -> Option { self.data_column_custody_info_cache.clone() } @@ -151,10 +165,14 @@ impl BlockCache { pub fn delete_data_columns(&mut self, block_root: &Hash256) { let _ = self.data_column_cache.pop(block_root); } + pub fn delete_execution_proofs(&mut self, block_root: &Hash256) { + let _ = self.execution_proof_cache.pop(block_root); + } pub fn delete(&mut self, block_root: &Hash256) { self.delete_block(block_root); self.delete_blobs(block_root); self.delete_data_columns(block_root); + self.delete_execution_proofs(block_root); } } @@ -232,6 +250,7 @@ impl HotColdDB, MemoryStore> { anchor_info: RwLock::new(ANCHOR_UNINITIALIZED), blob_info: RwLock::new(BlobInfo::default()), data_column_info: RwLock::new(DataColumnInfo::default()), + execution_proof_info: RwLock::new(ExecutionProofInfo::default()), cold_db: MemoryStore::open(), blobs_db: MemoryStore::open(), hot_db: MemoryStore::open(), @@ -286,6 +305,7 @@ impl HotColdDB, BeaconNodeBackend> { anchor_info, blob_info: RwLock::new(BlobInfo::default()), data_column_info: RwLock::new(DataColumnInfo::default()), + execution_proof_info: RwLock::new(ExecutionProofInfo::default()), blobs_db: BeaconNodeBackend::open(&config, blobs_db_path)?, cold_db: BeaconNodeBackend::open(&config, cold_path)?, hot_db, @@ -395,10 +415,38 @@ impl HotColdDB, BeaconNodeBackend> { new_data_column_info.clone(), )?; + // Initialize execution proof info + let execution_proof_info = db.load_execution_proof_info()?; + let zkvm_fork_slot = db + .spec + .zkvm_fork_epoch() + .map(|epoch| epoch.start_slot(E::slots_per_epoch())); + let new_execution_proof_info = match &execution_proof_info { + Some(execution_proof_info) => { + // Set the oldest execution proof slot to the fork slot if it is not yet set. + let oldest_execution_proof_slot = execution_proof_info + .oldest_execution_proof_slot + .or(zkvm_fork_slot); + ExecutionProofInfo { + oldest_execution_proof_slot, + } + } + // First start. + None => ExecutionProofInfo { + // Set the oldest execution proof slot to the fork slot if it is not yet set. + oldest_execution_proof_slot: zkvm_fork_slot, + }, + }; + db.compare_and_set_execution_proof_info_with_write( + <_>::default(), + new_execution_proof_info.clone(), + )?; + info!( path = ?blobs_db_path, oldest_blob_slot = ?new_blob_info.oldest_blob_slot, oldest_data_column_slot = ?new_data_column_info.oldest_data_column_slot, + oldest_execution_proof_slot = ?new_execution_proof_info.oldest_execution_proof_slot, "Blob DB initialized" ); @@ -1027,6 +1075,47 @@ impl, Cold: ItemStore> HotColdDB } } + /// Store execution proofs for a block. + pub fn put_execution_proofs( + &self, + block_root: &Hash256, + proofs: &[ExecutionProof], + ) -> Result<(), Error> { + for proof in proofs { + self.blobs_db.put_bytes( + DBColumn::BeaconExecutionProof, + &get_execution_proof_key(block_root, proof.proof_id.as_u8()), + &proof.as_ssz_bytes(), + )?; + } + if !proofs.is_empty() { + let cached = proofs + .iter() + .map(|proof| Arc::new(proof.clone())) + .collect::>(); + self.block_cache + .as_ref() + .inspect(|cache| cache.lock().put_execution_proofs(*block_root, cached)); + } + Ok(()) + } + + /// Create key-value store operations for storing execution proofs. + pub fn execution_proofs_as_kv_store_ops( + &self, + block_root: &Hash256, + proofs: &[ExecutionProof], + ops: &mut Vec, + ) { + for proof in proofs { + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconExecutionProof, + get_execution_proof_key(block_root, proof.proof_id.as_u8()), + proof.as_ssz_bytes(), + )); + } + } + /// Store a state in the store. pub fn put_state(&self, state_root: &Hash256, state: &BeaconState) -> Result<(), Error> { let mut ops: Vec = Vec::new(); @@ -2558,6 +2647,47 @@ impl, Cold: ItemStore> HotColdDB } } + /// Fetch all execution proofs for a given block from the store. + pub fn get_execution_proofs( + &self, + block_root: &Hash256, + ) -> Result>, Error> { + if let Some(proofs) = self + .block_cache + .as_ref() + .and_then(|cache| cache.lock().get_execution_proofs(block_root)) + { + return Ok(proofs); + } + + let mut proofs = Vec::new(); + let prefix = block_root.as_slice(); + + for result in self + .blobs_db + .iter_column_from::>(DBColumn::BeaconExecutionProof, prefix) + { + let (key, value) = result?; + // Check if key starts with our block_root prefix + if !key.starts_with(prefix) { + // We've moved past this block's proofs + break; + } + let proof = Arc::new(ExecutionProof::from_ssz_bytes(&value)?); + proofs.push(proof); + } + + if !proofs.is_empty() { + self.block_cache.as_ref().inspect(|cache| { + cache + .lock() + .put_execution_proofs(*block_root, proofs.clone()) + }); + } + + Ok(proofs) + } + /// Fetch all keys in the data_column column with prefix `block_root` pub fn get_data_column_keys(&self, block_root: Hash256) -> Result, Error> { self.blobs_db @@ -2877,6 +3007,77 @@ impl, Cold: ItemStore> HotColdDB data_column_info.as_kv_store_op(DATA_COLUMN_INFO_KEY) } + /// Get a clone of the store's execution proof info. + /// + /// To do mutations, use `compare_and_set_execution_proof_info`. + pub fn get_execution_proof_info(&self) -> ExecutionProofInfo { + self.execution_proof_info.read_recursive().clone() + } + + /// Initialize the `ExecutionProofInfo` when starting from genesis or a checkpoint. + pub fn init_execution_proof_info(&self, anchor_slot: Slot) -> Result { + let oldest_execution_proof_slot = self.spec.zkvm_fork_epoch().map(|fork_epoch| { + std::cmp::max(anchor_slot, fork_epoch.start_slot(E::slots_per_epoch())) + }); + let execution_proof_info = ExecutionProofInfo { + oldest_execution_proof_slot, + }; + self.compare_and_set_execution_proof_info( + self.get_execution_proof_info(), + execution_proof_info, + ) + } + + /// Atomically update the execution proof info from `prev_value` to `new_value`. + /// + /// Return a `KeyValueStoreOp` which should be written to disk, possibly atomically with other + /// values. + /// + /// Return an `ExecutionProofInfoConcurrentMutation` error if the `prev_value` provided + /// is not correct. + pub fn compare_and_set_execution_proof_info( + &self, + prev_value: ExecutionProofInfo, + new_value: ExecutionProofInfo, + ) -> Result { + let mut execution_proof_info = self.execution_proof_info.write(); + if *execution_proof_info == prev_value { + let kv_op = self.store_execution_proof_info_in_batch(&new_value); + *execution_proof_info = new_value; + Ok(kv_op) + } else { + Err(Error::ExecutionProofInfoConcurrentMutation) + } + } + + /// As for `compare_and_set_execution_proof_info`, but also writes to disk immediately. + pub fn compare_and_set_execution_proof_info_with_write( + &self, + prev_value: ExecutionProofInfo, + new_value: ExecutionProofInfo, + ) -> Result<(), Error> { + let kv_store_op = self.compare_and_set_execution_proof_info(prev_value, new_value)?; + self.hot_db.do_atomically(vec![kv_store_op]) + } + + /// Load the execution proof info from disk, but do not set `self.execution_proof_info`. + fn load_execution_proof_info(&self) -> Result, Error> { + self.hot_db + .get(&EXECUTION_PROOF_INFO_KEY) + .map_err(|e| Error::LoadExecutionProofInfo(e.into())) + } + + /// Store the given `execution_proof_info` to disk. + /// + /// The argument is intended to be `self.execution_proof_info`, but is passed manually to avoid + /// issues with recursive locking. + fn store_execution_proof_info_in_batch( + &self, + execution_proof_info: &ExecutionProofInfo, + ) -> KeyValueStoreOp { + execution_proof_info.as_kv_store_op(EXECUTION_PROOF_INFO_KEY) + } + /// Return the slot-window describing the available historic states. /// /// Returns `(lower_limit, upper_limit)`. @@ -3395,6 +3596,178 @@ impl, Cold: ItemStore> HotColdDB Ok(()) } + /// Try to prune execution proofs older than the execution proof boundary. + /// + /// Proofs from the epoch `execution_proof_boundary` are retained. + /// This epoch is an _exclusive_ endpoint for the pruning process. + /// + /// This function only supports pruning execution proofs older than the split point, + /// which is older than (or equal to) finalization. + pub fn try_prune_execution_proofs( + &self, + force: bool, + execution_proof_boundary: Epoch, + ) -> Result<(), Error> { + // Check if zkvm fork is enabled + if self.spec.zkvm_fork_epoch().is_none() { + debug!("ZKVM fork is disabled"); + return Ok(()); + } + + let pruning_enabled = self.get_config().prune_blobs; // Use same config as blobs for now + if !force && !pruning_enabled { + debug!( + prune_blobs = pruning_enabled, + "Execution proof pruning is disabled" + ); + return Ok(()); + } + + let execution_proof_info = self.get_execution_proof_info(); + let Some(oldest_execution_proof_slot) = execution_proof_info.oldest_execution_proof_slot + else { + debug!("No execution proofs stored yet"); + return Ok(()); + }; + + let start_epoch = oldest_execution_proof_slot.epoch(E::slots_per_epoch()); + + // Prune execution proofs up until the `execution_proof_boundary - 1` or the split + // slot's epoch, whichever is older. + let split = self.get_split_info(); + let end_epoch = std::cmp::min( + execution_proof_boundary.saturating_sub(1u64), + split.slot.epoch(E::slots_per_epoch()).saturating_sub(1u64), + ); + let end_slot = end_epoch.end_slot(E::slots_per_epoch()); + + let can_prune = end_epoch != Epoch::new(0) && start_epoch <= end_epoch; + if !can_prune { + debug!( + %oldest_execution_proof_slot, + %execution_proof_boundary, + %split.slot, + %end_epoch, + %start_epoch, + "Execution proofs are pruned" + ); + return Ok(()); + } + + debug!( + %end_epoch, + %execution_proof_boundary, + "Pruning execution proofs" + ); + + // Iterate blocks backwards from the `end_epoch`. + let Some((end_block_root, _)) = self + .forwards_block_roots_iterator_until(end_slot, end_slot, || { + self.get_hot_state(&split.state_root, true)? + .ok_or(HotColdDBError::MissingSplitState( + split.state_root, + split.slot, + )) + .map(|state| (state, split.state_root)) + .map_err(Into::into) + })? + .next() + .transpose()? + else { + debug!( + %end_epoch, + %execution_proof_boundary, + "No execution proofs to prune" + ); + return Ok(()); + }; + + let mut db_ops = vec![]; + let mut removed_block_roots = vec![]; + let mut new_oldest_slot: Option = None; + + // Iterate blocks backwards until we reach blocks older than the boundary. + for tuple in ParentRootBlockIterator::new(self, end_block_root) { + let (block_root, blinded_block) = tuple?; + let slot = blinded_block.slot(); + + // Get all execution proof keys for this block + let keys = self.get_all_execution_proof_keys(&block_root); + + // Check if any proofs exist for this block + let mut block_has_proofs = false; + for key in keys { + if self + .blobs_db + .key_exists(DBColumn::BeaconExecutionProof, &key)? + { + block_has_proofs = true; + db_ops.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconExecutionProof, + key, + )); + } + } + + if block_has_proofs { + debug!( + ?block_root, + %slot, + "Pruning execution proofs for block" + ); + removed_block_roots.push(block_root); + new_oldest_slot = Some(slot); + } + // Continue iterating even if this block has no proofs - proofs may be sparse + } + + // Commit deletions + if !db_ops.is_empty() { + debug!( + num_deleted = db_ops.len(), + "Deleting execution proofs from disk" + ); + self.blobs_db.do_atomically(db_ops)?; + } + + // TODO(zkproofs): Fix this to make it more readable + if !removed_block_roots.is_empty() + && let Some(mut block_cache) = self.block_cache.as_ref().map(|cache| cache.lock()) + { + for block_root in removed_block_roots { + block_cache.delete_execution_proofs(&block_root); + } + } + + // Update the execution proof info with the new oldest slot + if let Some(new_slot) = new_oldest_slot { + let new_oldest = end_slot + 1; + self.compare_and_set_execution_proof_info_with_write( + execution_proof_info.clone(), + ExecutionProofInfo { + oldest_execution_proof_slot: Some(new_oldest), + }, + )?; + debug!( + old_oldest = %new_slot, + new_oldest = %new_oldest, + "Updated execution proof info" + ); + } + + debug!("Execution proof pruning complete"); + + Ok(()) + } + + /// Get all possible execution proof keys for a given block root. + /// Returns keys for proof_ids 0 to MAX_PROOFS-1. + fn get_all_execution_proof_keys(&self, block_root: &Hash256) -> Vec> { + (0..types::MAX_PROOFS as u8) + .map(|proof_id| get_execution_proof_key(block_root, proof_id)) + .collect() + } + /// Delete *all* states from the freezer database and update the anchor accordingly. /// /// WARNING: this method deletes the genesis state and replaces it with the provided diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index ae5b2e1e571..516e858e581 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -40,6 +40,7 @@ use strum::{EnumIter, EnumString, IntoStaticStr}; pub use types::*; const DATA_COLUMN_DB_KEY_SIZE: usize = 32 + 8; +const EXECUTION_PROOF_DB_KEY_SIZE: usize = 32 + 1; // block_root + proof_id pub type ColumnIter<'a, K> = Box), Error>> + 'a>; pub type ColumnKeyIter<'a, K> = Box> + 'a>; @@ -171,6 +172,25 @@ pub fn parse_data_column_key(data: Vec) -> Result<(Hash256, ColumnIndex), Er Ok((block_root, column_index)) } +pub fn get_execution_proof_key(block_root: &Hash256, proof_id: u8) -> Vec { + let mut result = block_root.as_slice().to_vec(); + result.push(proof_id); + result +} + +pub fn parse_execution_proof_key(data: Vec) -> Result<(Hash256, u8), Error> { + if data.len() != EXECUTION_PROOF_DB_KEY_SIZE { + return Err(Error::InvalidKey(format!( + "Unexpected BeaconExecutionProof key len {}", + data.len() + ))); + } + let (block_root_bytes, proof_id_bytes) = data.split_at(32); + let block_root = Hash256::from_slice(block_root_bytes); + let proof_id = proof_id_bytes[0]; + Ok((block_root, proof_id)) +} + #[must_use] #[derive(Clone)] pub enum KeyValueStoreOp { @@ -263,6 +283,12 @@ pub enum DBColumn { BeaconDataColumn, #[strum(serialize = "bdi")] BeaconDataColumnCustodyInfo, + /// For storing execution proofs (zkVM proofs) in the blob database. + /// + /// - Key: `Hash256` block root + `u8` proof_id (33 bytes total). + /// - Value: SSZ-encoded ExecutionProof. + #[strum(serialize = "bep")] + BeaconExecutionProof, /// For full `BeaconState`s in the hot database (finalized or fork-boundary states). /// /// DEPRECATED. @@ -437,6 +463,7 @@ impl DBColumn { | Self::LightClientUpdate | Self::Dummy => 8, Self::BeaconDataColumn => DATA_COLUMN_DB_KEY_SIZE, + Self::BeaconExecutionProof => EXECUTION_PROOF_DB_KEY_SIZE, } } } diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index cf494684515..7a5979481fe 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -19,6 +19,7 @@ pub const ANCHOR_INFO_KEY: Hash256 = Hash256::repeat_byte(5); pub const BLOB_INFO_KEY: Hash256 = Hash256::repeat_byte(6); pub const DATA_COLUMN_INFO_KEY: Hash256 = Hash256::repeat_byte(7); pub const DATA_COLUMN_CUSTODY_INFO_KEY: Hash256 = Hash256::repeat_byte(8); +pub const EXECUTION_PROOF_INFO_KEY: Hash256 = Hash256::repeat_byte(9); /// State upper limit value used to indicate that a node is not storing historic states. pub const STATE_UPPER_LIMIT_NO_RETAIN: Slot = Slot::new(u64::MAX); @@ -255,3 +256,30 @@ impl StoreItem for DataColumnInfo { Ok(Self::from_ssz_bytes(bytes)?) } } + +/// Database parameters relevant to execution proof sync. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, Serialize, Deserialize, Default)] +pub struct ExecutionProofInfo { + /// The slot after which execution proofs are or *will be* available (>=). + /// + /// If this slot is in the future, then it is the first slot of the ZKVM fork, from which + /// execution proofs will be available. + /// + /// If the `oldest_execution_proof_slot` is `None` then this means that the ZKVM fork epoch + /// is not yet known. + pub oldest_execution_proof_slot: Option, +} + +impl StoreItem for ExecutionProofInfo { + fn db_column() -> DBColumn { + DBColumn::BeaconMeta + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &[u8]) -> Result { + Ok(Self::from_ssz_bytes(bytes)?) + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 1bab464b689..21eb5e1b8c7 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -61,7 +61,7 @@ pub mod chain_spec { } // Re-export execution_proof types for backwards compatibility -pub use crate::execution_proof::{ExecutionProof, MAX_PROOF_DATA_BYTES}; +pub use crate::execution_proof::{ExecutionProof, MAX_PROOF_DATA_BYTES, MAX_PROOFS}; pub use crate::execution_proof_id::{EXECUTION_PROOF_TYPE_COUNT, ExecutionProofId}; pub mod beacon_block { diff --git a/dummy_el/geth-wrapper.sh b/dummy_el/geth-wrapper.sh index 76c85dd57f3..5705888b49c 100644 --- a/dummy_el/geth-wrapper.sh +++ b/dummy_el/geth-wrapper.sh @@ -18,6 +18,32 @@ if echo "$@" | grep -qE "^(version|--version|-v|help|--help|-h)$"; then exit 0 fi +# Filter out flags that we don't need for dummy_el +# These are geth-specific flags that kurtosis may pass +FILTERED_ARGS="" +for arg in "$@"; do + case "$arg" in + --override.*|--override*|-override.*|-override*) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --datadir=*|--datadir) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --syncmode=*|--syncmode) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --gcmode=*|--gcmode) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + --networkid=*|--networkid) + echo "[dummy_el geth-wrapper] Ignoring geth flag: $arg" + ;; + *) + FILTERED_ARGS="$FILTERED_ARGS $arg" + ;; + esac +done + # For any other command, we start dummy_el # Parse geth arguments to extract what we need diff --git a/execution-witness-sentry/Cargo.toml b/execution-witness-sentry/Cargo.toml index c0b1d524cb8..ceee0f20bed 100644 --- a/execution-witness-sentry/Cargo.toml +++ b/execution-witness-sentry/Cargo.toml @@ -8,8 +8,8 @@ description = "Monitors execution layer nodes and fetches execution witnesses" alloy-provider = { version = "1", features = ["ws"] } alloy-rpc-types-eth = "1" anyhow = "1" -discv5 = { workspace = true } clap = { version = "4", features = ["derive"] } +discv5 = { workspace = true } eventsource-client = "0.13" flate2 = "1.1" futures = { workspace = true } diff --git a/execution-witness-sentry/config.toml b/execution-witness-sentry/config.toml index cbf69fff69b..b35c2ca262e 100644 --- a/execution-witness-sentry/config.toml +++ b/execution-witness-sentry/config.toml @@ -4,20 +4,24 @@ retain = 10 num_proofs = 2 [[endpoints]] -name = "el-1-reth" -el_url = "http://127.0.0.1:33100" -el_ws_url = "ws://127.0.0.1:33101" +name = "el-1-reth-lighthouse" +el_url = "http://127.0.0.1:32003" +el_ws_url = "ws://127.0.0.1:32004" # Non-zkvm CL for head event subscription (to know when new blocks arrive) [[cl_endpoints]] name = "cl-1-lighthouse-reth" -url = "http://127.0.0.1:33130/" +url = "http://127.0.0.1:33001/" # zkvm-enabled CLs for proof submission [[cl_endpoints]] name = "cl-4-lighthouse-geth" -url = "http://127.0.0.1:33142/" +url = "http://127.0.0.1:33022/" [[cl_endpoints]] name = "cl-5-lighthouse-geth" -url = "http://127.0.0.1:33145/" +url = "http://127.0.0.1:33029/" + +[[cl_endpoints]] +name = "cl-6-lighthouse-geth" +url = "http://127.0.0.1:33036/" diff --git a/execution-witness-sentry/src/cl_subscription.rs b/execution-witness-sentry/src/cl_subscription.rs index 64137f15afa..040e01cc438 100644 --- a/execution-witness-sentry/src/cl_subscription.rs +++ b/execution-witness-sentry/src/cl_subscription.rs @@ -6,6 +6,7 @@ use std::task::{Context, Poll}; use eventsource_client::{Client, SSE}; use futures::Stream; use serde::Deserialize; +use url::Url; use crate::error::{Error, Result}; @@ -65,10 +66,10 @@ impl Stream for ClEventStream { let result = match event.event_type.as_str() { "head" => serde_json::from_str::(&event.data) .map(ClEvent::Head) - .map_err(|e| Error::Parse(e)), + .map_err(Error::Parse), "block" => serde_json::from_str::(&event.data) .map(ClEvent::Block) - .map_err(|e| Error::Parse(e)), + .map_err(Error::Parse), _ => continue, }; return Poll::Ready(Some(result)); @@ -87,9 +88,9 @@ impl Stream for ClEventStream { /// Subscribe to CL head events via SSE. pub fn subscribe_cl_events(base_url: &str) -> Result { - let url = format!("{}eth/v1/events?topics=head,block", base_url); + let url = build_events_url(base_url)?; - let client = eventsource_client::ClientBuilder::for_url(&url) + let client = eventsource_client::ClientBuilder::for_url(url.as_str()) .map_err(|e| Error::Config(format!("Invalid SSE URL: {}", e)))? .build(); @@ -97,3 +98,31 @@ pub fn subscribe_cl_events(base_url: &str) -> Result { client: Box::pin(client.stream()), }) } + +fn build_events_url(base_url: &str) -> Result { + let base = Url::parse(base_url)?; + Ok(base.join("/eth/v1/events?topics=head,block")?) +} + +#[cfg(test)] +mod tests { + use super::build_events_url; + + #[test] + fn build_events_url_adds_path_without_trailing_slash() { + let url = build_events_url("http://localhost:5052").unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:5052/eth/v1/events?topics=head,block" + ); + } + + #[test] + fn build_events_url_adds_path_with_trailing_slash() { + let url = build_events_url("http://localhost:5052/").unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:5052/eth/v1/events?topics=head,block" + ); + } +} diff --git a/execution-witness-sentry/src/lib.rs b/execution-witness-sentry/src/lib.rs index 46113b7d3d2..fcb9b077cc8 100644 --- a/execution-witness-sentry/src/lib.rs +++ b/execution-witness-sentry/src/lib.rs @@ -32,12 +32,12 @@ pub mod storage; pub mod subscription; // Re-export main types at crate root for convenience. -pub use cl_subscription::{subscribe_cl_events, ClEvent, ClEventStream, HeadEvent, BlockEvent}; +pub use cl_subscription::{BlockEvent, ClEvent, ClEventStream, HeadEvent, subscribe_cl_events}; pub use config::{ClEndpoint, Config, Endpoint}; pub use error::{Error, Result}; pub use rpc::{BlockInfo, ClClient, ElClient, ExecutionProof, generate_random_proof}; pub use storage::{ - compress_gzip, decompress_gzip, load_block_data, BlockMetadata, BlockStorage, SavedProof, + BlockMetadata, BlockStorage, SavedProof, compress_gzip, decompress_gzip, load_block_data, }; pub use subscription::subscribe_blocks; diff --git a/execution-witness-sentry/src/main.rs b/execution-witness-sentry/src/main.rs index 4a71e2a3f75..8170bcda024 100644 --- a/execution-witness-sentry/src/main.rs +++ b/execution-witness-sentry/src/main.rs @@ -16,8 +16,8 @@ use tracing::{debug, error, info, warn}; use url::Url; use execution_witness_sentry::{ - subscribe_blocks, subscribe_cl_events, BlockInfo, BlockStorage, ClClient, ClEvent, Config, - ElClient, ExecutionProof, SavedProof, generate_random_proof, + BlockStorage, ClClient, ClEvent, Config, ElClient, ExecutionProof, SavedProof, + generate_random_proof, subscribe_blocks, subscribe_cl_events, }; /// Execution witness sentry - monitors EL nodes and fetches witnesses. @@ -32,9 +32,7 @@ struct Cli { /// Cached EL block data waiting for CL correlation. struct CachedElBlock { - block_hash: String, block_number: u64, - endpoint_name: String, timestamp: Instant, } @@ -52,13 +50,11 @@ impl ElBlockCache { } } - fn insert(&mut self, block_hash: String, block_number: u64, endpoint_name: String) { + fn insert(&mut self, block_hash: String, block_number: u64, _endpoint_name: String) { self.blocks.insert( - block_hash.clone(), + block_hash, CachedElBlock { - block_hash, block_number, - endpoint_name, timestamp: Instant::now(), }, ); @@ -186,47 +182,50 @@ async fn backfill_proofs( // Iterate through slots from zkvm_head + 1 to zkvm_head + slots_to_check for slot in (zkvm_head + 1)..=(zkvm_head + slots_to_check) { // First try to load saved proofs from disk - if let Some(storage) = storage { - if let Ok(Some((metadata, saved_proofs))) = storage.load_proofs_by_slot(slot) { - if !saved_proofs.is_empty() { - debug!(slot = slot, num_proofs = saved_proofs.len(), "Using saved proofs from disk"); - - for saved_proof in &saved_proofs { - let proof = ExecutionProof { - proof_id: saved_proof.proof_id, - slot: saved_proof.slot, - block_hash: saved_proof.block_hash.clone(), - block_root: saved_proof.block_root.clone(), - proof_data: saved_proof.proof_data.clone(), - }; + if let Some(storage) = storage + && let Ok(Some((_metadata, saved_proofs))) = storage.load_proofs_by_slot(slot) + && !saved_proofs.is_empty() + { + debug!( + slot = slot, + num_proofs = saved_proofs.len(), + "Using saved proofs from disk" + ); + + for saved_proof in &saved_proofs { + let proof = ExecutionProof { + proof_id: saved_proof.proof_id, + slot: saved_proof.slot, + block_hash: saved_proof.block_hash.clone(), + block_root: saved_proof.block_root.clone(), + proof_data: saved_proof.proof_data.clone(), + }; - match zkvm_client.submit_execution_proof(&proof).await { - Ok(()) => { - debug!( - name = %zkvm_name, - slot = slot, - proof_id = saved_proof.proof_id, - "Backfill proof submitted (from disk)" - ); - proofs_submitted += 1; - } - Err(e) => { - let msg = e.to_string(); - if !msg.contains("already known") { - debug!( - name = %zkvm_name, - slot = slot, - proof_id = saved_proof.proof_id, - error = %e, - "Backfill proof failed" - ); - } - } + match zkvm_client.submit_execution_proof(&proof).await { + Ok(()) => { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = saved_proof.proof_id, + "Backfill proof submitted (from disk)" + ); + proofs_submitted += 1; + } + Err(e) => { + let msg = e.to_string(); + if !msg.contains("already known") { + debug!( + name = %zkvm_name, + slot = slot, + proof_id = saved_proof.proof_id, + error = %e, + "Backfill proof failed" + ); } } - continue; // Move to next slot } } + continue; // Move to next slot } // No saved proofs, fetch block info and generate new proofs @@ -243,12 +242,9 @@ async fn backfill_proofs( }; // Only submit proofs for blocks with execution payloads - let exec_hash = match block_info.execution_block_hash { - Some(hash) => hash, - None => { - debug!(slot = slot, "No execution payload, skipping"); - continue; - } + let Some(exec_hash) = block_info.execution_block_hash else { + debug!(slot = slot, "No execution payload, skipping"); + continue; }; // Generate and submit proofs @@ -323,7 +319,7 @@ async fn main() -> anyhow::Result<()> { // Set up CL clients - separate zkvm targets from event sources let mut zkvm_clients: Vec<(String, ClClient)> = Vec::new(); // zkvm-enabled nodes for proof submission let mut event_source_client: Option<(String, String, ClClient)> = None; // First available CL for events - + if let Some(endpoints) = config.cl_endpoints.as_ref() { for endpoint in endpoints { let url = match Url::parse(&endpoint.url) { @@ -345,7 +341,8 @@ async fn main() -> anyhow::Result<()> { // Use first non-zkvm CL as event source if event_source_client.is_none() { info!(name = %endpoint.name, "Using as event source"); - event_source_client = Some((endpoint.name.clone(), endpoint.url.clone(), client)); + event_source_client = + Some((endpoint.name.clone(), endpoint.url.clone(), client)); } } Err(e) => { @@ -355,14 +352,14 @@ async fn main() -> anyhow::Result<()> { } } - info!(zkvm_targets = zkvm_clients.len(), "zkvm-enabled CL endpoints configured"); + info!( + zkvm_targets = zkvm_clients.len(), + "zkvm-enabled CL endpoints configured" + ); - let event_source = match event_source_client { - Some(es) => es, - None => { - error!("No non-zkvm CL endpoint available for event source"); - return Ok(()); - } + let Some(event_source) = event_source_client else { + error!("No non-zkvm CL endpoint available for event source"); + return Ok(()); }; info!(name = %event_source.0, "CL event source configured"); @@ -425,9 +422,11 @@ async fn main() -> anyhow::Result<()> { }); } + let (es_name, es_url, es_client) = event_source; + let source_client_for_monitor = es_client.clone(); + // Spawn CL subscription task for the event source (non-zkvm CL) { - let (es_name, es_url, es_client) = event_source; let tx = cl_tx.clone(); tokio::spawn(async move { @@ -447,11 +446,23 @@ async fn main() -> anyhow::Result<()> { while let Some(result) = stream.next().await { match result { Ok(ClEvent::Head(head)) => { - let slot: u64 = head.slot.parse().unwrap_or(0); + let slot: u64 = match head.slot.parse() { + Ok(slot) => slot, + Err(e) => { + warn!( + name = %es_name, + error = %e, + slot = %head.slot, + "Invalid head slot value" + ); + continue; + } + }; let block_root = head.block.clone(); // Fetch the execution block hash for this beacon block - let exec_hash = match es_client.get_block_execution_hash(&block_root).await { + let exec_hash = match es_client.get_block_execution_hash(&block_root).await + { Ok(Some(hash)) => hash, Ok(None) => { debug!(name = %es_name, slot = slot, "No execution hash for block"); @@ -492,11 +503,6 @@ async fn main() -> anyhow::Result<()> { let mut monitor_interval = tokio::time::interval(Duration::from_millis(500)); monitor_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Clone what we need for the monitoring - let source_client_for_monitor = ClClient::new( - Url::parse(&config.cl_endpoints.as_ref().unwrap()[0].url).unwrap(), - ); - info!("Waiting for events (with monitoring every 500ms)"); // Process events from both EL and CL @@ -506,7 +512,7 @@ async fn main() -> anyhow::Result<()> { _ = monitor_interval.tick() => { // Monitor zkvm CL status let statuses = monitor_zkvm_status(&source_client_for_monitor, &zkvm_clients).await; - + for status in &statuses { if status.gap < -5 { // More than 5 slots behind - log warning and backfill @@ -516,7 +522,7 @@ async fn main() -> anyhow::Result<()> { gap = status.gap, "zkvm CL is behind, starting backfill" ); - + // Find the client and backfill if let Some((_, client)) = zkvm_clients.iter().find(|(n, _)| n == &status.name) { backfill_proofs( @@ -557,14 +563,12 @@ async fn main() -> anyhow::Result<()> { ); // Find the endpoint and fetch block + witness - let endpoint = match config.endpoints.iter().find(|e| e.name == el_event.endpoint_name) { - Some(e) => e, - None => continue, + let Some(endpoint) = config.endpoints.iter().find(|e| e.name == el_event.endpoint_name) else { + continue; }; - let el_url = match Url::parse(&endpoint.el_url) { - Ok(u) => u, - Err(_) => continue, + let Ok(el_url) = Url::parse(&endpoint.el_url) else { + continue; }; let el_client = ElClient::new(el_url); diff --git a/execution-witness-sentry/src/rpc.rs b/execution-witness-sentry/src/rpc.rs index fc6f05e61d5..9722ee67085 100644 --- a/execution-witness-sentry/src/rpc.rs +++ b/execution-witness-sentry/src/rpc.rs @@ -280,7 +280,9 @@ impl ClClient { /// Get the execution block hash for a beacon block. pub async fn get_block_execution_hash(&self, block_root: &str) -> Result> { - let url = self.url.join(&format!("eth/v2/beacon/blocks/{}", block_root))?; + let url = self + .url + .join(&format!("eth/v2/beacon/blocks/{}", block_root))?; let response = self.http_client.get(url).send().await?; if response.status() == reqwest::StatusCode::NOT_FOUND { @@ -288,7 +290,12 @@ impl ClClient { } let block_response: BlockResponse = response.json().await?; - Ok(block_response.data.message.body.execution_payload.map(|p| p.block_hash)) + Ok(block_response + .data + .message + .body + .execution_payload + .map(|p| p.block_hash)) } /// Get the current head slot. diff --git a/execution-witness-sentry/src/storage.rs b/execution-witness-sentry/src/storage.rs index 214c513daf6..58a80d449d4 100644 --- a/execution-witness-sentry/src/storage.rs +++ b/execution-witness-sentry/src/storage.rs @@ -4,9 +4,9 @@ use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use alloy_rpc_types_eth::Block; +use flate2::Compression; use flate2::read::GzDecoder; use flate2::write::GzEncoder; -use flate2::Compression; use serde::{Deserialize, Serialize}; use crate::error::Result; @@ -72,7 +72,11 @@ pub struct BlockStorage { impl BlockStorage { /// Create a new block storage manager. - pub fn new(output_dir: impl Into, chain: impl Into, retain: Option) -> Self { + pub fn new( + output_dir: impl Into, + chain: impl Into, + retain: Option, + ) -> Self { Self { output_dir: output_dir.into(), chain: chain.into(), @@ -113,10 +117,10 @@ impl BlockStorage { std::fs::write(data_path, combined_data)?; // Clean up old blocks if retention is configured - if let Some(retain) = self.retain { - if block_number > retain { - self.delete_old_block(block_number - retain)?; - } + if let Some(retain) = self.retain + && block_number > retain + { + self.delete_old_block(block_number - retain)?; } Ok(()) @@ -133,7 +137,7 @@ impl BlockStorage { proofs: &[SavedProof], ) -> Result<()> { let block_dir = self.block_dir(block_number); - + // Create dir if it doesn't exist (in case block wasn't saved yet) std::fs::create_dir_all(&block_dir)?; @@ -170,7 +174,10 @@ impl BlockStorage { /// Load proofs for a given slot. /// Searches for a block directory that has matching slot in metadata. - pub fn load_proofs_by_slot(&self, slot: u64) -> Result)>> { + pub fn load_proofs_by_slot( + &self, + slot: u64, + ) -> Result)>> { let chain_dir = self.output_dir.join(&self.chain); if !chain_dir.exists() { return Ok(None); @@ -180,7 +187,7 @@ impl BlockStorage { for entry in std::fs::read_dir(&chain_dir)? { let entry = entry?; let block_dir = entry.path(); - + if !block_dir.is_dir() { continue; } @@ -216,7 +223,7 @@ impl BlockStorage { pub fn load_metadata(&self, block_number: u64) -> Result> { let block_dir = self.block_dir(block_number); let metadata_path = block_dir.join("metadata.json"); - + if !metadata_path.exists() { return Ok(None); } diff --git a/execution-witness-sentry/tests/cl_subscription.rs b/execution-witness-sentry/tests/cl_subscription.rs new file mode 100644 index 00000000000..40d86890eb0 --- /dev/null +++ b/execution-witness-sentry/tests/cl_subscription.rs @@ -0,0 +1,11 @@ +use execution_witness_sentry::subscribe_cl_events; + +#[test] +fn subscribe_cl_events_accepts_base_url_without_trailing_slash() { + assert!(subscribe_cl_events("http://localhost:5052").is_ok()); +} + +#[test] +fn subscribe_cl_events_accepts_base_url_with_trailing_slash() { + assert!(subscribe_cl_events("http://localhost:5052/").is_ok()); +} diff --git a/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml b/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml index 98f2af283df..b7d7e7f62ea 100644 --- a/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml +++ b/scripts/local_testnet/network_params_mixed_proof_gen_verify.yaml @@ -1,23 +1,22 @@ -# Mixed configuration: 3 normal nodes, 1 node with dummy EL +# Mixed configuration: 3 normal nodes with reth, 3 zkvm nodes with dummy_el participants: - # Nodes with real execution layer (nodes 1-3) + # Normal nodes with real EL (nodes 1-3) - el_type: reth el_image: ghcr.io/paradigmxyz/reth:latest cl_type: lighthouse cl_image: lighthouse:local cl_extra_params: - - --target-peers=3 + - --target-peers=5 count: 3 - # Node with dummy execution layer (node 4) - # TODO(zkproofs): Currently there is no way to add no client here - # We likely want to use our dummy zkvm EL here + # ZKVM nodes with dummy EL (nodes 4-6) + # Uses dummy_el wrapped as geth - returns SYNCING for all engine calls - el_type: geth el_image: dummy_el:local cl_type: lighthouse cl_image: lighthouse:local cl_extra_params: - --activate-zkvm - - --target-peers=3 + - --target-peers=5 count: 3 network_params: electra_fork_epoch: 0 @@ -28,3 +27,10 @@ snooper_enabled: false additional_services: - dora - prometheus_grafana +port_publisher: + el: + enabled: true + public_port_start: 32000 + cl: + enabled: true + public_port_start: 33000 \ No newline at end of file