From 1c8a80d5aec6a1783d249b8d34fb4fa05f2f1794 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 20 May 2026 12:30:10 +1200 Subject: [PATCH 01/15] Add sequencer subcommand with embedded store --- bin/node/.env | 15 +- bin/node/src/commands/block_producer.rs | 2 +- bin/node/src/commands/mod.rs | 1 + bin/node/src/commands/sequencer.rs | 290 ++++++++++++++++++++++++ bin/node/src/main.rs | 6 + crates/block-producer/Cargo.toml | 2 +- crates/block-producer/src/errors.rs | 2 + crates/block-producer/src/lib.rs | 2 +- crates/block-producer/src/server/mod.rs | 119 ++++++++++ crates/block-producer/src/store/mod.rs | 286 +++++++++++++++++++++-- crates/rpc/Cargo.toml | 2 +- crates/rpc/src/lib.rs | 2 +- crates/rpc/src/server/api.rs | 243 ++++++++++++++++++-- crates/rpc/src/server/mod.rs | 78 +++++++ crates/store/src/lib.rs | 7 +- crates/store/src/server/api.rs | 2 +- crates/store/src/server/mod.rs | 89 ++++++++ scripts/run-node.sh | 96 ++++---- 18 files changed, 1156 insertions(+), 88 deletions(-) create mode 100644 bin/node/src/commands/sequencer.rs diff --git a/bin/node/.env b/bin/node/.env index 3ec5b4e82e..7335a99398 100644 --- a/bin/node/.env +++ b/bin/node/.env @@ -5,7 +5,16 @@ MIDEN_NODE_ENABLE_OTEL=true MIDEN_NODE_DATA_DIRECTORY= -# Block Producer +# Sequencer (store + block-producer + rpc in one process) +MIDEN_NODE_SEQUENCER_RPC_LISTEN=0.0.0.0:57291 +MIDEN_NODE_SEQUENCER_BLOCK_PRODUCER_LISTEN= +MIDEN_NODE_SEQUENCER_NTX_BUILDER_LISTEN= +MIDEN_NODE_SEQUENCER_REPLICA_LISTEN= +MIDEN_NODE_SEQUENCER_VALIDATOR_URL= +MIDEN_NODE_SEQUENCER_BLOCK_PROVER_URL= +MIDEN_NODE_SEQUENCER_SQLITE_CONNECTION_POOL_SIZE= + +# Block Producer (standalone) MIDEN_NODE_BLOCK_PRODUCER_LISTEN= MIDEN_NODE_BLOCK_PRODUCER_STORE_URL= MIDEN_NODE_BLOCK_PRODUCER_VALIDATOR_URL= @@ -14,14 +23,14 @@ MIDEN_NODE_BLOCK_PRODUCER_MAX_BATCHES_PER_BLOCK= MIDEN_NODE_BLOCK_PRODUCER_MEMPOOL_TX_CAPACITY= MIDEN_NODE_BLOCK_PRODUCER_BATCH_PROVER_URL= -# Store +# Store (standalone) MIDEN_NODE_STORE_RPC_LISTEN= MIDEN_NODE_STORE_UPSTREAM_RPC_URL= MIDEN_NODE_STORE_NTX_BUILDER_LISTEN= MIDEN_NODE_STORE_BLOCK_PRODUCER_LISTEN= MIDEN_NODE_STORE_BLOCK_PROVER_URL= -# RPC +# RPC (standalone) MIDEN_NODE_RPC_LISTEN=0.0.0.0:57291 MIDEN_NODE_RPC_STORE_URL= MIDEN_NODE_RPC_BLOCK_PRODUCER_URL= diff --git a/bin/node/src/commands/block_producer.rs b/bin/node/src/commands/block_producer.rs index b921868231..8a4ec23b8e 100644 --- a/bin/node/src/commands/block_producer.rs +++ b/bin/node/src/commands/block_producer.rs @@ -222,5 +222,5 @@ pub struct BlockProducerConfig { env = ENV_MEMPOOL_TX_CAPACITY, value_name = "NUM" )] - mempool_tx_capacity: NonZeroUsize, + pub mempool_tx_capacity: NonZeroUsize, } diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index ccf3c92c00..2e09acd857 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -1,5 +1,6 @@ pub mod block_producer; pub mod rpc; +pub mod sequencer; pub mod store; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs new file mode 100644 index 0000000000..ec58836a79 --- /dev/null +++ b/bin/node/src/commands/sequencer.rs @@ -0,0 +1,290 @@ +use std::net::SocketAddr; +use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::Context; +use miden_node_block_producer::EmbeddedBlockProducer; +use miden_node_rpc::EmbeddedRpc; +use miden_node_store::genesis::GenesisBlock; +use miden_node_store::{ + ApplyBlockError, + DatabaseOptions, + DEFAULT_MAX_CONCURRENT_PROOFS, + State, + StoreApi, + default_sqlite_connection_pool_size, + serve_ntx_builder_and_replica, +}; +use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; +use miden_node_utils::fs::ensure_empty_directory; +use miden_protocol::block::SignedBlock; +use miden_protocol::utils::serde::Deserializable; +use url::Url; + +use super::ENV_ENABLE_OTEL; +use crate::commands::ENV_DATA_DIRECTORY; +use crate::commands::block_producer::BlockProducerConfig; + +const ENV_RPC_LISTEN: &str = "MIDEN_NODE_SEQUENCER_RPC_LISTEN"; +const ENV_BLOCK_PRODUCER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PRODUCER_LISTEN"; +const ENV_NTX_BUILDER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_NTX_BUILDER_LISTEN"; +const ENV_REPLICA_LISTEN: &str = "MIDEN_NODE_SEQUENCER_REPLICA_LISTEN"; +const ENV_VALIDATOR_URL: &str = "MIDEN_NODE_SEQUENCER_VALIDATOR_URL"; +const ENV_BLOCK_PROVER_URL: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PROVER_URL"; +const ENV_SQLITE_CONNECTION_POOL_SIZE: &str = "MIDEN_NODE_SEQUENCER_SQLITE_CONNECTION_POOL_SIZE"; + +#[derive(clap::Subcommand)] +pub enum SequencerCommand { + /// Bootstraps the blockchain database with a pre-existing genesis block. + /// + /// The genesis block file should be produced by `miden-validator bootstrap`. + Bootstrap { + /// Directory in which to store the database and raw block data. + #[arg(long, env = ENV_DATA_DIRECTORY, value_name = "DIR")] + data_directory: PathBuf, + /// Path to the pre-signed genesis block file produced by the validator. + #[arg(long, value_name = "FILE")] + genesis_block: PathBuf, + }, + + /// Starts the sequencer: store, block-producer, and RPC in a single process. + /// + /// Exposes four gRPC endpoints: the client-facing RPC API, the block-producer API, + /// the replica streaming API (for downstream replicas), and the network transaction + /// builder API. + Start { + /// Socket address at which to serve the client-facing RPC API. + #[arg(long = "rpc.listen", env = ENV_RPC_LISTEN, value_name = "LISTEN")] + rpc_listen: SocketAddr, + + /// Socket address at which to serve the block-producer gRPC API. + #[arg(long = "block-producer.listen", env = ENV_BLOCK_PRODUCER_LISTEN, value_name = "LISTEN")] + block_producer_listen: SocketAddr, + + /// Socket address at which to serve the network transaction builder API. + #[arg(long = "ntx-builder.listen", env = ENV_NTX_BUILDER_LISTEN, value_name = "LISTEN")] + ntx_builder_listen: SocketAddr, + + /// Socket address at which to serve the replica streaming API. + #[arg(long = "replica.listen", env = ENV_REPLICA_LISTEN, value_name = "LISTEN")] + replica_listen: SocketAddr, + + /// The validator's gRPC url. + #[arg(long = "validator.url", env = ENV_VALIDATOR_URL, value_name = "URL")] + validator_url: Url, + + /// The remote block prover's gRPC url. If not provided, a local block prover will be used. + #[arg(long = "block-prover.url", env = ENV_BLOCK_PROVER_URL, value_name = "URL")] + block_prover_url: Option, + + /// Directory in which to store the database and raw block data. + #[arg(long, env = ENV_DATA_DIRECTORY, value_name = "DIR")] + data_directory: PathBuf, + + /// Enables the exporting of traces for OpenTelemetry. + #[arg(long = "enable-otel", default_value_t = false, env = ENV_ENABLE_OTEL, value_name = "BOOL")] + enable_otel: bool, + + /// Maximum number of concurrent block proofs to be scheduled. + #[arg( + long = "max-concurrent-proofs", + default_value_t = DEFAULT_MAX_CONCURRENT_PROOFS, + value_name = "NUM" + )] + max_concurrent_proofs: NonZeroUsize, + + /// Maximum number of SQLite connections in the store database connection pool. + #[arg( + long = "sqlite.connection_pool_size", + env = ENV_SQLITE_CONNECTION_POOL_SIZE, + default_value_t = default_sqlite_connection_pool_size(), + value_name = "NUM" + )] + sqlite_connection_pool_size: NonZeroUsize, + + #[command(flatten)] + block_producer: BlockProducerConfig, + + #[command(flatten)] + grpc_options: GrpcOptionsExternal, + + #[command(flatten)] + storage_options: StorageOptions, + }, +} + +impl SequencerCommand { + pub async fn handle(self) -> anyhow::Result<()> { + match self { + Self::Bootstrap { data_directory, genesis_block } => { + ensure_empty_directory(&data_directory)?; + bootstrap_sequencer(&data_directory, &genesis_block) + }, + Self::Start { + rpc_listen, + block_producer_listen, + ntx_builder_listen, + replica_listen, + validator_url, + block_prover_url, + data_directory, + enable_otel: _, + max_concurrent_proofs, + sqlite_connection_pool_size, + block_producer, + grpc_options, + storage_options, + } => { + if block_producer.max_batches_per_block > miden_protocol::MAX_BATCHES_PER_BLOCK { + anyhow::bail!( + "max-batches-per-block cannot exceed protocol limit of {}", + miden_protocol::MAX_BATCHES_PER_BLOCK + ); + } + if block_producer.max_txs_per_batch > miden_protocol::MAX_ACCOUNTS_PER_BATCH { + anyhow::bail!( + "max-txs-per-batch cannot exceed protocol limit of {}", + miden_protocol::MAX_ACCOUNTS_PER_BATCH + ); + } + + Self::start( + rpc_listen, + block_producer_listen, + ntx_builder_listen, + replica_listen, + validator_url, + block_prover_url, + data_directory, + max_concurrent_proofs, + DatabaseOptions { connection_pool_size: sqlite_connection_pool_size }, + block_producer, + grpc_options, + storage_options, + ) + .await + }, + } + } + + pub fn is_open_telemetry_enabled(&self) -> bool { + match self { + Self::Start { enable_otel, .. } => *enable_otel, + Self::Bootstrap { .. } => false, + } + } + + #[expect(clippy::too_many_arguments)] + async fn start( + rpc_listen: SocketAddr, + block_producer_listen: SocketAddr, + ntx_builder_listen: SocketAddr, + replica_listen: SocketAddr, + validator_url: Url, + block_prover_url: Option, + data_directory: PathBuf, + max_concurrent_proofs: NonZeroUsize, + database_options: DatabaseOptions, + block_producer_config: BlockProducerConfig, + grpc_options: GrpcOptionsExternal, + storage_options: StorageOptions, + ) -> anyhow::Result<()> { + // Bind eagerly to catch address conflicts before loading state. + let rpc_listener = tokio::net::TcpListener::bind(rpc_listen) + .await + .context("Failed to bind to RPC gRPC socket")?; + let ntx_builder_listener = tokio::net::TcpListener::bind(ntx_builder_listen) + .await + .context("Failed to bind to ntx-builder gRPC socket")?; + let replica_listener = tokio::net::TcpListener::bind(replica_listen) + .await + .context("Failed to bind to replica gRPC socket")?; + + let (termination_ask, mut termination_signal) = + tokio::sync::mpsc::channel::(1); + + let (state, proven_tip) = State::load_with_database_options( + &data_directory, + storage_options, + database_options, + termination_ask, + ) + .await + .context("failed to load state")?; + + let state = Arc::new(state); + let store_api = Arc::new(StoreApi::new(Arc::clone(&state))); + let grpc_internal = grpc_options.into(); + + let ntx_replica_task = tokio::spawn(serve_ntx_builder_and_replica( + Arc::clone(&state), + proven_tip, + ntx_builder_listener, + replica_listener, + block_prover_url, + max_concurrent_proofs, + grpc_internal, + )); + + let block_producer_task = tokio::spawn( + EmbeddedBlockProducer { + block_producer_address: block_producer_listen, + state: Arc::clone(&state), + validator_url: validator_url.clone(), + batch_prover_url: block_producer_config.batch_prover_url, + batch_interval: block_producer_config.batch_interval, + block_interval: block_producer_config.block_interval, + max_txs_per_batch: block_producer_config.max_txs_per_batch, + max_batches_per_block: block_producer_config.max_batches_per_block, + grpc_options: grpc_internal, + mempool_tx_capacity: block_producer_config.mempool_tx_capacity, + } + .serve(), + ); + + // The embedded block producer exposes its own gRPC endpoint which the RPC connects to. + let block_producer_url = + Url::parse(&format!("http://{block_producer_listen}")).context("invalid block-producer URL")?; + // Similarly, the embedded ntx-builder is reachable on its own gRPC endpoint. + let ntx_builder_url = + Url::parse(&format!("http://{ntx_builder_listen}")).context("invalid ntx-builder URL")?; + + let rpc_task = tokio::spawn( + EmbeddedRpc { + listener: rpc_listener, + state: store_api, + block_producer_url: Some(block_producer_url), + validator_url, + ntx_builder_url: Some(ntx_builder_url), + grpc_options, + } + .serve(), + ); + + tokio::select! { + result = ntx_replica_task => { + result.context("ntx-builder/replica task panicked")?.context("ntx-builder/replica task failed") + }, + result = block_producer_task => { + result.context("block-producer task panicked")?.context("block-producer task failed") + }, + result = rpc_task => { + result.context("rpc task panicked")?.context("rpc task failed") + }, + Some(err) = termination_signal.recv() => { + Err(anyhow::anyhow!("received termination signal from apply_block").context(err)) + }, + } + } +} + +fn bootstrap_sequencer(data_directory: &Path, genesis_block_path: &Path) -> anyhow::Result<()> { + let bytes = fs_err::read(genesis_block_path).context("failed to read genesis block")?; + let signed_block = SignedBlock::read_from_bytes(&bytes) + .context("failed to deserialize genesis block from file")?; + let genesis_block = + GenesisBlock::try_from(signed_block).context("genesis block validation failed")?; + + miden_node_store::Store::bootstrap(genesis_block, data_directory) +} diff --git a/bin/node/src/main.rs b/bin/node/src/main.rs index a9ef2f75ac..cf3dcf6b29 100644 --- a/bin/node/src/main.rs +++ b/bin/node/src/main.rs @@ -32,6 +32,10 @@ pub enum Command { /// Commands related to the node's block-producer component. #[command(subcommand)] BlockProducer(commands::block_producer::BlockProducerCommand), + + /// Commands related to the sequencer (store + block-producer + rpc in one process). + #[command(subcommand)] + Sequencer(commands::sequencer::SequencerCommand), } impl Command { @@ -43,6 +47,7 @@ impl Command { Command::Store(subcommand) => subcommand.is_open_telemetry_enabled(), Command::Rpc(subcommand) => subcommand.is_open_telemetry_enabled(), Command::BlockProducer(subcommand) => subcommand.is_open_telemetry_enabled(), + Command::Sequencer(subcommand) => subcommand.is_open_telemetry_enabled(), } { OpenTelemetry::Enabled } else { @@ -55,6 +60,7 @@ impl Command { Command::Rpc(rpc_command) => rpc_command.handle().await, Command::Store(store_command) => store_command.handle().await, Command::BlockProducer(block_producer_command) => block_producer_command.handle().await, + Command::Sequencer(sequencer_command) => sequencer_command.handle().await, } } } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 5509826e74..64d72c4a8b 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -27,6 +27,7 @@ futures = { workspace = true } itertools = { workspace = true } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, workspace = true } miden-remote-prover-client = { features = ["batch-prover", "block-prover"], workspace = true } @@ -44,7 +45,6 @@ url = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } -miden-node-store = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { features = ["testing"], workspace = true } diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index b008f30b06..a7756201ac 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -182,6 +182,8 @@ pub enum StoreError { MalformedResponse(String), #[error("failed to parse response")] DeserializationError(#[from] ConversionError), + #[error("{0}")] + Internal(String), } impl From for StoreError { diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 955aa23565..60889389af 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -18,7 +18,7 @@ pub mod errors; mod errors; pub mod server; -pub use server::BlockProducer; +pub use server::{BlockProducer, EmbeddedBlockProducer}; // CONSTANTS // ================================================================================================= diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 54f11ba93d..55df97436c 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -35,6 +35,125 @@ use crate::store::StoreClient; use crate::validator::BlockProducerValidatorClient; use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BUILDERS}; +// EMBEDDED BLOCK PRODUCER +// ================================================================================================ + +/// Block producer variant that uses an in-process store instead of a remote gRPC store. +pub struct EmbeddedBlockProducer { + /// The address of the block producer component. + pub block_producer_address: SocketAddr, + /// The in-process store state. + pub state: Arc, + /// The address of the validator component. + pub validator_url: Url, + /// The address of the batch prover component. + pub batch_prover_url: Option, + /// The interval at which to produce batches. + pub batch_interval: Duration, + /// The interval at which to produce blocks. + pub block_interval: Duration, + /// The maximum number of transactions per batch. + pub max_txs_per_batch: usize, + /// The maximum number of batches per block. + pub max_batches_per_block: usize, + /// Server-side gRPC options. + pub grpc_options: GrpcOptionsInternal, + /// The maximum number of inflight transactions allowed in the mempool at once. + pub mempool_tx_capacity: NonZeroUsize, +} + +impl EmbeddedBlockProducer { + /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// + /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is + /// encountered. + pub async fn serve(self) -> anyhow::Result<()> { + info!(target: COMPONENT, endpoint=?self.block_producer_address, "Initializing embedded server"); + let store = StoreClient::new_local(self.state.clone()); + let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); + + let chain_tip = self + .state + .chain_tip(miden_node_store::Finality::Committed) + .await; + + let listener = TcpListener::bind(self.block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Embedded server initialized"); + + let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); + let batch_builder = BatchBuilder::new( + store.clone(), + SERVER_NUM_BATCH_BUILDERS, + self.batch_prover_url, + self.batch_interval, + ); + let mempool = MempoolConfig { + batch_budget: BatchBudget { + transactions: self.max_txs_per_batch, + ..BatchBudget::default() + }, + block_budget: BlockBudget { batches: self.max_batches_per_block }, + tx_capacity: self.mempool_tx_capacity, + ..Default::default() + }; + let mempool = Mempool::shared(chain_tip, mempool); + + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn({ + let mempool = mempool.clone(); + async move { + BlockProducerRpcServer::new(mempool, store) + .serve(listener, self.grpc_options) + .await + } + }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + let block_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { block_builder.run(mempool).await } + }) + .id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? + } +} + #[cfg(test)] mod tests; diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 92c78cfe4d..061d2e696c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -1,13 +1,17 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::{Display, Formatter}; use std::num::NonZeroU32; +use std::sync::Arc; use itertools::Itertools; use miden_node_proto::clients::{Builder, StoreBlockProducerClient}; use miden_node_proto::decode::{ConversionResultExt, GrpcDecodeExt}; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_proto::domain::proof_request::BlockProofRequest; use miden_node_proto::errors::ConversionError; use miden_node_proto::{AccountState, decode, generated as proto}; +use miden_node_store::State; +use miden_node_store::state::Finality; use miden_node_utils::formatting::format_opt; use miden_protocol::Word; use miden_protocol::account::AccountId; @@ -104,20 +108,17 @@ impl TryFrom for TransactionInputs { } } -// STORE CLIENT +// REMOTE STORE CLIENT // ================================================================================================ -/// Interface to the store's block-producer gRPC API. -/// -/// Essentially just a thin wrapper around the generated gRPC client which improves type safety. +/// gRPC-based interface to the store's block-producer API. #[derive(Clone, Debug)] -pub struct StoreClient { +pub(crate) struct RemoteStoreClient { client: StoreBlockProducerClient, } -impl StoreClient { - /// Creates a new store client with a lazy connection. - pub fn new(store_url: Url) -> Self { +impl RemoteStoreClient { + fn new(store_url: Url) -> Self { info!(target: COMPONENT, store_endpoint = %store_url, "Initializing store client"); let store = Builder::new(store_url) @@ -132,7 +133,7 @@ impl StoreClient { } /// Returns the latest block's header from the store. - #[instrument(target = COMPONENT, name = "store.client.latest_header", skip_all, err)] + #[instrument(target = COMPONENT, name = "store.remote_client.latest_header", skip_all, err)] pub async fn latest_header(&self) -> Result { let response = self .client @@ -152,7 +153,7 @@ impl StoreClient { BlockHeader::try_from(response).map_err(StoreError::DeserializationError) } - #[instrument(target = COMPONENT, name = "store.client.get_tx_inputs", skip_all, err)] + #[instrument(target = COMPONENT, name = "store.remote_client.get_tx_inputs", skip_all, err)] pub async fn get_tx_inputs( &self, proven_tx: &ProvenTransaction, @@ -197,7 +198,7 @@ impl StoreClient { Ok(tx_inputs) } - #[instrument(target = COMPONENT, name = "store.client.get_block_inputs", skip_all, err)] + #[instrument(target = COMPONENT, name = "store.remote_client.get_block_inputs", skip_all, err)] pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, @@ -219,7 +220,7 @@ impl StoreClient { store_response.try_into().map_err(StoreError::DeserializationError) } - #[instrument(target = COMPONENT, name = "store.client.get_batch_inputs", skip_all, err)] + #[instrument(target = COMPONENT, name = "store.remote_client.get_batch_inputs", skip_all, err)] pub async fn get_batch_inputs( &self, block_references: impl Iterator + Send, @@ -235,7 +236,7 @@ impl StoreClient { store_response.try_into().map_err(StoreError::DeserializationError) } - #[instrument(target = COMPONENT, name = "store.client.apply_block", skip_all, err)] + #[instrument(target = COMPONENT, name = "store.remote_client.apply_block", skip_all, err)] pub async fn apply_block( &self, ordered_batches: &OrderedBatches, @@ -249,3 +250,260 @@ impl StoreClient { self.client.clone().apply_block(request).await.map(|_| ()).map_err(Into::into) } } + +// LOCAL STORE CLIENT +// ================================================================================================ + +/// In-process interface to the store's block-producer API, bypassing gRPC. +#[derive(Clone)] +pub(crate) struct LocalStoreClient { + state: Arc, +} + +impl std::fmt::Debug for LocalStoreClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalStoreClient").finish_non_exhaustive() + } +} + +impl LocalStoreClient { + async fn latest_header(&self) -> Result { + let (header, _) = self + .state + .get_block_header(None, false) + .await + .map_err(|e| StoreError::Internal(e.to_string()))?; + + header.ok_or_else(|| StoreError::Internal("store has no latest block header".into())) + } + + async fn get_tx_inputs( + &self, + proven_tx: &ProvenTransaction, + ) -> Result { + let account_id = proven_tx.account_id(); + let nullifiers: Vec = proven_tx.nullifiers().collect(); + let unauthenticated_notes: Vec = + proven_tx.unauthenticated_notes().map(|note| note.to_commitment()).collect(); + + let tx_inputs = self + .state + .get_transaction_inputs(account_id, &nullifiers, unauthenticated_notes) + .await + .map_err(|e| StoreError::Internal(e.to_string()))?; + + if let Some(false) = tx_inputs.new_account_id_prefix_is_unique { + return Err(StoreError::DuplicateAccountIdPrefix(account_id)); + } + + let current_block_height = self.state.chain_tip(Finality::Committed).await; + + let nullifiers: HashMap> = tx_inputs + .nullifiers + .into_iter() + .map(|info| (info.nullifier, NonZeroU32::new(info.block_num.as_u32()))) + .collect(); + + let account_commitment = if tx_inputs.account_commitment.is_empty() { + None + } else { + Some(tx_inputs.account_commitment) + }; + + Ok(TransactionInputs { + account_id, + account_commitment, + nullifiers, + found_unauthenticated_notes: tx_inputs.found_unauthenticated_notes, + current_block_height, + }) + } + + async fn get_block_inputs( + &self, + updated_accounts: impl Iterator + Send, + created_nullifiers: impl Iterator + Send, + unauthenticated_notes: impl Iterator + Send, + reference_blocks: impl Iterator + Send, + ) -> Result { + self.state + .get_block_inputs( + updated_accounts.collect(), + created_nullifiers.collect(), + unauthenticated_notes.collect(), + reference_blocks.collect(), + ) + .await + .map_err(|e| StoreError::Internal(e.to_string())) + } + + async fn get_batch_inputs( + &self, + block_references: impl Iterator + Send, + note_commitments: impl Iterator + Send, + ) -> Result { + let reference_blocks: BTreeSet = + block_references.map(|(block_num, _)| block_num).collect(); + let note_commitments: BTreeSet = note_commitments.collect(); + + self.state + .get_batch_inputs(reference_blocks, note_commitments) + .await + .map_err(|e| StoreError::Internal(e.to_string())) + } + + async fn apply_block( + &self, + ordered_batches: &OrderedBatches, + signed_block: &SignedBlock, + ) -> Result<(), StoreError> { + // Extract block inputs from the ordered batches (mirrors + // StoreApi::block_inputs_from_ordered_batches). + let mut account_ids = BTreeSet::new(); + let mut nullifiers = Vec::new(); + let mut unauthenticated_note_commitments = BTreeSet::new(); + let mut reference_blocks = BTreeSet::new(); + + for batch in ordered_batches.as_slice() { + account_ids.extend(batch.updated_accounts()); + nullifiers.extend(batch.created_nullifiers()); + reference_blocks.insert(batch.reference_block_num()); + + for note in batch.input_notes().iter() { + if let Some(header) = note.header() { + unauthenticated_note_commitments.insert(header.to_commitment()); + } + } + } + + let block_inputs = self + .state + .get_block_inputs( + account_ids.into_iter().collect(), + nullifiers, + unauthenticated_note_commitments, + reference_blocks, + ) + .await + .map_err(|e| StoreError::Internal(e.to_string()))?; + + let proving_inputs = BlockProofRequest { + tx_batches: ordered_batches.clone(), + block_header: signed_block.header().clone(), + block_inputs, + }; + + self.state + .save_proving_inputs(signed_block.header().block_num(), &proving_inputs) + .await + .map_err(|e| StoreError::Internal(e.to_string()))?; + + self.state + .apply_block(signed_block.clone()) + .await + .map_err(|e| StoreError::Internal(e.to_string())) + } +} + +// STORE CLIENT +// ================================================================================================ + +/// Interface to the store's block-producer API. +/// +/// Supports two backends: a gRPC client for the legacy out-of-process store, and a direct +/// in-process handle for the embedded sequencer. +#[derive(Clone, Debug)] +#[allow(private_interfaces)] +pub enum StoreClient { + /// Connects to a remote store over gRPC. + Remote(RemoteStoreClient), + /// Calls an in-process `State` directly, bypassing gRPC. + Local(LocalStoreClient), +} + +impl StoreClient { + /// Creates a gRPC-backed store client with a lazy connection. + pub fn new(store_url: Url) -> Self { + Self::Remote(RemoteStoreClient::new(store_url)) + } + + /// Creates an in-process store client backed by the given `State`. + pub fn new_local(state: Arc) -> Self { + Self::Local(LocalStoreClient { state }) + } + + /// Returns the latest block's header from the store. + #[instrument(target = COMPONENT, name = "store.client.latest_header", skip_all, err)] + pub async fn latest_header(&self) -> Result { + match self { + Self::Remote(c) => c.latest_header().await, + Self::Local(c) => c.latest_header().await, + } + } + + #[instrument(target = COMPONENT, name = "store.client.get_tx_inputs", skip_all, err)] + pub async fn get_tx_inputs( + &self, + proven_tx: &ProvenTransaction, + ) -> Result { + info!(target: COMPONENT, tx_id = %proven_tx.id().to_hex()); + match self { + Self::Remote(c) => c.get_tx_inputs(proven_tx).await, + Self::Local(c) => c.get_tx_inputs(proven_tx).await, + } + } + + #[instrument(target = COMPONENT, name = "store.client.get_block_inputs", skip_all, err)] + pub async fn get_block_inputs( + &self, + updated_accounts: impl Iterator + Send, + created_nullifiers: impl Iterator + Send, + unauthenticated_notes: impl Iterator + Send, + reference_blocks: impl Iterator + Send, + ) -> Result { + match self { + Self::Remote(c) => { + c.get_block_inputs( + updated_accounts, + created_nullifiers, + unauthenticated_notes, + reference_blocks, + ) + .await + }, + Self::Local(c) => { + c.get_block_inputs( + updated_accounts, + created_nullifiers, + unauthenticated_notes, + reference_blocks, + ) + .await + }, + } + } + + #[instrument(target = COMPONENT, name = "store.client.get_batch_inputs", skip_all, err)] + pub async fn get_batch_inputs( + &self, + block_references: impl Iterator + Send, + note_commitments: impl Iterator + Send, + ) -> Result { + match self { + Self::Remote(c) => c.get_batch_inputs(block_references, note_commitments).await, + Self::Local(c) => c.get_batch_inputs(block_references, note_commitments).await, + } + } + + #[instrument(target = COMPONENT, name = "store.client.apply_block", skip_all, err)] + pub async fn apply_block( + &self, + ordered_batches: &OrderedBatches, + signed_block: &SignedBlock, + ) -> Result<(), StoreError> { + match self { + Self::Remote(c) => c.apply_block(ordered_batches, signed_block).await, + Self::Local(c) => c.apply_block(ordered_batches, signed_block).await, + } + } +} diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index ed3d8ff903..7d91e97a90 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -24,6 +24,7 @@ http = { workspace = true } mediatype = { version = "0.21" } miden-node-proto = { workspace = true } miden-node-proto-build = { workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-protocol = { default-features = true, workspace = true } miden-tx = { features = ["concurrent"], workspace = true } @@ -41,7 +42,6 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] -miden-node-store = { features = ["rocksdb"], workspace = true } miden-node-utils = { features = ["testing", "tracing-forest"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { workspace = true } diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 368cb63de4..460ead7d92 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -2,7 +2,7 @@ mod server; #[cfg(test)] mod tests; -pub use server::Rpc; +pub use server::{EmbeddedRpc, Rpc}; // CONSTANTS // ================================================================================================= diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 8ceac6b511..708aff733d 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -1,5 +1,5 @@ use std::num::NonZeroUsize; -use std::sync::LazyLock; +use std::sync::{Arc, LazyLock}; use std::time::Duration; use anyhow::Context; @@ -10,6 +10,7 @@ use miden_node_proto::clients::{ StoreRpcClient, ValidatorClient, }; +use miden_node_proto::generated::store::rpc_server; use miden_node_proto::decode::{read_account_id, read_account_ids, read_block_range}; use miden_node_proto::domain::account::{AccountRequest, SlotData}; use miden_node_proto::errors::ConversionError; @@ -46,11 +47,154 @@ use url::Url; use crate::COMPONENT; +// STORE BACKEND +// ================================================================================================ + +/// Dispatches store read calls to either a remote gRPC store or an in-process `StoreApi`. +enum StoreBackend { + Remote(StoreRpcClient), + Embedded(Arc), +} + +impl StoreBackend { + async fn sync_nullifiers( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_nullifiers(request).await, + Self::Embedded(api) => rpc_server::Rpc::sync_nullifiers(api.as_ref(), request).await, + } + } + + async fn get_block_header_by_number( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().get_block_header_by_number(request).await, + Self::Embedded(api) => { + rpc_server::Rpc::get_block_header_by_number(api.as_ref(), request).await + }, + } + } + + async fn get_block_by_number( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().get_block_by_number(request.into_inner()).await, + Self::Embedded(api) => { + rpc_server::Rpc::get_block_by_number(api.as_ref(), request).await + }, + } + } + + async fn sync_chain_mmr( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_chain_mmr(request).await, + Self::Embedded(api) => rpc_server::Rpc::sync_chain_mmr(api.as_ref(), request).await, + } + } + + async fn sync_notes( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_notes(request).await, + Self::Embedded(api) => rpc_server::Rpc::sync_notes(api.as_ref(), request).await, + } + } + + async fn get_notes_by_id( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().get_notes_by_id(request).await, + Self::Embedded(api) => rpc_server::Rpc::get_notes_by_id(api.as_ref(), request).await, + } + } + + async fn get_note_script_by_root( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().get_note_script_by_root(request).await, + Self::Embedded(api) => { + rpc_server::Rpc::get_note_script_by_root(api.as_ref(), request).await + }, + } + } + + async fn sync_account_storage_maps( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_account_storage_maps(request).await, + Self::Embedded(api) => { + rpc_server::Rpc::sync_account_storage_maps(api.as_ref(), request).await + }, + } + } + + async fn sync_account_vault( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_account_vault(request).await, + Self::Embedded(api) => { + rpc_server::Rpc::sync_account_vault(api.as_ref(), request).await + }, + } + } + + async fn get_account( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().get_account(request.into_inner()).await, + Self::Embedded(api) => rpc_server::Rpc::get_account(api.as_ref(), request).await, + } + } + + async fn sync_transactions( + &self, + request: Request, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().sync_transactions(request).await, + Self::Embedded(api) => { + rpc_server::Rpc::sync_transactions(api.as_ref(), request).await + }, + } + } + + async fn status( + &self, + request: Request<()>, + ) -> Result, Status> { + match self { + Self::Remote(c) => c.clone().status(request).await, + Self::Embedded(api) => rpc_server::Rpc::status(api.as_ref(), request).await, + } + } +} + // RPC SERVICE // ================================================================================================ pub struct RpcService { - store: StoreRpcClient, + store: StoreBackend, block_producer: Option, validator: ValidatorClient, ntx_builder: Option, @@ -68,15 +212,79 @@ impl RpcService { ) -> Self { let store = { info!(target: COMPONENT, store_endpoint = %store_url, "Initializing store client"); - Builder::new(store_url) + StoreBackend::Remote( + Builder::new(store_url) + .without_tls() + .without_timeout() + .without_metadata_version() + .without_metadata_genesis() + .with_otel_context_injection() + .connect_lazy::(), + ) + }; + + let block_producer = block_producer_url.map(|block_producer_url| { + info!( + target: COMPONENT, + block_producer_endpoint = %block_producer_url, + "Initializing block producer client", + ); + Builder::new(block_producer_url) + .without_tls() + .without_timeout() + .without_metadata_version() + .without_metadata_genesis() + .with_otel_context_injection() + .connect_lazy::() + }); + + let validator = { + info!( + target: COMPONENT, + validator_endpoint = %validator_url, + "Initializing validator client", + ); + Builder::new(validator_url) .without_tls() .without_timeout() .without_metadata_version() .without_metadata_genesis() .with_otel_context_injection() - .connect_lazy::() + .connect_lazy::() }; + let ntx_builder = ntx_builder_url.map(|ntx_builder_url| { + info!( + target: COMPONENT, + ntx_builder_endpoint = %ntx_builder_url, + "Initializing ntx-builder client", + ); + Builder::new(ntx_builder_url) + .without_tls() + .without_timeout() + .without_metadata_version() + .without_metadata_genesis() + .with_otel_context_injection() + .connect_lazy::() + }); + + Self { + store, + block_producer, + validator, + ntx_builder, + genesis_commitment: None, + block_commitment_cache: LruCache::new(commitment_cache_capacity), + } + } + + pub(super) fn new_embedded( + state: Arc, + block_producer_url: Option, + validator_url: Url, + ntx_builder_url: Option, + commitment_cache_capacity: NonZeroUsize, + ) -> Self { let block_producer = block_producer_url.map(|block_producer_url| { info!( target: COMPONENT, @@ -123,7 +331,7 @@ impl RpcService { }); Self { - store, + store: StoreBackend::Embedded(state), block_producer, validator, ntx_builder, @@ -203,7 +411,6 @@ impl RpcService { let header = self .store - .clone() .get_block_header_by_number(Request::new(proto::rpc::BlockHeaderByNumberRequest { block_num: Some(block.as_u32()), include_mmr_proof: false.into(), @@ -261,7 +468,7 @@ impl api_server::Api for RpcService { check::(request.get_ref().nullifiers.len())?; - self.store.clone().sync_nullifiers(request).await + self.store.sync_nullifiers(request).await } // -- Block endpoints --------------------------------------------------------------------- @@ -274,7 +481,7 @@ impl api_server::Api for RpcService { Span::current().set_attribute("block.number", request.get_ref().block_num()); - self.store.clone().get_block_header_by_number(request).await + self.store.get_block_header_by_number(request).await } async fn get_block_by_number( @@ -287,7 +494,7 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, ?request); - self.store.clone().get_block_by_number(request).await + self.store.get_block_by_number(Request::new(request)).await } async fn sync_chain_mmr( @@ -302,7 +509,7 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, request = ?request_ref); - self.store.clone().sync_chain_mmr(request).await + self.store.sync_chain_mmr(request).await } // -- Note endpoints ---------------------------------------------------------------------- @@ -320,7 +527,7 @@ impl api_server::Api for RpcService { check::(request.get_ref().note_tags.len())?; - self.store.clone().sync_notes(request).await + self.store.sync_notes(request).await } async fn get_notes_by_id( @@ -341,7 +548,7 @@ impl api_server::Api for RpcService { Status::invalid_argument(err.as_report_context("invalid NoteId")) })?; - self.store.clone().get_notes_by_id(request).await + self.store.get_notes_by_id(request).await } async fn get_note_script_by_root( @@ -350,7 +557,7 @@ impl api_server::Api for RpcService { ) -> Result, Status> { debug!(target: COMPONENT, request = ?request); - self.store.clone().get_note_script_by_root(request).await + self.store.get_note_script_by_root(request).await } // -- Account endpoints ------------------------------------------------------------------- @@ -374,7 +581,7 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, request = ?request.get_ref()); - self.store.clone().sync_account_storage_maps(request).await + self.store.sync_account_storage_maps(request).await } async fn sync_account_vault( @@ -395,7 +602,7 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, request = ?request.get_ref()); - self.store.clone().sync_account_vault(request).await + self.store.sync_account_vault(request).await } /// Validates storage map key limits before forwarding the account request to the store. @@ -428,7 +635,7 @@ impl api_server::Api for RpcService { check::(total_keys)?; } - self.store.clone().get_account(raw_request).await + self.store.get_account(Request::new(raw_request)).await } // -- Transaction submission -------------------------------------------------------------- @@ -632,7 +839,7 @@ impl api_server::Api for RpcService { check::(request.get_ref().account_ids.len())?; - self.store.clone().sync_transactions(request).await + self.store.sync_transactions(request).await } async fn status( @@ -642,7 +849,7 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, request = ?request); let store_status = - self.store.clone().status(Request::new(())).await.map(Response::into_inner).ok(); + self.store.status(Request::new(())).await.map(Response::into_inner).ok(); let block_producer_status = if let Some(block_producer) = &self.block_producer { block_producer .clone() diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 21ca958e57..9eb7875399 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -1,4 +1,5 @@ use std::num::NonZeroUsize; +use std::sync::Arc; use accept::AcceptHeaderLayer; use anyhow::Context; @@ -39,6 +40,83 @@ pub struct Rpc { pub grpc_options: GrpcOptionsExternal, } +/// RPC server variant that reads from an in-process `StoreApi` instead of a remote gRPC store. +pub struct EmbeddedRpc { + pub listener: TcpListener, + pub state: Arc, + pub block_producer_url: Option, + pub validator_url: Url, + pub ntx_builder_url: Option, + pub grpc_options: GrpcOptionsExternal, +} + +impl EmbeddedRpc { + /// Serves the RPC API using an in-process store. + /// + /// Note: Executes in place (i.e. not spawned) and will run indefinitely until + /// a fatal error is encountered. + pub async fn serve(self) -> anyhow::Result<()> { + let mut api = api::RpcService::new_embedded( + self.state, + self.block_producer_url.clone(), + self.validator_url, + self.ntx_builder_url.clone(), + NonZeroUsize::new(1_000_000).unwrap(), + ); + + let genesis = api + .get_genesis_header_with_retry() + .await + .context("Fetching genesis header from embedded store")?; + + api.set_genesis_commitment(genesis.commitment())?; + + let api_service = api_server::ApiServer::new(api); + let reflection_service = server::Builder::configure() + .register_file_descriptor_set(rpc_api_descriptor()) + .build_v1() + .context("failed to build reflection service")?; + + info!(target: COMPONENT, endpoint=?self.listener, "Embedded RPC server initialized"); + + let rpc_version = env!("CARGO_PKG_VERSION"); + let rpc_version = + semver::Version::parse(rpc_version).context("failed to parse crate version")?; + + tonic::transport::Server::builder() + .accept_http1(true) + .max_connection_age(self.grpc_options.max_connection_age) + .timeout(self.grpc_options.request_timeout) + .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) + .layer( + TraceLayer::new(SharedClassifier::new( + GrpcErrorsAsFailures::new() + .with_success(GrpcCode::InvalidArgument) + .with_success(GrpcCode::NotFound) + .with_success(GrpcCode::ResourceExhausted) + .with_success(GrpcCode::Unimplemented) + .with_success(GrpcCode::Unknown), + )) + .make_span_with(grpc_trace_fn), + ) + .layer(HealthCheckLayer) + .layer(cors_for_grpc_web_layer()) + .layer(GrpcWebLayer::new()) + .layer(grpc::rate_limit_concurrent_connections(self.grpc_options)) + .layer(grpc::rate_limit_per_ip(self.grpc_options)?) + .layer( + AcceptHeaderLayer::new(&rpc_version, genesis.commitment()) + .with_genesis_enforced_method("SubmitProvenTx") + .with_genesis_enforced_method("SubmitProvenTxBatch"), + ) + .add_service(api_service) + .add_service(reflection_service) + .serve_with_incoming(TcpListenerStream::new(self.listener)) + .await + .context("failed to serve embedded RPC API") + } +} + impl Rpc { /// Serves the RPC API. /// diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index a2a30d4c84..b2474a0126 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -13,11 +13,14 @@ pub use accounts::PersistentAccountTree; pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; pub use db::Db; pub use db::models::conv::SqlTypeConvert; -pub use errors::DatabaseError; +pub use errors::{ApplyBlockError, DatabaseError, StateInitializationError}; pub use genesis::GenesisState; +pub use proven_tip::ProvenTipWriter; pub use server::block_prover_client::BlockProver; pub use server::proof_scheduler::DEFAULT_MAX_CONCURRENT_PROOFS; -pub use server::{DataDirectory, DatabaseOptions, Store, StoreMode}; +pub use server::{DataDirectory, DatabaseOptions, Store, StoreApi, StoreMode}; +pub use server::serve_ntx_builder_and_replica; +pub use state::{Finality, State}; /// Returns the default number of SQLite connections used by store database pools. pub fn default_sqlite_connection_pool_size() -> std::num::NonZeroUsize { diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index c5d393eb65..db67372b42 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -41,7 +41,7 @@ pub struct StoreApi { } impl StoreApi { - pub(super) fn new(state: Arc) -> Self { + pub fn new(state: Arc) -> Self { let committed_tip_rx = state.subscribe_committed_tip(); let proven_tip_rx = state.subscribe_proven_tip(); let block_cache = state.block_cache.clone(); diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 5c01b75e51..e25645b5be 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -40,6 +40,8 @@ pub mod proof_scheduler; mod replica; mod rpc_api; +pub use api::StoreApi; + /// Determines how the store receives new blocks. /// /// The two modes are mutually exclusive: a store either accepts blocks from a block producer @@ -519,3 +521,90 @@ impl DataDirectory { self.0.display() } } + +// EMBEDDED SEQUENCER SERVICES +// ================================================================================================ + +/// Runs the proof scheduler and serves the `StoreReplica` and `NtxBuilder` gRPC services. +/// +/// Intended for use by the embedded sequencer, where the store's `BlockProducer` and `Rpc` gRPC +/// services are replaced by in-process equivalents. The proof scheduler subscribes directly to the +/// state's committed-tip watch channel, eliminating the need for the explicit sender used in the +/// legacy `BlockProducerApi`. +/// +/// Runs until any service encounters a fatal error. +pub async fn serve_ntx_builder_and_replica( + state: Arc, + proven_tip: crate::proven_tip::ProvenTipWriter, + ntx_builder_listener: TcpListener, + replica_listener: TcpListener, + block_prover_url: Option, + max_concurrent_proofs: NonZeroUsize, + grpc_options: GrpcOptionsInternal, +) -> anyhow::Result<()> { + let proof_cache = state.proof_cache.clone(); + let chain_tip_rx = state.subscribe_committed_tip(); + + let block_prover = if let Some(url) = block_prover_url { + Arc::new(BlockProver::remote(url)) + } else { + Arc::new(BlockProver::local()) + }; + + let proof_scheduler_task = proof_scheduler::spawn( + block_prover, + state.block_store(), + chain_tip_rx, + proven_tip, + max_concurrent_proofs, + proof_cache, + ); + + let store_api = api::StoreApi::new(state); + + let replica_service = + store::store_replica_server::StoreReplicaServer::new(store_api.clone()); + let ntx_builder_service = + store::ntx_builder_server::NtxBuilderServer::new(store_api); + + let reflection_service = tonic_reflection::server::Builder::configure() + .register_file_descriptor_set(store_api_descriptor()) + .build_v1() + .context("failed to build reflection service")?; + + let make_server = || { + tonic::transport::Server::builder() + .timeout(grpc_options.request_timeout) + .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) + .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn)) + }; + + let mut grpc_servers = JoinSet::new(); + + grpc_servers.spawn( + make_server() + .add_service(replica_service) + .add_service(reflection_service.clone()) + .serve_with_incoming(TcpListenerStream::new(replica_listener)), + ); + + grpc_servers.spawn( + make_server() + .add_service(ntx_builder_service) + .add_service(reflection_service) + .serve_with_incoming(TcpListenerStream::new(ntx_builder_listener)), + ); + + tokio::select! { + result = grpc_servers.join_next() => { + result.expect("grpc_servers joinset is not empty")?.map_err(Into::into) + }, + result = proof_scheduler_task => { + match result { + Ok(Ok(())) => Err(anyhow::anyhow!("proof scheduler exited unexpectedly")), + Ok(Err(err)) => Err(err).context("proof scheduler fatal error"), + Err(join_err) => Err(anyhow::anyhow!(join_err)).context("proof scheduler panicked"), + } + } + } +} diff --git a/scripts/run-node.sh b/scripts/run-node.sh index 3ad9698f45..dac45fe18b 100755 --- a/scripts/run-node.sh +++ b/scripts/run-node.sh @@ -14,26 +14,27 @@ if [[ -n "$KMS_KEY_ID" ]]; then fi GENESIS_CONFIG="crates/store/src/genesis/config/samples/01-simple.toml" -STORE_DIR="/tmp/store" +SEQUENCER_DIR="/tmp/sequencer" STORE_REPLICA_1_DIR="/tmp/store-replica-1" STORE_REPLICA_2_DIR="/tmp/store-replica-2" VALIDATOR_DIR="/tmp/validator" NTX_BUILDER_DIR="/tmp/ntx-builder" ACCOUNTS_DIR="/tmp/accounts" -# Primary store (block-producer mode): 3 APIs. -STORE_RPC_PORT=50001 -STORE_NTX_BUILDER_PORT=50002 -STORE_BLOCK_PRODUCER_PORT=50003 +# Sequencer (store + block-producer + rpc) exposes four endpoints. +SEQUENCER_RPC_PORT=57291 +SEQUENCER_BLOCK_PRODUCER_PORT=50201 +SEQUENCER_NTX_BUILDER_PORT=50002 +SEQUENCER_REPLICA_PORT=50001 -# Replica stores expose only the RPC API (no block-producer or ntx-builder endpoints). +# Replica stores expose only the RPC API. STORE_REPLICA_1_RPC_PORT=50011 STORE_REPLICA_2_RPC_PORT=50021 VALIDATOR_PORT=50101 -BLOCK_PRODUCER_PORT=50201 NTX_BUILDER_PORT=50301 -RPC_PORT=57291 + +# RPC servers backed by replica stores. RPC_REPLICA_1_PORT=57292 RPC_REPLICA_2_PORT=57293 @@ -51,7 +52,18 @@ trap cleanup EXIT INT TERM # --- Kill processes on required ports --- -PORTS=(50001 50002 50003 50011 50021 50101 50201 50301 57291 57292 57293) +PORTS=( + $SEQUENCER_RPC_PORT + $SEQUENCER_BLOCK_PRODUCER_PORT + $SEQUENCER_NTX_BUILDER_PORT + $SEQUENCER_REPLICA_PORT + $STORE_REPLICA_1_RPC_PORT + $STORE_REPLICA_2_RPC_PORT + $VALIDATOR_PORT + $NTX_BUILDER_PORT + $RPC_REPLICA_1_PORT + $RPC_REPLICA_2_PORT +) echo "=== Killing processes on required ports ===" for port in "${PORTS[@]}"; do pids=$(lsof -ti :"$port" 2>/dev/null || true) @@ -69,7 +81,7 @@ sleep 1 if [[ "$SKIP_BOOTSTRAP" != "true" ]]; then echo "=== Bootstrapping ===" - rm -rf "$VALIDATOR_DIR" "$ACCOUNTS_DIR" "$STORE_DIR" \ + rm -rf "$VALIDATOR_DIR" "$ACCOUNTS_DIR" "$SEQUENCER_DIR" \ "$STORE_REPLICA_1_DIR" "$STORE_REPLICA_2_DIR" "$NTX_BUILDER_DIR" mkdir -p "$NTX_BUILDER_DIR" @@ -86,9 +98,9 @@ if [[ "$SKIP_BOOTSTRAP" != "true" ]]; then --genesis-config-file "$GENESIS_CONFIG" \ "${KMS_BOOTSTRAP_ARGS[@]+"${KMS_BOOTSTRAP_ARGS[@]}"}" - echo "Bootstrapping store..." - $BINARY store bootstrap \ - --data-directory "$STORE_DIR" \ + echo "Bootstrapping sequencer..." + $BINARY sequencer bootstrap \ + --data-directory "$SEQUENCER_DIR" \ --genesis-block "$VALIDATOR_DIR/genesis.dat" echo "Bootstrapping store replica 1..." @@ -108,12 +120,19 @@ fi echo "=== Starting components ===" -echo "Starting store (block-producer mode)..." -OTEL_SERVICE_NAME=miden-store-primary $BINARY store start \ - --rpc.listen "0.0.0.0:$STORE_RPC_PORT" \ - --ntx-builder.listen "0.0.0.0:$STORE_NTX_BUILDER_PORT" \ - --block-producer.listen "0.0.0.0:$STORE_BLOCK_PRODUCER_PORT" \ - --data-directory "$STORE_DIR" \ +# The sequencer runs store + block-producer + rpc in a single process and exposes four endpoints: +# rpc.listen → client-facing RPC API +# block-producer.listen → block-producer gRPC API (used by downstream rpc replicas) +# ntx-builder.listen → NtxBuilder gRPC API (used by the standalone ntx-builder binary) +# replica.listen → StoreReplica gRPC API (used by downstream store replicas) +echo "Starting sequencer..." +OTEL_SERVICE_NAME=miden-sequencer $BINARY sequencer start \ + --rpc.listen "0.0.0.0:$SEQUENCER_RPC_PORT" \ + --block-producer.listen "0.0.0.0:$SEQUENCER_BLOCK_PRODUCER_PORT" \ + --ntx-builder.listen "0.0.0.0:$SEQUENCER_NTX_BUILDER_PORT" \ + --replica.listen "0.0.0.0:$SEQUENCER_REPLICA_PORT" \ + --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ + --data-directory "$SEQUENCER_DIR" \ $EXTRA_ARGS & PIDS+=($!) @@ -129,14 +148,14 @@ OTEL_SERVICE_NAME=miden-validator $VALIDATOR_BINARY start --listen "0.0.0.0:$VAL "${KMS_START_ARGS[@]+"${KMS_START_ARGS[@]}"}" & PIDS+=($!) -# Give store and validator a moment to bind their ports. +# Give sequencer and validator a moment to bind their ports. sleep 2 -# Replica 1 syncs from the primary store. -echo "Starting store replica 1 (upstream: primary store at 127.0.0.1:$STORE_RPC_PORT)..." +# Replica 1 syncs from the sequencer's StoreReplica endpoint. +echo "Starting store replica 1 (upstream: sequencer replica at 127.0.0.1:$SEQUENCER_REPLICA_PORT)..." OTEL_SERVICE_NAME=miden-store-replica-1 $BINARY store start-replica \ --rpc.listen "0.0.0.0:$STORE_REPLICA_1_RPC_PORT" \ - --upstream-store.url "http://127.0.0.1:$STORE_RPC_PORT" \ + --upstream-store.url "http://127.0.0.1:$SEQUENCER_REPLICA_PORT" \ --data-directory "$STORE_REPLICA_1_DIR" \ $EXTRA_ARGS & PIDS+=($!) @@ -150,27 +169,12 @@ OTEL_SERVICE_NAME=miden-store-replica-2 $BINARY store start-replica \ $EXTRA_ARGS & PIDS+=($!) -echo "Starting block producer..." -OTEL_SERVICE_NAME=miden-block-producer $BINARY block-producer start --listen "0.0.0.0:$BLOCK_PRODUCER_PORT" \ - --store.url "http://127.0.0.1:$STORE_BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ - $EXTRA_ARGS & -PIDS+=($!) - -echo "Starting RPC server (primary store)..." -OTEL_SERVICE_NAME=miden-rpc-primary $BINARY rpc start \ - --listen "0.0.0.0:$RPC_PORT" \ - --store.url "http://127.0.0.1:$STORE_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ - $EXTRA_ARGS & -PIDS+=($!) - +# RPC servers backed by replica stores forward writes to the sequencer's block-producer endpoint. echo "Starting RPC server (replica 1)..." OTEL_SERVICE_NAME=miden-rpc-replica-1 $BINARY rpc start \ --listen "0.0.0.0:$RPC_REPLICA_1_PORT" \ --store.url "http://127.0.0.1:$STORE_REPLICA_1_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ + --block-producer.url "http://127.0.0.1:$SEQUENCER_BLOCK_PRODUCER_PORT" \ --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ $EXTRA_ARGS & PIDS+=($!) @@ -179,22 +183,24 @@ echo "Starting RPC server (replica 2)..." OTEL_SERVICE_NAME=miden-rpc-replica-2 $BINARY rpc start \ --listen "0.0.0.0:$RPC_REPLICA_2_PORT" \ --store.url "http://127.0.0.1:$STORE_REPLICA_2_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ + --block-producer.url "http://127.0.0.1:$SEQUENCER_BLOCK_PRODUCER_PORT" \ --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ $EXTRA_ARGS & PIDS+=($!) +# The standalone ntx-builder connects to the sequencer's NtxBuilder gRPC endpoint. echo "Starting network transaction builder..." OTEL_SERVICE_NAME=miden-ntx-builder $NTX_BUILDER_BINARY start \ --listen "0.0.0.0:$NTX_BUILDER_PORT" \ - --store.url "http://127.0.0.1:$STORE_NTX_BUILDER_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ + --store.url "http://127.0.0.1:$SEQUENCER_NTX_BUILDER_PORT" \ + --block-producer.url "http://127.0.0.1:$SEQUENCER_BLOCK_PRODUCER_PORT" \ --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ --data-directory "$NTX_BUILDER_DIR" \ $EXTRA_ARGS & PIDS+=($!) echo "=== All components running. Ctrl+C to stop. ===" -echo "=== Block propagation chain: :$STORE_RPC_PORT -> :$STORE_REPLICA_1_RPC_PORT -> :$STORE_REPLICA_2_RPC_PORT ===" -echo "=== RPC endpoints: :$RPC_PORT, :$RPC_REPLICA_1_PORT, :$RPC_REPLICA_2_PORT ===" +echo "=== Sequencer RPC endpoint: :$SEQUENCER_RPC_PORT ===" +echo "=== Replica RPC endpoints: :$RPC_REPLICA_1_PORT, :$RPC_REPLICA_2_PORT ===" +echo "=== Block propagation chain: sequencer(:$SEQUENCER_REPLICA_PORT) -> replica-1(:$STORE_REPLICA_1_RPC_PORT) -> replica-2(:$STORE_REPLICA_2_RPC_PORT) ===" wait From e0d538827f9717fc896b5e1cb02aa5024af59a6a Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 20 May 2026 12:58:14 +1200 Subject: [PATCH 02/15] Lint --- bin/node/src/commands/sequencer.rs | 14 ++++++++------ crates/block-producer/src/server/mod.rs | 5 +---- crates/block-producer/src/store/mod.rs | 10 +++++----- crates/rpc/src/server/api.rs | 18 +++++++----------- crates/store/src/lib.rs | 10 ++++++++-- crates/store/src/server/mod.rs | 6 ++---- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index ec58836a79..022733a7f5 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -9,8 +9,8 @@ use miden_node_rpc::EmbeddedRpc; use miden_node_store::genesis::GenesisBlock; use miden_node_store::{ ApplyBlockError, - DatabaseOptions, DEFAULT_MAX_CONCURRENT_PROOFS, + DatabaseOptions, State, StoreApi, default_sqlite_connection_pool_size, @@ -158,7 +158,9 @@ impl SequencerCommand { block_prover_url, data_directory, max_concurrent_proofs, - DatabaseOptions { connection_pool_size: sqlite_connection_pool_size }, + DatabaseOptions { + connection_pool_size: sqlite_connection_pool_size, + }, block_producer, grpc_options, storage_options, @@ -244,11 +246,11 @@ impl SequencerCommand { ); // The embedded block producer exposes its own gRPC endpoint which the RPC connects to. - let block_producer_url = - Url::parse(&format!("http://{block_producer_listen}")).context("invalid block-producer URL")?; + let block_producer_url = Url::parse(&format!("http://{block_producer_listen}")) + .context("invalid block-producer URL")?; // Similarly, the embedded ntx-builder is reachable on its own gRPC endpoint. - let ntx_builder_url = - Url::parse(&format!("http://{ntx_builder_listen}")).context("invalid ntx-builder URL")?; + let ntx_builder_url = Url::parse(&format!("http://{ntx_builder_listen}")) + .context("invalid ntx-builder URL")?; let rpc_task = tokio::spawn( EmbeddedRpc { diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 55df97436c..3ec825f19e 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -72,10 +72,7 @@ impl EmbeddedBlockProducer { let store = StoreClient::new_local(self.state.clone()); let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); - let chain_tip = self - .state - .chain_tip(miden_node_store::Finality::Committed) - .await; + let chain_tip = self.state.chain_tip(miden_node_store::Finality::Committed).await; let listener = TcpListener::bind(self.block_producer_address) .await diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 061d2e696c..a8f842c81c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -17,7 +17,7 @@ use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::batch::OrderedBatches; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, SignedBlock}; -use miden_protocol::note::Nullifier; +use miden_protocol::note::{NoteHeader, Nullifier}; use miden_protocol::transaction::ProvenTransaction; use miden_protocol::utils::serde::Serializable; use tracing::{debug, info, instrument}; @@ -284,7 +284,7 @@ impl LocalStoreClient { let account_id = proven_tx.account_id(); let nullifiers: Vec = proven_tx.nullifiers().collect(); let unauthenticated_notes: Vec = - proven_tx.unauthenticated_notes().map(|note| note.to_commitment()).collect(); + proven_tx.unauthenticated_notes().map(NoteHeader::to_commitment).collect(); let tx_inputs = self .state @@ -413,10 +413,10 @@ impl LocalStoreClient { /// Supports two backends: a gRPC client for the legacy out-of-process store, and a direct /// in-process handle for the embedded sequencer. #[derive(Clone, Debug)] -#[allow(private_interfaces)] +#[expect(private_interfaces)] pub enum StoreClient { /// Connects to a remote store over gRPC. - Remote(RemoteStoreClient), + Remote(Box), /// Calls an in-process `State` directly, bypassing gRPC. Local(LocalStoreClient), } @@ -424,7 +424,7 @@ pub enum StoreClient { impl StoreClient { /// Creates a gRPC-backed store client with a lazy connection. pub fn new(store_url: Url) -> Self { - Self::Remote(RemoteStoreClient::new(store_url)) + Self::Remote(RemoteStoreClient::new(store_url).into()) } /// Creates an in-process store client backed by the given `State`. diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 708aff733d..10f9cbdebe 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -10,12 +10,12 @@ use miden_node_proto::clients::{ StoreRpcClient, ValidatorClient, }; -use miden_node_proto::generated::store::rpc_server; use miden_node_proto::decode::{read_account_id, read_account_ids, read_block_range}; use miden_node_proto::domain::account::{AccountRequest, SlotData}; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated::rpc::MempoolStats; use miden_node_proto::generated::rpc::api_server::{self, Api}; +use miden_node_proto::generated::store::rpc_server; use miden_node_proto::generated::{self as proto}; use miden_node_proto::try_convert; use miden_node_utils::ErrorReport; @@ -52,7 +52,7 @@ use crate::COMPONENT; /// Dispatches store read calls to either a remote gRPC store or an in-process `StoreApi`. enum StoreBackend { - Remote(StoreRpcClient), + Remote(Box), Embedded(Arc), } @@ -151,9 +151,7 @@ impl StoreBackend { ) -> Result, Status> { match self { Self::Remote(c) => c.clone().sync_account_vault(request).await, - Self::Embedded(api) => { - rpc_server::Rpc::sync_account_vault(api.as_ref(), request).await - }, + Self::Embedded(api) => rpc_server::Rpc::sync_account_vault(api.as_ref(), request).await, } } @@ -173,9 +171,7 @@ impl StoreBackend { ) -> Result, Status> { match self { Self::Remote(c) => c.clone().sync_transactions(request).await, - Self::Embedded(api) => { - rpc_server::Rpc::sync_transactions(api.as_ref(), request).await - }, + Self::Embedded(api) => rpc_server::Rpc::sync_transactions(api.as_ref(), request).await, } } @@ -219,7 +215,8 @@ impl RpcService { .without_metadata_version() .without_metadata_genesis() .with_otel_context_injection() - .connect_lazy::(), + .connect_lazy::() + .into(), ) }; @@ -848,8 +845,7 @@ impl api_server::Api for RpcService { ) -> Result, Status> { debug!(target: COMPONENT, request = ?request); - let store_status = - self.store.status(Request::new(())).await.map(Response::into_inner).ok(); + let store_status = self.store.status(Request::new(())).await.map(Response::into_inner).ok(); let block_producer_status = if let Some(block_producer) = &self.block_producer { block_producer .clone() diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index b2474a0126..c60dbb9b68 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -18,8 +18,14 @@ pub use genesis::GenesisState; pub use proven_tip::ProvenTipWriter; pub use server::block_prover_client::BlockProver; pub use server::proof_scheduler::DEFAULT_MAX_CONCURRENT_PROOFS; -pub use server::{DataDirectory, DatabaseOptions, Store, StoreApi, StoreMode}; -pub use server::serve_ntx_builder_and_replica; +pub use server::{ + DataDirectory, + DatabaseOptions, + Store, + StoreApi, + StoreMode, + serve_ntx_builder_and_replica, +}; pub use state::{Finality, State}; /// Returns the default number of SQLite connections used by store database pools. diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index e25645b5be..cce6cb95db 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -562,10 +562,8 @@ pub async fn serve_ntx_builder_and_replica( let store_api = api::StoreApi::new(state); - let replica_service = - store::store_replica_server::StoreReplicaServer::new(store_api.clone()); - let ntx_builder_service = - store::ntx_builder_server::NtxBuilderServer::new(store_api); + let replica_service = store::store_replica_server::StoreReplicaServer::new(store_api.clone()); + let ntx_builder_service = store::ntx_builder_server::NtxBuilderServer::new(store_api); let reflection_service = tonic_reflection::server::Builder::configure() .register_file_descriptor_set(store_api_descriptor()) From 95e32ddcb3a05dfef835ed45ac99af8c1fd4e750 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 20 May 2026 15:04:30 +1200 Subject: [PATCH 03/15] Add replica.proto and use block producer handle --- Cargo.lock | 1 + bin/node/src/commands/sequencer.rs | 67 +++---- crates/block-producer/src/lib.rs | 2 +- crates/block-producer/src/server/mod.rs | 233 ++++++++++++++++++------ crates/proto/build.rs | 2 + crates/rpc/Cargo.toml | 7 +- crates/rpc/src/lib.rs | 2 +- crates/rpc/src/server/api.rs | 97 +++++----- crates/rpc/src/server/mod.rs | 13 +- crates/store/src/lib.rs | 2 +- crates/store/src/server/mod.rs | 30 ++- crates/store/src/server/replica.rs | 8 +- crates/store/src/server/replica_sync.rs | 14 +- proto/proto/internal/replica.proto | 59 ++++++ 14 files changed, 343 insertions(+), 194 deletions(-) create mode 100644 proto/proto/internal/replica.proto diff --git a/Cargo.lock b/Cargo.lock index e04755e2fb..6a4ed8702e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3344,6 +3344,7 @@ dependencies = [ "futures", "http 1.4.0", "mediatype", + "miden-node-block-producer", "miden-node-proto", "miden-node-proto-build", "miden-node-store", diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index 022733a7f5..2787f80f61 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use anyhow::Context; use miden_node_block_producer::EmbeddedBlockProducer; -use miden_node_rpc::EmbeddedRpc; +use miden_node_rpc::{BlockProducerBackend, EmbeddedRpc}; use miden_node_store::genesis::GenesisBlock; use miden_node_store::{ ApplyBlockError, @@ -14,7 +14,7 @@ use miden_node_store::{ State, StoreApi, default_sqlite_connection_pool_size, - serve_ntx_builder_and_replica, + serve_replica, }; use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; use miden_node_utils::fs::ensure_empty_directory; @@ -28,7 +28,6 @@ use crate::commands::block_producer::BlockProducerConfig; const ENV_RPC_LISTEN: &str = "MIDEN_NODE_SEQUENCER_RPC_LISTEN"; const ENV_BLOCK_PRODUCER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PRODUCER_LISTEN"; -const ENV_NTX_BUILDER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_NTX_BUILDER_LISTEN"; const ENV_REPLICA_LISTEN: &str = "MIDEN_NODE_SEQUENCER_REPLICA_LISTEN"; const ENV_VALIDATOR_URL: &str = "MIDEN_NODE_SEQUENCER_VALIDATOR_URL"; const ENV_BLOCK_PROVER_URL: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PROVER_URL"; @@ -50,9 +49,8 @@ pub enum SequencerCommand { /// Starts the sequencer: store, block-producer, and RPC in a single process. /// - /// Exposes four gRPC endpoints: the client-facing RPC API, the block-producer API, - /// the replica streaming API (for downstream replicas), and the network transaction - /// builder API. + /// Exposes three gRPC endpoints: the client-facing RPC API, the block-producer API, + /// and the replica streaming API (for downstream replicas). Start { /// Socket address at which to serve the client-facing RPC API. #[arg(long = "rpc.listen", env = ENV_RPC_LISTEN, value_name = "LISTEN")] @@ -62,10 +60,6 @@ pub enum SequencerCommand { #[arg(long = "block-producer.listen", env = ENV_BLOCK_PRODUCER_LISTEN, value_name = "LISTEN")] block_producer_listen: SocketAddr, - /// Socket address at which to serve the network transaction builder API. - #[arg(long = "ntx-builder.listen", env = ENV_NTX_BUILDER_LISTEN, value_name = "LISTEN")] - ntx_builder_listen: SocketAddr, - /// Socket address at which to serve the replica streaming API. #[arg(long = "replica.listen", env = ENV_REPLICA_LISTEN, value_name = "LISTEN")] replica_listen: SocketAddr, @@ -124,7 +118,6 @@ impl SequencerCommand { Self::Start { rpc_listen, block_producer_listen, - ntx_builder_listen, replica_listen, validator_url, block_prover_url, @@ -152,7 +145,6 @@ impl SequencerCommand { Self::start( rpc_listen, block_producer_listen, - ntx_builder_listen, replica_listen, validator_url, block_prover_url, @@ -181,7 +173,6 @@ impl SequencerCommand { async fn start( rpc_listen: SocketAddr, block_producer_listen: SocketAddr, - ntx_builder_listen: SocketAddr, replica_listen: SocketAddr, validator_url: Url, block_prover_url: Option, @@ -196,9 +187,6 @@ impl SequencerCommand { let rpc_listener = tokio::net::TcpListener::bind(rpc_listen) .await .context("Failed to bind to RPC gRPC socket")?; - let ntx_builder_listener = tokio::net::TcpListener::bind(ntx_builder_listen) - .await - .context("Failed to bind to ntx-builder gRPC socket")?; let replica_listener = tokio::net::TcpListener::bind(replica_listen) .await .context("Failed to bind to replica gRPC socket")?; @@ -219,54 +207,47 @@ impl SequencerCommand { let store_api = Arc::new(StoreApi::new(Arc::clone(&state))); let grpc_internal = grpc_options.into(); - let ntx_replica_task = tokio::spawn(serve_ntx_builder_and_replica( + let replica_task = tokio::spawn(serve_replica( Arc::clone(&state), proven_tip, - ntx_builder_listener, replica_listener, block_prover_url, max_concurrent_proofs, grpc_internal, )); - let block_producer_task = tokio::spawn( - EmbeddedBlockProducer { - block_producer_address: block_producer_listen, - state: Arc::clone(&state), - validator_url: validator_url.clone(), - batch_prover_url: block_producer_config.batch_prover_url, - batch_interval: block_producer_config.batch_interval, - block_interval: block_producer_config.block_interval, - max_txs_per_batch: block_producer_config.max_txs_per_batch, - max_batches_per_block: block_producer_config.max_batches_per_block, - grpc_options: grpc_internal, - mempool_tx_capacity: block_producer_config.mempool_tx_capacity, - } - .serve(), - ); + let (block_producer_handle, block_producer_serve) = EmbeddedBlockProducer { + block_producer_address: block_producer_listen, + state: Arc::clone(&state), + validator_url: validator_url.clone(), + batch_prover_url: block_producer_config.batch_prover_url, + batch_interval: block_producer_config.batch_interval, + block_interval: block_producer_config.block_interval, + max_txs_per_batch: block_producer_config.max_txs_per_batch, + max_batches_per_block: block_producer_config.max_batches_per_block, + grpc_options: grpc_internal, + mempool_tx_capacity: block_producer_config.mempool_tx_capacity, + } + .start() + .await + .context("failed to start embedded block producer")?; - // The embedded block producer exposes its own gRPC endpoint which the RPC connects to. - let block_producer_url = Url::parse(&format!("http://{block_producer_listen}")) - .context("invalid block-producer URL")?; - // Similarly, the embedded ntx-builder is reachable on its own gRPC endpoint. - let ntx_builder_url = Url::parse(&format!("http://{ntx_builder_listen}")) - .context("invalid ntx-builder URL")?; + let block_producer_task = tokio::spawn(block_producer_serve); let rpc_task = tokio::spawn( EmbeddedRpc { listener: rpc_listener, state: store_api, - block_producer_url: Some(block_producer_url), + block_producer: Some(BlockProducerBackend::Embedded(block_producer_handle)), validator_url, - ntx_builder_url: Some(ntx_builder_url), grpc_options, } .serve(), ); tokio::select! { - result = ntx_replica_task => { - result.context("ntx-builder/replica task panicked")?.context("ntx-builder/replica task failed") + result = replica_task => { + result.context("replica task panicked")?.context("replica task failed") }, result = block_producer_task => { result.context("block-producer task panicked")?.context("block-producer task failed") diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 60889389af..3244b0c8a2 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -18,7 +18,7 @@ pub mod errors; mod errors; pub mod server; -pub use server::{BlockProducer, EmbeddedBlockProducer}; +pub use server::{BlockProducer, BlockProducerHandle, EmbeddedBlockProducer}; // CONSTANTS // ================================================================================================= diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 3ec825f19e..9249b945a5 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -35,6 +35,94 @@ use crate::store::StoreClient; use crate::validator::BlockProducerValidatorClient; use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BUILDERS}; +// BLOCK PRODUCER HANDLE +// ================================================================================================ + +/// A cloneable in-process handle to the embedded block producer. +/// +/// Exposes the same submission and status interface as the gRPC `block_producer.Api` client, but +/// routes calls directly to the in-process mempool without any gRPC round-trip. +#[derive(Clone)] +pub struct BlockProducerHandle { + mempool: SharedMempool, + store: StoreClient, + cached_mempool_stats: Arc>, +} + +impl BlockProducerHandle { + pub async fn submit_proven_tx( + &self, + request: proto::transaction::ProvenTransaction, + ) -> Result, Status> { + let tx = ProvenTransaction::read_from_bytes(&request.transaction) + .map_err(MempoolSubmissionError::DeserializationFailed)?; + + let inputs = self + .store + .get_tx_inputs(&tx) + .await + .map_err(MempoolSubmissionError::StoreConnectionFailed)?; + + let tx = AuthenticatedTransaction::new_unchecked(Arc::new(tx), inputs) + .map(Arc::new) + .map_err(MempoolSubmissionError::StateConflict)?; + + self.mempool + .lock() + .await + .add_transaction(tx) + .map(Into::into) + .map(tonic::Response::new) + .map_err(Into::into) + } + + pub async fn submit_proven_tx_batch( + &self, + request: proto::transaction::TransactionBatch, + ) -> Result, Status> { + let proposed = request + .proposed_batch + .expect("proposed batch existence is enforced by RPC component"); + let batch = ProposedBatch::read_from_bytes(&proposed) + .map_err(MempoolSubmissionError::DeserializationFailed)?; + + let mut txs = Vec::with_capacity(batch.transactions().len()); + for tx in batch.transactions() { + let inputs = self + .store + .get_tx_inputs(tx) + .await + .map_err(MempoolSubmissionError::StoreConnectionFailed)?; + + let tx = AuthenticatedTransaction::new_unchecked(Arc::clone(tx), inputs) + .map(Arc::new) + .map_err(MempoolSubmissionError::StateConflict)?; + txs.push(tx); + } + + self.mempool + .lock() + .await + .add_user_batch(&txs) + .map(Into::into) + .map(tonic::Response::new) + .map_err(Into::into) + } + + pub async fn status( + &self, + _request: tonic::Request<()>, + ) -> Result, Status> { + let stats = *self.cached_mempool_stats.read().await; + Ok(tonic::Response::new(proto::rpc::BlockProducerStatus { + version: env!("CARGO_PKG_VERSION").to_string(), + status: "connected".to_string(), + chain_tip: stats.chain_tip.as_u32(), + mempool_stats: Some(stats.into()), + })) + } +} + // EMBEDDED BLOCK PRODUCER // ================================================================================================ @@ -63,23 +151,23 @@ pub struct EmbeddedBlockProducer { } impl EmbeddedBlockProducer { - /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// Initialises the block producer internals and returns an in-process [`BlockProducerHandle`] + /// together with a future that runs the gRPC server, batch builder, and block builder. /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { + /// Use this when the caller needs to submit transactions directly to the mempool without a + /// gRPC round-trip (e.g. the embedded sequencer's RPC). + pub async fn start( + self, + ) -> anyhow::Result<( + BlockProducerHandle, + impl Future> + Send, + )> { info!(target: COMPONENT, endpoint=?self.block_producer_address, "Initializing embedded server"); let store = StoreClient::new_local(self.state.clone()); let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); let chain_tip = self.state.chain_tip(miden_node_store::Finality::Committed).await; - let listener = TcpListener::bind(self.block_producer_address) - .await - .context("failed to bind to block producer address")?; - - info!(target: COMPONENT, "Embedded server initialized"); - let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); let batch_builder = BatchBuilder::new( store.clone(), @@ -87,7 +175,7 @@ impl EmbeddedBlockProducer { self.batch_prover_url, self.batch_interval, ); - let mempool = MempoolConfig { + let mempool = Mempool::shared(chain_tip, MempoolConfig { batch_budget: BatchBudget { transactions: self.max_txs_per_batch, ..BatchBudget::default() @@ -95,59 +183,86 @@ impl EmbeddedBlockProducer { block_budget: BlockBudget { batches: self.max_batches_per_block }, tx_capacity: self.mempool_tx_capacity, ..Default::default() - }; - let mempool = Mempool::shared(chain_tip, mempool); - - let mut tasks = tokio::task::JoinSet::new(); - - let rpc_id = tasks - .spawn({ - let mempool = mempool.clone(); - async move { - BlockProducerRpcServer::new(mempool, store) - .serve(listener, self.grpc_options) - .await - } - }) - .id(); - - let batch_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { - batch_builder.run(mempool).await; - Ok(()) - } - }) - .id(); - let block_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { block_builder.run(mempool).await } - }) - .id(); + }); - let task_ids = HashMap::from([ - (batch_builder_id, "batch-builder"), - (block_builder_id, "block-builder"), - (rpc_id, "rpc"), - ]); + let cached_mempool_stats = Arc::new(RwLock::new(MempoolStats::default())); + let handle = BlockProducerHandle { + mempool: mempool.clone(), + store: store.clone(), + cached_mempool_stats: Arc::clone(&cached_mempool_stats), + }; - let task_result = tasks.join_next_with_id().await.unwrap(); + let grpc_options = self.grpc_options; + let block_producer_address = self.block_producer_address; - let task_id = match &task_result { - Ok((id, _)) => *id, - Err(err) => err.id(), + let serve = async move { + let listener = TcpListener::bind(block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Embedded server initialized"); + + let rpc_server = BlockProducerRpcServer { + mempool: Mutex::new(mempool.clone()), + store: store.clone(), + cached_mempool_stats, + }; + + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn(async move { rpc_server.serve(listener, grpc_options).await }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + + let block_builder_id = tasks + .spawn({ + async { block_builder.run(mempool).await } + }) + .id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? }; - let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - task_result - .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, result)| match result { - Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), - Err(source) => Err(BlockProducerError::TaskError { task, source }), - }) - .and_then(|x| x)? + Ok((handle, serve)) + } + + /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// + /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is + /// encountered. + pub async fn serve(self) -> anyhow::Result<()> { + let (_handle, serve) = self.start().await?; + serve.await } } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index fa69a59045..16e5e6be79 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -8,6 +8,7 @@ use miden_node_proto_build::{ block_producer_api_descriptor, ntx_builder_api_descriptor, remote_prover_api_descriptor, + replica_api_descriptor, rpc_api_descriptor, store_api_descriptor, validator_api_descriptor, @@ -33,6 +34,7 @@ fn main() -> miette::Result<()> { remote_prover_api_descriptor(), validator_api_descriptor(), ntx_builder_api_descriptor(), + replica_api_descriptor(), ]; for file_descriptors in &descriptor_sets { diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 7d91e97a90..1aef66f9db 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -22,9 +22,10 @@ anyhow = { workspace = true } futures = { workspace = true } http = { workspace = true } mediatype = { version = "0.21" } -miden-node-proto = { workspace = true } -miden-node-proto-build = { workspace = true } -miden-node-store = { workspace = true } +miden-node-block-producer = { workspace = true } +miden-node-proto = { workspace = true } +miden-node-proto-build = { workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-protocol = { default-features = true, workspace = true } miden-tx = { features = ["concurrent"], workspace = true } diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 460ead7d92..fb4e55fe70 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -2,7 +2,7 @@ mod server; #[cfg(test)] mod tests; -pub use server::{EmbeddedRpc, Rpc}; +pub use server::{BlockProducerBackend, EmbeddedRpc, Rpc}; // CONSTANTS // ================================================================================================= diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 10f9cbdebe..fb9429024e 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -191,7 +191,7 @@ impl StoreBackend { pub struct RpcService { store: StoreBackend, - block_producer: Option, + block_producer: Option, validator: ValidatorClient, ntx_builder: Option, genesis_commitment: Option, @@ -226,13 +226,15 @@ impl RpcService { block_producer_endpoint = %block_producer_url, "Initializing block producer client", ); - Builder::new(block_producer_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::() + super::BlockProducerBackend::Remote( + Builder::new(block_producer_url) + .without_tls() + .without_timeout() + .without_metadata_version() + .without_metadata_genesis() + .with_otel_context_injection() + .connect_lazy::(), + ) }); let validator = { @@ -277,26 +279,10 @@ impl RpcService { pub(super) fn new_embedded( state: Arc, - block_producer_url: Option, + block_producer: Option, validator_url: Url, - ntx_builder_url: Option, commitment_cache_capacity: NonZeroUsize, ) -> Self { - let block_producer = block_producer_url.map(|block_producer_url| { - info!( - target: COMPONENT, - block_producer_endpoint = %block_producer_url, - "Initializing block producer client", - ); - Builder::new(block_producer_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::() - }); - let validator = { info!( target: COMPONENT, @@ -312,26 +298,11 @@ impl RpcService { .connect_lazy::() }; - let ntx_builder = ntx_builder_url.map(|ntx_builder_url| { - info!( - target: COMPONENT, - ntx_builder_endpoint = %ntx_builder_url, - "Initializing ntx-builder client", - ); - Builder::new(ntx_builder_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::() - }); - Self { store: StoreBackend::Embedded(state), block_producer, validator, - ntx_builder, + ntx_builder: None, genesis_commitment: None, block_commitment_cache: LruCache::new(commitment_cache_capacity), } @@ -646,7 +617,7 @@ impl api_server::Api for RpcService { ) -> Result, Status> { debug!(target: COMPONENT, request = ?request.get_ref()); - let Some(block_producer) = &self.block_producer else { + let Some(ref block_producer) = self.block_producer else { return Err(Status::unavailable( "Transaction submission not available in read-only mode", )); @@ -720,7 +691,14 @@ impl api_server::Api for RpcService { return Err(Status::invalid_argument("Transaction inputs must be provided")); } - block_producer.clone().submit_proven_tx(request).await + match block_producer { + super::BlockProducerBackend::Embedded(handle) => { + handle.clone().submit_proven_tx(request).await + }, + super::BlockProducerBackend::Remote(client) => { + client.clone().submit_proven_tx(request).await + }, + } } /// Deserializes the batch, strips MAST decorators from full output note scripts, rebuilds @@ -729,7 +707,7 @@ impl api_server::Api for RpcService { &self, request: tonic::Request, ) -> Result, Status> { - let Some(block_producer) = &self.block_producer else { + let Some(ref block_producer) = self.block_producer else { return Err(Status::unavailable("Batch submission not available in read-only mode")); }; @@ -811,7 +789,14 @@ impl api_server::Api for RpcService { self.validator.clone().submit_proven_transaction(request).await?; } - block_producer.clone().submit_proven_tx_batch(request).await + match block_producer { + super::BlockProducerBackend::Embedded(handle) => { + handle.clone().submit_proven_tx_batch(request).await + }, + super::BlockProducerBackend::Remote(client) => { + client.clone().submit_proven_tx_batch(request).await + }, + } } // -- Status & utility endpoints ---------------------------------------------------------- @@ -846,13 +831,21 @@ impl api_server::Api for RpcService { debug!(target: COMPONENT, request = ?request); let store_status = self.store.status(Request::new(())).await.map(Response::into_inner).ok(); - let block_producer_status = if let Some(block_producer) = &self.block_producer { - block_producer - .clone() - .status(Request::new(())) - .await - .map(Response::into_inner) - .ok() + let block_producer_status = if let Some(ref block_producer) = self.block_producer { + match block_producer { + super::BlockProducerBackend::Embedded(handle) => handle + .clone() + .status(Request::new(())) + .await + .map(Response::into_inner) + .ok(), + super::BlockProducerBackend::Remote(client) => client + .clone() + .status(Request::new(())) + .await + .map(Response::into_inner) + .ok(), + } } else { None }; diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 9eb7875399..f34222b455 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use accept::AcceptHeaderLayer; use anyhow::Context; +use miden_node_proto::clients::BlockProducerClient; use miden_node_proto::generated::rpc::api_server; use miden_node_proto_build::rpc_api_descriptor; use miden_node_utils::clap::GrpcOptionsExternal; @@ -26,6 +27,12 @@ mod accept; mod api; mod health; +/// Dispatches block-producer calls to either an in-process handle or a remote gRPC client. +pub enum BlockProducerBackend { + Embedded(miden_node_block_producer::BlockProducerHandle), + Remote(BlockProducerClient), +} + /// The RPC server component. /// /// On startup, binds to the provided listener and starts serving the RPC API. @@ -44,9 +51,8 @@ pub struct Rpc { pub struct EmbeddedRpc { pub listener: TcpListener, pub state: Arc, - pub block_producer_url: Option, + pub block_producer: Option, pub validator_url: Url, - pub ntx_builder_url: Option, pub grpc_options: GrpcOptionsExternal, } @@ -58,9 +64,8 @@ impl EmbeddedRpc { pub async fn serve(self) -> anyhow::Result<()> { let mut api = api::RpcService::new_embedded( self.state, - self.block_producer_url.clone(), + self.block_producer, self.validator_url, - self.ntx_builder_url.clone(), NonZeroUsize::new(1_000_000).unwrap(), ); diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index c60dbb9b68..ca19ffc202 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -24,7 +24,7 @@ pub use server::{ Store, StoreApi, StoreMode, - serve_ntx_builder_and_replica, + serve_replica, }; pub use state::{Finality, State}; diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index cce6cb95db..9b84d7a07e 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -5,8 +5,9 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Context; +use miden_node_proto::generated::replica::api_server as replica_api_server; use miden_node_proto::generated::store; -use miden_node_proto_build::store_api_descriptor; +use miden_node_proto_build::{replica_api_descriptor, store_api_descriptor}; use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::spawn::spawn_blocking_in_span; @@ -324,14 +325,14 @@ impl Store { let mut join_set = JoinSet::new(); let rpc_service = store::rpc_server::RpcServer::new(store_api.clone()); - let replica_service = - store::store_replica_server::StoreReplicaServer::new(store_api.clone()); + let replica_service = replica_api_server::ApiServer::new(store_api.clone()); let ntx_builder_service = store::ntx_builder_server::NtxBuilderServer::new(store_api); let block_producer_service = store::block_producer_server::BlockProducerServer::new(block_producer_api); let reflection_service = tonic_reflection::server::Builder::configure() .register_file_descriptor_set(store_api_descriptor()) + .register_file_descriptor_set(replica_api_descriptor()) .build_v1() .context("failed to build reflection service")?; @@ -380,10 +381,11 @@ impl Store { let mut join_set = JoinSet::new(); let rpc_service = store::rpc_server::RpcServer::new(store_api.clone()); - let replica_service = store::store_replica_server::StoreReplicaServer::new(store_api); + let replica_service = replica_api_server::ApiServer::new(store_api); let reflection_service = tonic_reflection::server::Builder::configure() .register_file_descriptor_set(store_api_descriptor()) + .register_file_descriptor_set(replica_api_descriptor()) .build_v1() .context("failed to build reflection service")?; @@ -525,7 +527,7 @@ impl DataDirectory { // EMBEDDED SEQUENCER SERVICES // ================================================================================================ -/// Runs the proof scheduler and serves the `StoreReplica` and `NtxBuilder` gRPC services. +/// Runs the proof scheduler and serves the `replica.Api` gRPC service. /// /// Intended for use by the embedded sequencer, where the store's `BlockProducer` and `Rpc` gRPC /// services are replaced by in-process equivalents. The proof scheduler subscribes directly to the @@ -533,10 +535,9 @@ impl DataDirectory { /// legacy `BlockProducerApi`. /// /// Runs until any service encounters a fatal error. -pub async fn serve_ntx_builder_and_replica( +pub async fn serve_replica( state: Arc, proven_tip: crate::proven_tip::ProvenTipWriter, - ntx_builder_listener: TcpListener, replica_listener: TcpListener, block_prover_url: Option, max_concurrent_proofs: NonZeroUsize, @@ -561,12 +562,10 @@ pub async fn serve_ntx_builder_and_replica( ); let store_api = api::StoreApi::new(state); - - let replica_service = store::store_replica_server::StoreReplicaServer::new(store_api.clone()); - let ntx_builder_service = store::ntx_builder_server::NtxBuilderServer::new(store_api); + let replica_service = replica_api_server::ApiServer::new(store_api); let reflection_service = tonic_reflection::server::Builder::configure() - .register_file_descriptor_set(store_api_descriptor()) + .register_file_descriptor_set(replica_api_descriptor()) .build_v1() .context("failed to build reflection service")?; @@ -582,15 +581,8 @@ pub async fn serve_ntx_builder_and_replica( grpc_servers.spawn( make_server() .add_service(replica_service) - .add_service(reflection_service.clone()) - .serve_with_incoming(TcpListenerStream::new(replica_listener)), - ); - - grpc_servers.spawn( - make_server() - .add_service(ntx_builder_service) .add_service(reflection_service) - .serve_with_incoming(TcpListenerStream::new(ntx_builder_listener)), + .serve_with_incoming(TcpListenerStream::new(replica_listener)), ); tokio::select! { diff --git a/crates/store/src/server/replica.rs b/crates/store/src/server/replica.rs index 5f0162262b..399b33aeb8 100644 --- a/crates/store/src/server/replica.rs +++ b/crates/store/src/server/replica.rs @@ -2,12 +2,12 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use miden_node_proto::generated::store::{ +use miden_node_proto::generated::replica::{ BlockProof, BlockSubscriptionRequest, ProofSubscriptionRequest, SignedBlock, - store_replica_server, + api_server, }; use miden_node_utils::ErrorReport; use miden_protocol::block::BlockNumber; @@ -39,11 +39,11 @@ impl Stream for GuardedStream { } } -// STORE REPLICA API +// REPLICA API // ================================================================================================ #[tonic::async_trait] -impl store_replica_server::StoreReplica for StoreApi { +impl api_server::Api for StoreApi { type BlockSubscriptionStream = Pin< Box< dyn tonic::codegen::tokio_stream::Stream> diff --git a/crates/store/src/server/replica_sync.rs b/crates/store/src/server/replica_sync.rs index 0bb0553443..cf68433aa4 100644 --- a/crates/store/src/server/replica_sync.rs +++ b/crates/store/src/server/replica_sync.rs @@ -4,10 +4,10 @@ use std::time::Duration; use anyhow::Context; use async_trait::async_trait; use miden_crypto::utils::Deserializable; -use miden_node_proto::generated::store::{ +use miden_node_proto::generated::replica::{ BlockSubscriptionRequest, ProofSubscriptionRequest, - store_replica_client, + api_client, }; use miden_protocol::block::{BlockNumber, SignedBlock}; use tokio_stream::StreamExt; @@ -18,7 +18,7 @@ use crate::state::{Finality, State}; pub(crate) const RECONNECT_DELAY: Duration = Duration::from_secs(5); -type StoreReplicaClient = store_replica_client::StoreReplicaClient; +type ReplicaApiClient = api_client::ApiClient; // REPLICA SYNC // ================================================================================================ @@ -40,7 +40,7 @@ pub(crate) trait ReplicaSync: Sized + Send + Sync + 'static { /// Subscribes to the upstream stream via `client` and processes events until the stream ends /// or an error occurs. - async fn subscribe(&self, client: StoreReplicaClient) -> anyhow::Result<()>; + async fn subscribe(&self, client: ReplicaApiClient) -> anyhow::Result<()>; /// Opens a connection to [`upstream_url`](Self::upstream_url) and calls /// [`subscribe`](Self::subscribe) with the resulting client. @@ -48,7 +48,7 @@ pub(crate) trait ReplicaSync: Sized + Send + Sync + 'static { let channel = tonic::transport::Channel::from_shared(self.upstream_url().to_string())? .connect() .await?; - self.subscribe(StoreReplicaClient::new(channel)).await + self.subscribe(ReplicaApiClient::new(channel)).await } /// Runs [`sync`](Self::sync) in an infinite loop, sleeping [`RECONNECT_DELAY`] on failure. @@ -98,7 +98,7 @@ impl ReplicaSync for BlockReplicaSync { &self.upstream_url } - async fn subscribe(&self, mut client: StoreReplicaClient) -> anyhow::Result<()> { + async fn subscribe(&self, mut client: ReplicaApiClient) -> anyhow::Result<()> { let block_from = self.state.chain_tip(Finality::Committed).await.child().as_u32(); info!(block_from, upstream_url = %self.upstream_url, "Connecting to upstream store for blocks"); @@ -141,7 +141,7 @@ impl ReplicaSync for ProofReplicaSync { &self.upstream_url } - async fn subscribe(&self, mut client: StoreReplicaClient) -> anyhow::Result<()> { + async fn subscribe(&self, mut client: ReplicaApiClient) -> anyhow::Result<()> { let block_from = self.state.chain_tip(Finality::Proven).await.as_u32().saturating_add(1); info!(block_from, upstream_url = %self.upstream_url, "Connecting to upstream store for proofs"); diff --git a/proto/proto/internal/replica.proto b/proto/proto/internal/replica.proto new file mode 100644 index 0000000000..b265a5a2df --- /dev/null +++ b/proto/proto/internal/replica.proto @@ -0,0 +1,59 @@ +// Internal gRPC API for replica synchronization. +syntax = "proto3"; +package replica; + +// API for replica synchronization. +// +// Any sequencer exposes this service so that replicas can subscribe to a continuous stream +// of committed blocks and block proofs. Replicas connect, specify their current local tip (or 0 +// for genesis), receive historical blocks first (catch-up), and then receive live blocks as they +// are committed. +service Api { + // Streams committed blocks starting from the given block number (inclusive). + // + // Replays historical blocks first, then streams live blocks as they are committed. + // On lag (replica falls too far behind), the stream is closed with a DATA_LOSS error and + // the client should reconnect from its local tip. + rpc BlockSubscription(BlockSubscriptionRequest) returns (stream SignedBlock) {} + + // Streams block proofs starting from the given block number (inclusive). + // + // Replays existing proofs first, then streams new proofs as they are generated. + // On lag, the stream is closed with a DATA_LOSS error. + rpc ProofSubscription(ProofSubscriptionRequest) returns (stream BlockProof) {} +} + +// BLOCK SUBSCRIPTION +// ================================================================================================ + +// Request to subscribe to the committed block stream. +message BlockSubscriptionRequest { + // The block number to start streaming from (inclusive). + fixed32 block_from = 1; +} + +// A committed block streamed to a replica. +message SignedBlock { + // The block encoded using [miden_serde_utils::Serializable] implementation for + // [miden_protocol::block::SignedBlock]. + bytes block = 1; +} + +// PROOF SUBSCRIPTION +// ================================================================================================ + +// Request to subscribe to the block proof stream. +message ProofSubscriptionRequest { + // The block number to start streaming from (inclusive). + fixed32 block_from = 1; +} + +// A block proof streamed to a replica. +message BlockProof { + // The block number this proof corresponds to. + fixed32 block_num = 1; + + // The block proof encoded using [miden_serde_utils::Serializable] implementation for + // [miden_protocol::block::BlockProof]. + bytes proof = 2; +} From 5a14064ddfc3a583cd71186d12c13f4d3a19b9de Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 20 May 2026 15:51:02 +1200 Subject: [PATCH 04/15] Move ntx builder api to sequencer --- bin/node/src/commands/sequencer.rs | 21 +++++++ bin/ntx-builder/src/clients/store.rs | 8 +-- crates/proto/build.rs | 2 + crates/proto/src/clients/mod.rs | 24 ++++++++ crates/store/src/lib.rs | 1 + crates/store/src/server/mod.rs | 32 ++++++++++- crates/store/src/server/ntx_builder.rs | 76 ++++++++++++++++++++++++++ scripts/run-node.sh | 6 +- 8 files changed, 162 insertions(+), 8 deletions(-) diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index 2787f80f61..51286c9de9 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -15,6 +15,7 @@ use miden_node_store::{ StoreApi, default_sqlite_connection_pool_size, serve_replica, + serve_sequencer_ntx_builder, }; use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; use miden_node_utils::fs::ensure_empty_directory; @@ -28,6 +29,7 @@ use crate::commands::block_producer::BlockProducerConfig; const ENV_RPC_LISTEN: &str = "MIDEN_NODE_SEQUENCER_RPC_LISTEN"; const ENV_BLOCK_PRODUCER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PRODUCER_LISTEN"; +const ENV_NTX_BUILDER_LISTEN: &str = "MIDEN_NODE_SEQUENCER_NTX_BUILDER_LISTEN"; const ENV_REPLICA_LISTEN: &str = "MIDEN_NODE_SEQUENCER_REPLICA_LISTEN"; const ENV_VALIDATOR_URL: &str = "MIDEN_NODE_SEQUENCER_VALIDATOR_URL"; const ENV_BLOCK_PROVER_URL: &str = "MIDEN_NODE_SEQUENCER_BLOCK_PROVER_URL"; @@ -60,6 +62,10 @@ pub enum SequencerCommand { #[arg(long = "block-producer.listen", env = ENV_BLOCK_PRODUCER_LISTEN, value_name = "LISTEN")] block_producer_listen: SocketAddr, + /// Socket address at which to serve the sequencer.NtxBuilder gRPC API for the ntx-builder. + #[arg(long = "ntx-builder.listen", env = ENV_NTX_BUILDER_LISTEN, value_name = "LISTEN")] + ntx_builder_listen: SocketAddr, + /// Socket address at which to serve the replica streaming API. #[arg(long = "replica.listen", env = ENV_REPLICA_LISTEN, value_name = "LISTEN")] replica_listen: SocketAddr, @@ -118,6 +124,7 @@ impl SequencerCommand { Self::Start { rpc_listen, block_producer_listen, + ntx_builder_listen, replica_listen, validator_url, block_prover_url, @@ -145,6 +152,7 @@ impl SequencerCommand { Self::start( rpc_listen, block_producer_listen, + ntx_builder_listen, replica_listen, validator_url, block_prover_url, @@ -173,6 +181,7 @@ impl SequencerCommand { async fn start( rpc_listen: SocketAddr, block_producer_listen: SocketAddr, + ntx_builder_listen: SocketAddr, replica_listen: SocketAddr, validator_url: Url, block_prover_url: Option, @@ -187,6 +196,9 @@ impl SequencerCommand { let rpc_listener = tokio::net::TcpListener::bind(rpc_listen) .await .context("Failed to bind to RPC gRPC socket")?; + let ntx_builder_listener = tokio::net::TcpListener::bind(ntx_builder_listen) + .await + .context("Failed to bind to ntx-builder gRPC socket")?; let replica_listener = tokio::net::TcpListener::bind(replica_listen) .await .context("Failed to bind to replica gRPC socket")?; @@ -216,6 +228,12 @@ impl SequencerCommand { grpc_internal, )); + let ntx_builder_task = tokio::spawn(serve_sequencer_ntx_builder( + Arc::clone(&state), + ntx_builder_listener, + grpc_internal, + )); + let (block_producer_handle, block_producer_serve) = EmbeddedBlockProducer { block_producer_address: block_producer_listen, state: Arc::clone(&state), @@ -249,6 +267,9 @@ impl SequencerCommand { result = replica_task => { result.context("replica task panicked")?.context("replica task failed") }, + result = ntx_builder_task => { + result.context("ntx-builder task panicked")?.context("ntx-builder task failed") + }, result = block_producer_task => { result.context("block-producer task panicked")?.context("block-producer task failed") }, diff --git a/bin/ntx-builder/src/clients/store.rs b/bin/ntx-builder/src/clients/store.rs index c901fc69ab..2513f07606 100644 --- a/bin/ntx-builder/src/clients/store.rs +++ b/bin/ntx-builder/src/clients/store.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use std::ops::RangeInclusive; use std::time::Duration; -use miden_node_proto::clients::{Builder, StoreNtxBuilderClient}; +use miden_node_proto::clients::{Builder, SequencerNtxBuilderClient}; use miden_node_proto::decode::ConversionResultExt; use miden_node_proto::domain::account::{AccountDetails, AccountResponse, NetworkAccountId}; use miden_node_proto::errors::ConversionError; @@ -38,12 +38,12 @@ use crate::COMPONENT; // STORE CLIENT // ================================================================================================ -/// Interface to the store's ntx-builder gRPC API. +/// Interface to the sequencer's ntx-builder gRPC API. /// /// Essentially just a thin wrapper around the generated gRPC client which improves type safety. #[derive(Clone, Debug)] pub struct StoreClient { - inner: StoreNtxBuilderClient, + inner: SequencerNtxBuilderClient, } impl StoreClient { @@ -57,7 +57,7 @@ impl StoreClient { .without_metadata_version() .without_metadata_genesis() .with_otel_context_injection() - .connect_lazy::(); + .connect_lazy::(); Self { inner: store } } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index 16e5e6be79..408a9d26fa 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -10,6 +10,7 @@ use miden_node_proto_build::{ remote_prover_api_descriptor, replica_api_descriptor, rpc_api_descriptor, + sequencer_api_descriptor, store_api_descriptor, validator_api_descriptor, }; @@ -35,6 +36,7 @@ fn main() -> miette::Result<()> { validator_api_descriptor(), ntx_builder_api_descriptor(), replica_api_descriptor(), + sequencer_api_descriptor(), ]; for file_descriptors in &descriptor_sets { diff --git a/crates/proto/src/clients/mod.rs b/crates/proto/src/clients/mod.rs index 66f11a5268..abae2b4be4 100644 --- a/crates/proto/src/clients/mod.rs +++ b/crates/proto/src/clients/mod.rs @@ -121,6 +121,8 @@ type GeneratedProxyStatusClient = type GeneratedProverClient = generated::remote_prover::api_client::ApiClient; type GeneratedValidatorClient = generated::validator::api_client::ApiClient; type GeneratedNtxBuilderClient = generated::ntx_builder::api_client::ApiClient; +type GeneratedSequencerNtxBuilderClient = + generated::sequencer::ntx_builder_client::NtxBuilderClient; // gRPC CLIENTS // ================================================================================================ @@ -143,6 +145,8 @@ pub struct RemoteProverClient(GeneratedProverClient); pub struct ValidatorClient(GeneratedValidatorClient); #[derive(Debug, Clone)] pub struct NtxBuilderClient(GeneratedNtxBuilderClient); +#[derive(Debug, Clone)] +pub struct SequencerNtxBuilderClient(GeneratedSequencerNtxBuilderClient); impl DerefMut for RpcClient { fn deref_mut(&mut self) -> &mut Self::Target { @@ -338,6 +342,26 @@ impl GrpcClient for NtxBuilderClient { } } +impl DerefMut for SequencerNtxBuilderClient { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for SequencerNtxBuilderClient { + type Target = GeneratedSequencerNtxBuilderClient; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl GrpcClient for SequencerNtxBuilderClient { + fn with_interceptor(channel: Channel, interceptor: Interceptor) -> Self { + Self(GeneratedSequencerNtxBuilderClient::new(InterceptedService::new(channel, interceptor))) + } +} + // STRICT TYPE-SAFE BUILDER (NO DEFAULTS) // ================================================================================================ diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ca19ffc202..e1aa0f47c2 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -25,6 +25,7 @@ pub use server::{ StoreApi, StoreMode, serve_replica, + serve_sequencer_ntx_builder, }; pub use state::{Finality, State}; diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 9b84d7a07e..da1b7b58f2 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -6,8 +6,9 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::generated::replica::api_server as replica_api_server; +use miden_node_proto::generated::sequencer::ntx_builder_server as sequencer_ntx_builder_server; use miden_node_proto::generated::store; -use miden_node_proto_build::{replica_api_descriptor, store_api_descriptor}; +use miden_node_proto_build::{replica_api_descriptor, sequencer_api_descriptor, store_api_descriptor}; use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::spawn::spawn_blocking_in_span; @@ -598,3 +599,32 @@ pub async fn serve_replica( } } } + +/// Serves the `sequencer.NtxBuilder` gRPC service for the embedded sequencer. +/// +/// The ntx-builder binary connects here (via `--store.url`) to get chain data and network notes. +/// This mirrors `store.NtxBuilder` in functionality but is served on its own listener so the +/// embedded sequencer does not depend on the standalone store binary. +pub async fn serve_sequencer_ntx_builder( + state: Arc, + ntx_builder_listener: TcpListener, + grpc_options: GrpcOptionsInternal, +) -> anyhow::Result<()> { + let store_api = api::StoreApi::new(state); + let ntx_builder_service = sequencer_ntx_builder_server::NtxBuilderServer::new(store_api); + + let reflection_service = tonic_reflection::server::Builder::configure() + .register_file_descriptor_set(sequencer_api_descriptor()) + .build_v1() + .context("failed to build reflection service")?; + + tonic::transport::Server::builder() + .timeout(grpc_options.request_timeout) + .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) + .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn)) + .add_service(ntx_builder_service) + .add_service(reflection_service) + .serve_with_incoming(TcpListenerStream::new(ntx_builder_listener)) + .await + .context("failed to serve sequencer ntx-builder API") +} diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 3b114da5f0..9c88c6e618 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -7,6 +7,7 @@ use miden_node_proto::domain::account::AccountInfo; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated as proto; use miden_node_proto::generated::rpc::BlockRange; +use miden_node_proto::generated::sequencer::ntx_builder_server as sequencer_ntx_builder_server; use miden_node_proto::generated::store::ntx_builder_server; use miden_node_utils::ErrorReport; use miden_protocol::account::{StorageMapKey, StorageSlotName}; @@ -321,3 +322,78 @@ impl ntx_builder_server::NtxBuilder for StoreApi { })) } } + +// SEQUENCER NTX BUILDER ENDPOINTS +// ================================================================================================ + +/// Forwards `sequencer.NtxBuilder` calls to the same `store.NtxBuilder` implementation. +/// +/// The two services expose an identical interface; the only difference is which proto package +/// hosts the service declaration. The embedded sequencer serves `sequencer.NtxBuilder` so the +/// ntx-builder binary can point its `--store.url` at the sequencer without depending on the +/// standalone store binary. +#[tonic::async_trait] +impl sequencer_ntx_builder_server::NtxBuilder for StoreApi { + async fn get_block_header_by_number( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_block_header_by_number(self, request).await + } + + async fn get_current_blockchain_data( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_current_blockchain_data(self, request).await + } + + async fn get_network_account_details_by_id( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_network_account_details_by_id(self, request).await + } + + async fn get_unconsumed_network_notes( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_unconsumed_network_notes(self, request).await + } + + async fn get_network_account_ids( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_network_account_ids(self, request).await + } + + async fn get_account( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_account(self, request).await + } + + async fn get_note_script_by_root( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_note_script_by_root(self, request).await + } + + async fn get_vault_asset_witnesses( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_vault_asset_witnesses(self, request).await + } + + async fn get_storage_map_witness( + &self, + request: Request, + ) -> Result, Status> { + ntx_builder_server::NtxBuilder::get_storage_map_witness(self, request).await + } +} diff --git a/scripts/run-node.sh b/scripts/run-node.sh index dac45fe18b..70b52d45d3 100755 --- a/scripts/run-node.sh +++ b/scripts/run-node.sh @@ -123,8 +123,8 @@ echo "=== Starting components ===" # The sequencer runs store + block-producer + rpc in a single process and exposes four endpoints: # rpc.listen → client-facing RPC API # block-producer.listen → block-producer gRPC API (used by downstream rpc replicas) -# ntx-builder.listen → NtxBuilder gRPC API (used by the standalone ntx-builder binary) -# replica.listen → StoreReplica gRPC API (used by downstream store replicas) +# ntx-builder.listen → sequencer.NtxBuilder gRPC API (used by the standalone ntx-builder binary) +# replica.listen → replica.Api gRPC API (used by downstream store replicas) echo "Starting sequencer..." OTEL_SERVICE_NAME=miden-sequencer $BINARY sequencer start \ --rpc.listen "0.0.0.0:$SEQUENCER_RPC_PORT" \ @@ -151,7 +151,7 @@ PIDS+=($!) # Give sequencer and validator a moment to bind their ports. sleep 2 -# Replica 1 syncs from the sequencer's StoreReplica endpoint. +# Replica 1 syncs from the sequencer's replica.Api endpoint. echo "Starting store replica 1 (upstream: sequencer replica at 127.0.0.1:$SEQUENCER_REPLICA_PORT)..." OTEL_SERVICE_NAME=miden-store-replica-1 $BINARY store start-replica \ --rpc.listen "0.0.0.0:$STORE_REPLICA_1_RPC_PORT" \ From 3243abc3b375f5deb91949dacbb2ad35de490f6d Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 14:05:14 +1200 Subject: [PATCH 05/15] Comments and include seq proto --- bin/node/.env | 8 ++--- proto/proto/internal/sequencer.proto | 48 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 proto/proto/internal/sequencer.proto diff --git a/bin/node/.env b/bin/node/.env index 7335a99398..7fbc857284 100644 --- a/bin/node/.env +++ b/bin/node/.env @@ -5,7 +5,7 @@ MIDEN_NODE_ENABLE_OTEL=true MIDEN_NODE_DATA_DIRECTORY= -# Sequencer (store + block-producer + rpc in one process) +# Sequencer MIDEN_NODE_SEQUENCER_RPC_LISTEN=0.0.0.0:57291 MIDEN_NODE_SEQUENCER_BLOCK_PRODUCER_LISTEN= MIDEN_NODE_SEQUENCER_NTX_BUILDER_LISTEN= @@ -14,7 +14,7 @@ MIDEN_NODE_SEQUENCER_VALIDATOR_URL= MIDEN_NODE_SEQUENCER_BLOCK_PROVER_URL= MIDEN_NODE_SEQUENCER_SQLITE_CONNECTION_POOL_SIZE= -# Block Producer (standalone) +# Block Producer MIDEN_NODE_BLOCK_PRODUCER_LISTEN= MIDEN_NODE_BLOCK_PRODUCER_STORE_URL= MIDEN_NODE_BLOCK_PRODUCER_VALIDATOR_URL= @@ -23,14 +23,14 @@ MIDEN_NODE_BLOCK_PRODUCER_MAX_BATCHES_PER_BLOCK= MIDEN_NODE_BLOCK_PRODUCER_MEMPOOL_TX_CAPACITY= MIDEN_NODE_BLOCK_PRODUCER_BATCH_PROVER_URL= -# Store (standalone) +# Store MIDEN_NODE_STORE_RPC_LISTEN= MIDEN_NODE_STORE_UPSTREAM_RPC_URL= MIDEN_NODE_STORE_NTX_BUILDER_LISTEN= MIDEN_NODE_STORE_BLOCK_PRODUCER_LISTEN= MIDEN_NODE_STORE_BLOCK_PROVER_URL= -# RPC (standalone) +# RPC MIDEN_NODE_RPC_LISTEN=0.0.0.0:57291 MIDEN_NODE_RPC_STORE_URL= MIDEN_NODE_RPC_BLOCK_PRODUCER_URL= diff --git a/proto/proto/internal/sequencer.proto b/proto/proto/internal/sequencer.proto new file mode 100644 index 0000000000..7cb544aeaf --- /dev/null +++ b/proto/proto/internal/sequencer.proto @@ -0,0 +1,48 @@ +// Internal gRPC API for the embedded sequencer component. +syntax = "proto3"; +package sequencer; + +import "types/account.proto"; +import "types/blockchain.proto"; +import "types/note.proto"; +import "rpc.proto"; +import "internal/store.proto"; + +// NTX BUILDER API +// ================================================================================================ + +// Sequencer-side store API for the network transaction builder. +// +// Mirrors store.NtxBuilder but is served by the embedded sequencer rather than the standalone +// store. Point the ntx-builder's --store.url at the sequencer's --ntx-builder.listen address. +service NtxBuilder { + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. + rpc GetBlockHeaderByNumber(rpc.BlockHeaderByNumberRequest) returns (rpc.BlockHeaderByNumberResponse) {} + + // Returns a paginated list of unconsumed network notes. + rpc GetUnconsumedNetworkNotes(store.UnconsumedNetworkNotesRequest) returns (store.UnconsumedNetworkNotes) {} + + // Returns the block header at the chain tip, as well as the MMR peaks corresponding to this + // header for executing network transactions. If the block number is not provided, the latest + // header and peaks will be retrieved. + rpc GetCurrentBlockchainData(blockchain.MaybeBlockNumber) returns (store.CurrentBlockchainData) {} + + // Returns the latest state of a network account with the specified account ID. + rpc GetNetworkAccountDetailsById(account.AccountId) returns (store.MaybeAccountDetails) {} + + // Returns a list of all network account ids. + rpc GetNetworkAccountIds(rpc.BlockRange) returns (store.NetworkAccountIdList) {} + + // Returns the latest details of the specified account. + rpc GetAccount(rpc.AccountRequest) returns (rpc.AccountResponse) {} + + // Returns the script for a note by its root. + rpc GetNoteScriptByRoot(note.NoteScriptRoot) returns (rpc.MaybeNoteScript) {} + + // Returns vault asset witnesses for the specified account. + rpc GetVaultAssetWitnesses(store.VaultAssetWitnessesRequest) returns (store.VaultAssetWitnessesResponse) {} + + // Returns a storage map witness for the specified account and storage map entry. + rpc GetStorageMapWitness(store.StorageMapWitnessRequest) returns (store.StorageMapWitnessResponse) {} +} From 243b36e6eb68acadfe2ddeb65e3317302ff358fa Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 14:25:31 +1200 Subject: [PATCH 06/15] Fix lint --- bin/node/src/commands/sequencer.rs | 28 ++++++++++-------- crates/block-producer/src/server/mod.rs | 2 +- crates/rpc/Cargo.toml | 38 ++++++++++++------------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index 51286c9de9..9e3c06fa01 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -71,12 +71,12 @@ pub enum SequencerCommand { replica_listen: SocketAddr, /// The validator's gRPC url. - #[arg(long = "validator.url", env = ENV_VALIDATOR_URL, value_name = "URL")] - validator_url: Url, + #[arg(long = "validator.url", env = ENV_VALIDATOR_URL, value_name = "URL", value_parser = parse_boxed_url)] + validator_url: Box, /// The remote block prover's gRPC url. If not provided, a local block prover will be used. - #[arg(long = "block-prover.url", env = ENV_BLOCK_PROVER_URL, value_name = "URL")] - block_prover_url: Option, + #[arg(long = "block-prover.url", env = ENV_BLOCK_PROVER_URL, value_name = "URL", value_parser = parse_boxed_url)] + block_prover_url: Option>, /// Directory in which to store the database and raw block data. #[arg(long, env = ENV_DATA_DIRECTORY, value_name = "DIR")] @@ -104,13 +104,13 @@ pub enum SequencerCommand { sqlite_connection_pool_size: NonZeroUsize, #[command(flatten)] - block_producer: BlockProducerConfig, + block_producer: Box, #[command(flatten)] - grpc_options: GrpcOptionsExternal, + grpc_options: Box, #[command(flatten)] - storage_options: StorageOptions, + storage_options: Box, }, } @@ -154,16 +154,16 @@ impl SequencerCommand { block_producer_listen, ntx_builder_listen, replica_listen, - validator_url, - block_prover_url, + *validator_url, + block_prover_url.map(|b| *b), data_directory, max_concurrent_proofs, DatabaseOptions { connection_pool_size: sqlite_connection_pool_size, }, - block_producer, - grpc_options, - storage_options, + *block_producer, + *grpc_options, + *storage_options, ) .await }, @@ -292,3 +292,7 @@ fn bootstrap_sequencer(data_directory: &Path, genesis_block_path: &Path) -> anyh miden_node_store::Store::bootstrap(genesis_block, data_directory) } + +fn parse_boxed_url(s: &str) -> Result, url::ParseError> { + s.parse::().map(Box::new) +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index d554944ce2..f36b70211d 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -224,7 +224,7 @@ impl EmbeddedBlockProducer { }) .id(); - let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await } ).id(); + let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await }).id(); let task_ids = HashMap::from([ (batch_builder_id, "batch-builder"), diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 1aef66f9db..3819f6cc59 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -18,29 +18,29 @@ workspace = true doctest = false [dependencies] -anyhow = { workspace = true } -futures = { workspace = true } -http = { workspace = true } -mediatype = { version = "0.21" } +anyhow = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +mediatype = { version = "0.21" } miden-node-block-producer = { workspace = true } miden-node-proto = { workspace = true } miden-node-proto-build = { workspace = true } miden-node-store = { workspace = true } -miden-node-utils = { workspace = true } -miden-protocol = { default-features = true, workspace = true } -miden-tx = { features = ["concurrent"], workspace = true } -miden-tx-batch-prover = { workspace = true } -semver = { version = "1.0" } -thiserror = { workspace = true } -tokio = { features = ["macros", "net", "rt-multi-thread"], workspace = true } -tokio-stream = { features = ["net"], workspace = true } -tonic = { default-features = true, features = ["tls-native-roots", "tls-ring"], workspace = true } -tonic-reflection = { workspace = true } -tonic-web = { workspace = true } -tower = { workspace = true } -tower-http = { features = ["trace"], workspace = true } -tracing = { workspace = true } -url = { workspace = true } +miden-node-utils = { workspace = true } +miden-protocol = { default-features = true, workspace = true } +miden-tx = { features = ["concurrent"], workspace = true } +miden-tx-batch-prover = { workspace = true } +semver = { version = "1.0" } +thiserror = { workspace = true } +tokio = { features = ["macros", "net", "rt-multi-thread"], workspace = true } +tokio-stream = { features = ["net"], workspace = true } +tonic = { default-features = true, features = ["tls-native-roots", "tls-ring"], workspace = true } +tonic-reflection = { workspace = true } +tonic-web = { workspace = true } +tower = { workspace = true } +tower-http = { features = ["trace"], workspace = true } +tracing = { workspace = true } +url = { workspace = true } [dev-dependencies] miden-node-utils = { features = ["testing", "tracing-forest"], workspace = true } From ca1ff8525010eff0dda373e2a93b8ce14deafa0a Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 14:36:58 +1200 Subject: [PATCH 07/15] Rename to sequencer.NtxBuilderApi --- bin/node/src/commands/sequencer.rs | 2 +- crates/proto/src/clients/mod.rs | 2 +- crates/store/src/server/mod.rs | 6 +++--- crates/store/src/server/ntx_builder.rs | 10 +++++----- proto/proto/internal/sequencer.proto | 2 +- scripts/run-node.sh | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index 9e3c06fa01..ec4b4135e1 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -62,7 +62,7 @@ pub enum SequencerCommand { #[arg(long = "block-producer.listen", env = ENV_BLOCK_PRODUCER_LISTEN, value_name = "LISTEN")] block_producer_listen: SocketAddr, - /// Socket address at which to serve the sequencer.NtxBuilder gRPC API for the ntx-builder. + /// Socket address at which to serve the sequencer.NtxBuilderApi gRPC API for the ntx-builder. #[arg(long = "ntx-builder.listen", env = ENV_NTX_BUILDER_LISTEN, value_name = "LISTEN")] ntx_builder_listen: SocketAddr, diff --git a/crates/proto/src/clients/mod.rs b/crates/proto/src/clients/mod.rs index b83a1d7fc6..fcbb006299 100644 --- a/crates/proto/src/clients/mod.rs +++ b/crates/proto/src/clients/mod.rs @@ -122,7 +122,7 @@ type GeneratedProverClient = generated::remote_prover::api_client::ApiClient; type GeneratedNtxBuilderClient = generated::ntx_builder::api_client::ApiClient; type GeneratedSequencerNtxBuilderClient = - generated::sequencer::ntx_builder_client::NtxBuilderClient; + generated::sequencer::ntx_builder_api_client::NtxBuilderApiClient; // gRPC CLIENTS // ================================================================================================ diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 5fb104caa4..a6dd1f5352 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -6,7 +6,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::generated::replica::api_server as replica_api_server; -use miden_node_proto::generated::sequencer::ntx_builder_server as sequencer_ntx_builder_server; +use miden_node_proto::generated::sequencer::ntx_builder_api_server as sequencer_ntx_builder_server; use miden_node_proto::generated::store; use miden_node_proto_build::{ replica_api_descriptor, @@ -604,7 +604,7 @@ pub async fn serve_replica( } } -/// Serves the `sequencer.NtxBuilder` gRPC service for the embedded sequencer. +/// Serves the `sequencer.NtxBuilderApi` gRPC service for the embedded sequencer. /// /// The ntx-builder binary connects here (via `--store.url`) to get chain data and network notes. /// This mirrors `store.NtxBuilder` in functionality but is served on its own listener so the @@ -615,7 +615,7 @@ pub async fn serve_sequencer_ntx_builder( grpc_options: GrpcOptionsInternal, ) -> anyhow::Result<()> { let store_api = api::StoreApi::new(state); - let ntx_builder_service = sequencer_ntx_builder_server::NtxBuilderServer::new(store_api); + let ntx_builder_service = sequencer_ntx_builder_server::NtxBuilderApiServer::new(store_api); let reflection_service = tonic_reflection::server::Builder::configure() .register_file_descriptor_set(sequencer_api_descriptor()) diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 9c88c6e618..12d723cc36 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -7,7 +7,7 @@ use miden_node_proto::domain::account::AccountInfo; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated as proto; use miden_node_proto::generated::rpc::BlockRange; -use miden_node_proto::generated::sequencer::ntx_builder_server as sequencer_ntx_builder_server; +use miden_node_proto::generated::sequencer::ntx_builder_api_server as sequencer_ntx_builder_server; use miden_node_proto::generated::store::ntx_builder_server; use miden_node_utils::ErrorReport; use miden_protocol::account::{StorageMapKey, StorageSlotName}; @@ -326,14 +326,14 @@ impl ntx_builder_server::NtxBuilder for StoreApi { // SEQUENCER NTX BUILDER ENDPOINTS // ================================================================================================ -/// Forwards `sequencer.NtxBuilder` calls to the same `store.NtxBuilder` implementation. +/// Forwards `sequencer.NtxBuilderApi` calls to the same `store.NtxBuilder` implementation. /// /// The two services expose an identical interface; the only difference is which proto package -/// hosts the service declaration. The embedded sequencer serves `sequencer.NtxBuilder` so the -/// ntx-builder binary can point its `--store.url` at the sequencer without depending on the +/// hosts the service declaration. The embedded sequencer serves `sequencer.NtxBuilderApi` so +/// the ntx-builder binary can point its `--store.url` at the sequencer without depending on the /// standalone store binary. #[tonic::async_trait] -impl sequencer_ntx_builder_server::NtxBuilder for StoreApi { +impl sequencer_ntx_builder_server::NtxBuilderApi for StoreApi { async fn get_block_header_by_number( &self, request: Request, diff --git a/proto/proto/internal/sequencer.proto b/proto/proto/internal/sequencer.proto index 7cb544aeaf..9df05b6954 100644 --- a/proto/proto/internal/sequencer.proto +++ b/proto/proto/internal/sequencer.proto @@ -15,7 +15,7 @@ import "internal/store.proto"; // // Mirrors store.NtxBuilder but is served by the embedded sequencer rather than the standalone // store. Point the ntx-builder's --store.url at the sequencer's --ntx-builder.listen address. -service NtxBuilder { +service NtxBuilderApi { // Retrieves block header by given block number. Optionally, it also returns the MMR path // and current chain length to authenticate the block's inclusion. rpc GetBlockHeaderByNumber(rpc.BlockHeaderByNumberRequest) returns (rpc.BlockHeaderByNumberResponse) {} diff --git a/scripts/run-node.sh b/scripts/run-node.sh index 70b52d45d3..b75a946fa8 100755 --- a/scripts/run-node.sh +++ b/scripts/run-node.sh @@ -123,7 +123,7 @@ echo "=== Starting components ===" # The sequencer runs store + block-producer + rpc in a single process and exposes four endpoints: # rpc.listen → client-facing RPC API # block-producer.listen → block-producer gRPC API (used by downstream rpc replicas) -# ntx-builder.listen → sequencer.NtxBuilder gRPC API (used by the standalone ntx-builder binary) +# ntx-builder.listen → sequencer.NtxBuilderApi gRPC API (used by the standalone ntx-builder binary) # replica.listen → replica.Api gRPC API (used by downstream store replicas) echo "Starting sequencer..." OTEL_SERVICE_NAME=miden-sequencer $BINARY sequencer start \ @@ -188,7 +188,7 @@ OTEL_SERVICE_NAME=miden-rpc-replica-2 $BINARY rpc start \ $EXTRA_ARGS & PIDS+=($!) -# The standalone ntx-builder connects to the sequencer's NtxBuilder gRPC endpoint. +# The standalone ntx-builder connects to the sequencer's NtxBuilderApi gRPC endpoint. echo "Starting network transaction builder..." OTEL_SERVICE_NAME=miden-ntx-builder $NTX_BUILDER_BINARY start \ --listen "0.0.0.0:$NTX_BUILDER_PORT" \ From c3bd254f502053d49b0f5e86d152669b5fef4a51 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 14:44:18 +1200 Subject: [PATCH 08/15] Rename store.url to sequencer.url for ntx builder --- bin/ntx-builder/.env | 2 +- bin/ntx-builder/src/clients/store.rs | 6 +++--- bin/ntx-builder/src/commands/mod.rs | 12 ++++++------ bin/ntx-builder/src/lib.rs | 10 +++++----- docker-compose.yml | 2 +- scripts/run-node.sh | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/bin/ntx-builder/.env b/bin/ntx-builder/.env index 72f228ded4..1cc254a075 100644 --- a/bin/ntx-builder/.env +++ b/bin/ntx-builder/.env @@ -4,7 +4,7 @@ MIDEN_NODE_ENABLE_OTEL=true MIDEN_NODE_DATA_DIRECTORY= MIDEN_NODE_NTX_BUILDER_LISTEN= -MIDEN_NODE_NTX_BUILDER_STORE_URL= +MIDEN_NODE_NTX_BUILDER_SEQUENCER_URL= MIDEN_NODE_NTX_BUILDER_BLOCK_PRODUCER_URL= MIDEN_NODE_NTX_BUILDER_VALIDATOR_URL= MIDEN_NODE_NTX_BUILDER_NTX_PROVER_URL= diff --git a/bin/ntx-builder/src/clients/store.rs b/bin/ntx-builder/src/clients/store.rs index 2513f07606..8eb4c8b1fc 100644 --- a/bin/ntx-builder/src/clients/store.rs +++ b/bin/ntx-builder/src/clients/store.rs @@ -48,10 +48,10 @@ pub struct StoreClient { impl StoreClient { /// Creates a new store client with a lazy connection. - pub fn new(store_url: Url) -> Self { - info!(target: COMPONENT, store_endpoint = %store_url, "Initializing store client"); + pub fn new(sequencer_url: Url) -> Self { + info!(target: COMPONENT, store_endpoint = %sequencer_url, "Initializing store client"); - let store = Builder::new(store_url) + let store = Builder::new(sequencer_url) .without_tls() .without_timeout() .without_metadata_version() diff --git a/bin/ntx-builder/src/commands/mod.rs b/bin/ntx-builder/src/commands/mod.rs index a883b81a8b..cfd7032190 100644 --- a/bin/ntx-builder/src/commands/mod.rs +++ b/bin/ntx-builder/src/commands/mod.rs @@ -12,7 +12,7 @@ use url::Url; const ENV_ENABLE_OTEL: &str = "MIDEN_NODE_ENABLE_OTEL"; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; const ENV_LISTEN: &str = "MIDEN_NODE_NTX_BUILDER_LISTEN"; -const ENV_STORE_URL: &str = "MIDEN_NODE_NTX_BUILDER_STORE_URL"; +const ENV_SEQUENCER_URL: &str = "MIDEN_NODE_NTX_BUILDER_SEQUENCER_URL"; const ENV_BLOCK_PRODUCER_URL: &str = "MIDEN_NODE_NTX_BUILDER_BLOCK_PRODUCER_URL"; const ENV_VALIDATOR_URL: &str = "MIDEN_NODE_NTX_BUILDER_VALIDATOR_URL"; const ENV_TX_PROVER_URL: &str = "MIDEN_NODE_NTX_BUILDER_NTX_PROVER_URL"; @@ -33,9 +33,9 @@ pub enum NtxBuilderCommand { #[arg(long = "listen", env = ENV_LISTEN, value_name = "LISTEN")] listen: SocketAddr, - /// The store's ntx-builder service gRPC url. - #[arg(long = "store.url", env = ENV_STORE_URL, value_name = "URL")] - store_url: Url, + /// The sequencer's NtxBuilderApi gRPC url. + #[arg(long = "sequencer.url", env = ENV_SEQUENCER_URL, value_name = "URL")] + sequencer_url: Url, /// The block-producer's gRPC url. #[arg(long = "block-producer.url", env = ENV_BLOCK_PRODUCER_URL, value_name = "URL")] @@ -117,7 +117,7 @@ impl NtxBuilderCommand { pub async fn handle(self) -> anyhow::Result<()> { let Self::Start { listen, - store_url, + sequencer_url, block_producer_url, validator_url, tx_prover_url, @@ -137,7 +137,7 @@ impl NtxBuilderCommand { let database_filepath = data_directory.join("ntx-builder.sqlite3"); let config = miden_ntx_builder::NtxBuilderConfig::new( - store_url, + sequencer_url, block_producer_url, validator_url, database_filepath, diff --git a/bin/ntx-builder/src/lib.rs b/bin/ntx-builder/src/lib.rs index ec6cb147c9..ed2f3efcf1 100644 --- a/bin/ntx-builder/src/lib.rs +++ b/bin/ntx-builder/src/lib.rs @@ -80,8 +80,8 @@ const DEFAULT_MAX_TX_CYCLES: u32 = 1 << 19; /// This struct contains all the settings needed to create and run a `NetworkTransactionBuilder`. #[derive(Debug, Clone)] pub struct NtxBuilderConfig { - /// Address of the store gRPC server (ntx-builder API). - pub store_url: Url, + /// Address of the sequencer's NtxBuilderApi gRPC server. + pub sequencer_url: Url, /// Address of the block producer gRPC server. pub block_producer_url: Url, @@ -139,13 +139,13 @@ pub struct NtxBuilderConfig { impl NtxBuilderConfig { pub fn new( - store_url: Url, + sequencer_url: Url, block_producer_url: Url, validator_url: Url, database_filepath: PathBuf, ) -> Self { Self { - store_url, + sequencer_url, block_producer_url, validator_url, tx_prover_url: None, @@ -280,7 +280,7 @@ impl NtxBuilderConfig { let coordinator = Coordinator::new(self.max_concurrent_txs, self.max_account_crashes, db.clone()); - let store = StoreClient::new(self.store_url.clone()); + let store = StoreClient::new(self.sequencer_url.clone()); let block_producer = BlockProducerClient::new(self.block_producer_url.clone()); let validator = ValidatorClient::new(self.validator_url.clone()); let prover = self.tx_prover_url.clone().map(RemoteTransactionProver::new); diff --git a/docker-compose.yml b/docker-compose.yml index 2fbd170771..226a647179 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,7 +127,7 @@ services: - miden-ntx-builder - start - --listen=0.0.0.0:50301 - - --store.url=http://store:50002 + - --sequencer.url=http://store:50002 - --block-producer.url=http://block-producer:50201 - --validator.url=http://validator:50101 - --data-directory=/data/ntx-builder diff --git a/scripts/run-node.sh b/scripts/run-node.sh index b75a946fa8..f512a7fa2c 100755 --- a/scripts/run-node.sh +++ b/scripts/run-node.sh @@ -192,7 +192,7 @@ PIDS+=($!) echo "Starting network transaction builder..." OTEL_SERVICE_NAME=miden-ntx-builder $NTX_BUILDER_BINARY start \ --listen "0.0.0.0:$NTX_BUILDER_PORT" \ - --store.url "http://127.0.0.1:$SEQUENCER_NTX_BUILDER_PORT" \ + --sequencer.url "http://127.0.0.1:$SEQUENCER_NTX_BUILDER_PORT" \ --block-producer.url "http://127.0.0.1:$SEQUENCER_BLOCK_PRODUCER_PORT" \ --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ --data-directory "$NTX_BUILDER_DIR" \ From 28775dab8184c7f21169412b80dbc5d8a71f82ab Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 14:49:48 +1200 Subject: [PATCH 09/15] Changelog and lint --- CHANGELOG.md | 1 + bin/node/src/commands/sequencer.rs | 3 ++- bin/ntx-builder/src/commands/mod.rs | 2 +- bin/ntx-builder/src/lib.rs | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2bc8f0b3d..e9c434eb43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [BREAKING] Changed `SyncChainMmr` endpoint: the upper end of the block range we're syncing is now the chain tip with the requested finality level. Validator signature is also returned ([#2075](https://github.com/0xMiden/node/pull/2075)). - [BREAKING] Renamed `SubmitProvenTransaction` RPC endpoint to `SubmitProvenTx` ([#2094](https://github.com/0xMiden/node/pull/2094)). - [BREAKING] Renamed `SubmitProvenBatch` RPC endpoint to `SubmitProvenTxBatch` ([#2094](https://github.com/0xMiden/node/pull/2094)). +- Added `miden-node sequencer` command ([#2102](https://github.com/0xMiden/node/pull/2102)). ## v0.14.10 (2026-05-29) diff --git a/bin/node/src/commands/sequencer.rs b/bin/node/src/commands/sequencer.rs index ec4b4135e1..eb248a9544 100644 --- a/bin/node/src/commands/sequencer.rs +++ b/bin/node/src/commands/sequencer.rs @@ -62,7 +62,8 @@ pub enum SequencerCommand { #[arg(long = "block-producer.listen", env = ENV_BLOCK_PRODUCER_LISTEN, value_name = "LISTEN")] block_producer_listen: SocketAddr, - /// Socket address at which to serve the sequencer.NtxBuilderApi gRPC API for the ntx-builder. + /// Socket address at which to serve the sequencer.NtxBuilderApi gRPC API for the + /// ntx-builder. #[arg(long = "ntx-builder.listen", env = ENV_NTX_BUILDER_LISTEN, value_name = "LISTEN")] ntx_builder_listen: SocketAddr, diff --git a/bin/ntx-builder/src/commands/mod.rs b/bin/ntx-builder/src/commands/mod.rs index cfd7032190..079adb3ff5 100644 --- a/bin/ntx-builder/src/commands/mod.rs +++ b/bin/ntx-builder/src/commands/mod.rs @@ -33,7 +33,7 @@ pub enum NtxBuilderCommand { #[arg(long = "listen", env = ENV_LISTEN, value_name = "LISTEN")] listen: SocketAddr, - /// The sequencer's NtxBuilderApi gRPC url. + /// The sequencer's `NtxBuilderApi` gRPC url. #[arg(long = "sequencer.url", env = ENV_SEQUENCER_URL, value_name = "URL")] sequencer_url: Url, diff --git a/bin/ntx-builder/src/lib.rs b/bin/ntx-builder/src/lib.rs index ed2f3efcf1..09b1ef8353 100644 --- a/bin/ntx-builder/src/lib.rs +++ b/bin/ntx-builder/src/lib.rs @@ -80,7 +80,7 @@ const DEFAULT_MAX_TX_CYCLES: u32 = 1 << 19; /// This struct contains all the settings needed to create and run a `NetworkTransactionBuilder`. #[derive(Debug, Clone)] pub struct NtxBuilderConfig { - /// Address of the sequencer's NtxBuilderApi gRPC server. + /// Address of the sequencer's `NtxBuilderApi` gRPC server. pub sequencer_url: Url, /// Address of the block producer gRPC server. From 29c490641a3ca9486db943ecfb802d24616e1eb3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:03:24 +1200 Subject: [PATCH 10/15] Dedupe block producer handle --- crates/block-producer/src/server/mod.rs | 423 ++++++++++-------------- 1 file changed, 174 insertions(+), 249 deletions(-) diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index f36b70211d..e7b83f6072 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -38,36 +38,108 @@ use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BU // BLOCK PRODUCER HANDLE // ================================================================================================ -/// A cloneable in-process handle to the embedded block producer. +/// A cloneable in-process handle to the block producer's core logic. /// -/// Exposes the same submission and status interface as the gRPC `block_producer.Api` client, but -/// routes calls directly to the in-process mempool without any gRPC round-trip. +/// Holds the mempool, store, and cached stats. Both [`BlockProducerRpcServer`] and the embedded +/// sequencer's RPC use this handle — the server as its implementation backing, the sequencer to +/// submit transactions without a gRPC round-trip. +/// +/// The outer [`Mutex`] around the mempool rate-limits concurrent submissions, giving the batch and +/// block builders equal footing with all incoming transactions combined. #[derive(Clone)] pub struct BlockProducerHandle { - mempool: SharedMempool, + mempool: Arc>, store: StoreClient, cached_mempool_stats: Arc>, } impl BlockProducerHandle { + fn new(mempool: SharedMempool, store: StoreClient) -> Self { + Self { + mempool: Arc::new(Mutex::new(mempool)), + store, + cached_mempool_stats: Arc::new(RwLock::new(MempoolStats::default())), + } + } + + /// Starts a background task that periodically updates the cached mempool statistics. + /// + /// This prevents the need to lock the mempool for each status request. + async fn spawn_stats_updater(&self) { + let cached_mempool_stats = Arc::clone(&self.cached_mempool_stats); + let mempool = self.mempool.lock().await.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(CACHED_MEMPOOL_STATS_UPDATE_INTERVAL); + + loop { + interval.tick().await; + + let (chain_tip, unbatched_transactions, proposed_batches, proven_batches) = { + let mempool = mempool.lock().await; + ( + mempool.chain_tip(), + mempool.unbatched_transactions_count() as u64, + mempool.proposed_batches_count() as u64, + mempool.proven_batches_count() as u64, + ) + }; + + let mut cache = cached_mempool_stats.write().await; + *cache = MempoolStats { + chain_tip, + unbatched_transactions, + proposed_batches, + proven_batches, + }; + } + }); + } + + #[instrument( + target = COMPONENT, + name = "block_producer.server.submit_proven_tx", + skip_all, + err + )] pub async fn submit_proven_tx( &self, request: proto::transaction::ProvenTransaction, ) -> Result, Status> { + debug!(target: COMPONENT, ?request); + let tx = ProvenTransaction::read_from_bytes(&request.transaction) .map_err(MempoolSubmissionError::DeserializationFailed)?; + let tx_id = tx.id(); + + debug!( + target: COMPONENT, + tx_id = %tx_id.to_hex(), + account_id = %tx.account_id().to_hex(), + initial_state_commitment = %tx.account_update().initial_state_commitment(), + final_state_commitment = %tx.account_update().final_state_commitment(), + input_notes = %format_input_notes(tx.input_notes()), + output_notes = %format_output_notes(tx.output_notes()), + ref_block_commitment = %tx.ref_block_commitment(), + "Deserialized transaction" + ); + debug!(target: COMPONENT, proof = ?tx.proof()); + let inputs = self .store .get_tx_inputs(&tx) .await .map_err(MempoolSubmissionError::StoreConnectionFailed)?; + // SAFETY: we assume that the rpc component has verified the transaction proof already. let tx = AuthenticatedTransaction::new_unchecked(Arc::new(tx), inputs) .map(Arc::new) .map_err(MempoolSubmissionError::StateConflict)?; self.mempool + .lock() + .await .lock() .await .add_transaction(tx) @@ -76,6 +148,12 @@ impl BlockProducerHandle { .map_err(Into::into) } + #[instrument( + target = COMPONENT, + name = "block_producer.server.submit_proven_tx_batch", + skip_all, + err + )] pub async fn submit_proven_tx_batch( &self, request: proto::transaction::TransactionBatch, @@ -94,6 +172,8 @@ impl BlockProducerHandle { .await .map_err(MempoolSubmissionError::StoreConnectionFailed)?; + // SAFETY: We assume that the rpc component has verified the transaction proofs, as well + // as the batch integrity itself. let tx = AuthenticatedTransaction::new_unchecked(Arc::clone(tx), inputs) .map(Arc::new) .map_err(MempoolSubmissionError::StateConflict)?; @@ -101,6 +181,8 @@ impl BlockProducerHandle { } self.mempool + .lock() + .await .lock() .await .add_user_batch(&txs) @@ -121,6 +203,10 @@ impl BlockProducerHandle { mempool_stats: Some(stats.into()), })) } + + async fn subscribe(&self) -> ReceiverStream { + ReceiverStream::new(self.mempool.lock().await.lock().await.subscribe()) + } } // EMBEDDED BLOCK PRODUCER @@ -186,67 +272,63 @@ impl EmbeddedBlockProducer { }, ); - let cached_mempool_stats = Arc::new(RwLock::new(MempoolStats::default())); - let handle = BlockProducerHandle { - mempool: mempool.clone(), - store: store.clone(), - cached_mempool_stats: Arc::clone(&cached_mempool_stats), - }; + let handle = BlockProducerHandle::new(mempool.clone(), store); let grpc_options = self.grpc_options; let block_producer_address = self.block_producer_address; - let serve = async move { - let listener = TcpListener::bind(block_producer_address) - .await - .context("failed to bind to block producer address")?; - - info!(target: COMPONENT, "Embedded server initialized"); - - let rpc_server = BlockProducerRpcServer { - mempool: Mutex::new(mempool.clone()), - store: store.clone(), - cached_mempool_stats, - }; - - let mut tasks = tokio::task::JoinSet::new(); - - let rpc_id = - tasks.spawn(async move { rpc_server.serve(listener, grpc_options).await }).id(); - - let batch_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { - batch_builder.run(mempool).await; - Ok(()) - } - }) - .id(); - - let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await }).id(); - - let task_ids = HashMap::from([ - (batch_builder_id, "batch-builder"), - (block_builder_id, "block-builder"), - (rpc_id, "rpc"), - ]); - - let task_result = tasks.join_next_with_id().await.unwrap(); - - let task_id = match &task_result { - Ok((id, _)) => *id, - Err(err) => err.id(), - }; - let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - - task_result - .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, result)| match result { - Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), - Err(source) => Err(BlockProducerError::TaskError { task, source }), - }) - .and_then(|x| x)? + let serve = { + let handle = handle.clone(); + async move { + let listener = TcpListener::bind(block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Embedded server initialized"); + + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn(async move { + BlockProducerRpcServer::new(handle).serve(listener, grpc_options).await + }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + + let block_builder_id = + tasks.spawn(async { block_builder.run(mempool).await }).id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? + } }; Ok((handle, serve)) @@ -265,6 +347,9 @@ impl EmbeddedBlockProducer { #[cfg(test)] mod tests; +// BLOCK PRODUCER +// ================================================================================================ + /// The block producer server. /// /// Specifies how to connect to the store, batch prover, and block prover components. @@ -295,9 +380,6 @@ pub struct BlockProducer { pub mempool_tx_capacity: NonZeroUsize, } -// BLOCK PRODUCER -// ================================================================================================ - impl BlockProducer { /// Serves the block-producer RPC API, the batch-builder and the block-builder. /// @@ -350,16 +432,20 @@ impl BlockProducer { self.batch_prover_url, self.batch_interval, ); - let mempool = MempoolConfig { - batch_budget: BatchBudget { - transactions: self.max_txs_per_batch, - ..BatchBudget::default() + let mempool = Mempool::shared( + chain_tip, + MempoolConfig { + batch_budget: BatchBudget { + transactions: self.max_txs_per_batch, + ..BatchBudget::default() + }, + block_budget: BlockBudget { batches: self.max_batches_per_block }, + tx_capacity: self.mempool_tx_capacity, + ..Default::default() }, - block_budget: BlockBudget { batches: self.max_batches_per_block }, - tx_capacity: self.mempool_tx_capacity, - ..Default::default() - }; - let mempool = Mempool::shared(chain_tip, mempool); + ); + + let handle = BlockProducerHandle::new(mempool.clone(), store); // Spawn rpc server and batch and block provers. // @@ -369,15 +455,9 @@ impl BlockProducer { // any complete or fail, we can shutdown the rest (somewhat) gracefully. let mut tasks = tokio::task::JoinSet::new(); - // Launch the gRPC server. let rpc_id = tasks - .spawn({ - let mempool = mempool.clone(); - async move { - BlockProducerRpcServer::new(mempool, store) - .serve(listener, self.grpc_options) - .await - } + .spawn(async move { + BlockProducerRpcServer::new(handle).serve(listener, self.grpc_options).await }) .id(); @@ -430,49 +510,29 @@ impl BlockProducer { // ================================================================================================ /// Serves the block producer's RPC [api](api_server::Api). +/// +/// A thin gRPC adapter over [`BlockProducerHandle`] — all business logic lives on the handle. struct BlockProducerRpcServer { - /// The mutex effectively rate limits incoming transactions into the mempool by forcing them - /// through a queue. - /// - /// This gives mempool users such as the batch and block builders equal footing with __all__ - /// incoming transactions combined. Without this incoming transactions would greatly restrict - /// the block-producers usage of the mempool. - mempool: Mutex, - - store: StoreClient, - - /// Cached mempool statistics that are updated periodically to avoid locking the mempool for - /// each status request. - cached_mempool_stats: Arc>, + handle: BlockProducerHandle, } impl BlockProducerRpcServer { - pub fn new(mempool: SharedMempool, store: StoreClient) -> Self { - Self { - mempool: Mutex::new(mempool), - store, - cached_mempool_stats: Arc::new(RwLock::new(MempoolStats::default())), - } + fn new(handle: BlockProducerHandle) -> Self { + Self { handle } } - // SERVER STARTUP - // -------------------------------------------------------------------------------------------- - async fn serve( self, listener: TcpListener, grpc_options: GrpcOptionsInternal, ) -> anyhow::Result<()> { - // Start background task to periodically update cached mempool stats - self.spawn_mempool_stats_updater().await; + self.handle.spawn_stats_updater().await; let reflection_service = tonic_reflection::server::Builder::configure() .register_file_descriptor_set(block_producer_api_descriptor()) .build_v1() .context("failed to build reflection service")?; - // Build the gRPC server with the API service and trace layer. - tonic::transport::Server::builder() .accept_http1(true) .timeout(grpc_options.request_timeout) @@ -484,125 +544,6 @@ impl BlockProducerRpcServer { .await .context("failed to serve block producer API") } - - /// Starts a background task that periodically updates the cached mempool statistics. - /// - /// This prevents the need to lock the mempool for each status request. - async fn spawn_mempool_stats_updater(&self) { - let cached_mempool_stats = Arc::clone(&self.cached_mempool_stats); - let mempool = self.mempool.lock().await.clone(); - - tokio::spawn(async move { - let mut interval = tokio::time::interval(CACHED_MEMPOOL_STATS_UPDATE_INTERVAL); - - loop { - interval.tick().await; - - let (chain_tip, unbatched_transactions, proposed_batches, proven_batches) = { - let mempool = mempool.lock().await; - ( - mempool.chain_tip(), - mempool.unbatched_transactions_count() as u64, - mempool.proposed_batches_count() as u64, - mempool.proven_batches_count() as u64, - ) - }; - - let mut cache = cached_mempool_stats.write().await; - *cache = MempoolStats { - chain_tip, - unbatched_transactions, - proposed_batches, - proven_batches, - }; - } - }); - } - - // RPC ENDPOINTS - // -------------------------------------------------------------------------------------------- - - #[instrument( - target = COMPONENT, - name = "block_producer.server.submit_proven_tx", - skip_all, - err - )] - async fn submit_proven_tx( - &self, - request: proto::transaction::ProvenTransaction, - ) -> Result { - debug!(target: COMPONENT, ?request); - - let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(MempoolSubmissionError::DeserializationFailed)?; - - let tx_id = tx.id(); - - debug!( - target: COMPONENT, - tx_id = %tx_id.to_hex(), - account_id = %tx.account_id().to_hex(), - initial_state_commitment = %tx.account_update().initial_state_commitment(), - final_state_commitment = %tx.account_update().final_state_commitment(), - input_notes = %format_input_notes(tx.input_notes()), - output_notes = %format_output_notes(tx.output_notes()), - ref_block_commitment = %tx.ref_block_commitment(), - "Deserialized transaction" - ); - debug!(target: COMPONENT, proof = ?tx.proof()); - - let inputs = self - .store - .get_tx_inputs(&tx) - .await - .map_err(MempoolSubmissionError::StoreConnectionFailed)?; - - // SAFETY: we assume that the rpc component has verified the transaction proof already. - let tx = AuthenticatedTransaction::new_unchecked(Arc::new(tx), inputs) - .map(Arc::new) - .map_err(MempoolSubmissionError::StateConflict)?; - - self.mempool.lock().await.lock().await.add_transaction(tx).map(Into::into) - } - - #[instrument( - target = COMPONENT, - name = "block_producer.server.submit_proven_tx_batch", - skip_all, - err - )] - async fn submit_proven_tx_batch( - &self, - request: proto::transaction::TransactionBatch, - ) -> Result { - let proposed = request - .proposed_batch - .expect("proposed batch existence is enforced by RPC component"); - let batch = ProposedBatch::read_from_bytes(&proposed) - .map_err(MempoolSubmissionError::DeserializationFailed)?; - - // We assume that the rpc component has verified everything, including the transaction - // proofs. - - let mut txs = Vec::with_capacity(batch.transactions().len()); - for tx in batch.transactions() { - let inputs = self - .store - .get_tx_inputs(tx) - .await - .map_err(MempoolSubmissionError::StoreConnectionFailed)?; - - // SAFETY: We assume that the rpc component has verified the transaction proofs, as well - // as the batch integrity itself. - let tx = AuthenticatedTransaction::new_unchecked(Arc::clone(tx), inputs) - .map(Arc::new) - .map_err(MempoolSubmissionError::StateConflict)?; - txs.push(tx); - } - - self.mempool.lock().await.lock().await.add_user_batch(&txs).map(Into::into) - } } #[tonic::async_trait] @@ -613,46 +554,30 @@ impl api_server::Api for BlockProducerRpcServer { &self, request: tonic::Request, ) -> Result, Status> { - self.submit_proven_tx(request.into_inner()) - .await - .map(tonic::Response::new) - // This Status::from mapping takes care of hiding internal errors. - .map_err(Into::into) + self.handle.submit_proven_tx(request.into_inner()).await } async fn submit_proven_tx_batch( &self, request: tonic::Request, ) -> Result, Status> { - self.submit_proven_tx_batch(request.into_inner()) - .await - .map(tonic::Response::new) - // This Status::from mapping takes care of hiding internal errors. - .map_err(Into::into) + self.handle.submit_proven_tx_batch(request.into_inner()).await } async fn status( &self, - _request: tonic::Request<()>, + request: tonic::Request<()>, ) -> Result, Status> { - let mempool_stats = *self.cached_mempool_stats.read().await; - - Ok(tonic::Response::new(proto::rpc::BlockProducerStatus { - version: env!("CARGO_PKG_VERSION").to_string(), - status: "connected".to_string(), - chain_tip: mempool_stats.chain_tip.as_u32(), - mempool_stats: Some(mempool_stats.into()), - })) + self.handle.status(request).await } async fn mempool_subscription( &self, _request: tonic::Request<()>, ) -> Result, tonic::Status> { - let subscription = self.mempool.lock().await.lock().await.subscribe(); - let subscription = ReceiverStream::new(subscription); - - Ok(tonic::Response::new(MempoolEventSubscription { inner: subscription })) + Ok(tonic::Response::new(MempoolEventSubscription { + inner: self.handle.subscribe().await, + })) } } From b8e2772271829f33d10eae1f2cc5385058819e1b Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:05:11 +1200 Subject: [PATCH 11/15] Move tests mod line --- crates/block-producer/src/server/mod.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index e7b83f6072..f4bcd74d5b 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -35,6 +35,9 @@ use crate::store::StoreClient; use crate::validator::BlockProducerValidatorClient; use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BUILDERS}; +#[cfg(test)] +mod tests; + // BLOCK PRODUCER HANDLE // ================================================================================================ @@ -304,8 +307,7 @@ impl EmbeddedBlockProducer { }) .id(); - let block_builder_id = - tasks.spawn(async { block_builder.run(mempool).await }).id(); + let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await }).id(); let task_ids = HashMap::from([ (batch_builder_id, "batch-builder"), @@ -344,9 +346,6 @@ impl EmbeddedBlockProducer { } } -#[cfg(test)] -mod tests; - // BLOCK PRODUCER // ================================================================================================ From bbe63d39e26f8a7f14df031e40670e1e7036e6e8 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:18:58 +1200 Subject: [PATCH 12/15] Split bp mods --- crates/block-producer/src/server/embedded.rs | 154 +++++++++ crates/block-producer/src/server/mod.rs | 339 ++----------------- crates/block-producer/src/server/rpc.rs | 177 ++++++++++ crates/block-producer/src/server/tests.rs | 3 +- 4 files changed, 355 insertions(+), 318 deletions(-) create mode 100644 crates/block-producer/src/server/embedded.rs create mode 100644 crates/block-producer/src/server/rpc.rs diff --git a/crates/block-producer/src/server/embedded.rs b/crates/block-producer/src/server/embedded.rs new file mode 100644 index 0000000000..842ed72017 --- /dev/null +++ b/crates/block-producer/src/server/embedded.rs @@ -0,0 +1,154 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use miden_node_utils::clap::GrpcOptionsInternal; +use tokio::net::TcpListener; +use tracing::info; +use url::Url; + +use super::{BlockProducerHandle, BlockProducerRpcServer}; +use crate::batch_builder::BatchBuilder; +use crate::block_builder::BlockBuilder; +use crate::errors::BlockProducerError; +use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig}; +use crate::store::StoreClient; +use crate::validator::BlockProducerValidatorClient; +use crate::{COMPONENT, SERVER_NUM_BATCH_BUILDERS}; + +// EMBEDDED BLOCK PRODUCER +// ================================================================================================ + +/// Block producer variant that uses an in-process store instead of a remote gRPC store. +pub struct EmbeddedBlockProducer { + /// The address of the block producer component. + pub block_producer_address: SocketAddr, + /// The in-process store state. + pub state: Arc, + /// The address of the validator component. + pub validator_url: Url, + /// The address of the batch prover component. + pub batch_prover_url: Option, + /// The interval at which to produce batches. + pub batch_interval: Duration, + /// The interval at which to produce blocks. + pub block_interval: Duration, + /// The maximum number of transactions per batch. + pub max_txs_per_batch: usize, + /// The maximum number of batches per block. + pub max_batches_per_block: usize, + /// Server-side gRPC options. + pub grpc_options: GrpcOptionsInternal, + /// The maximum number of inflight transactions allowed in the mempool at once. + pub mempool_tx_capacity: NonZeroUsize, +} + +impl EmbeddedBlockProducer { + /// Initialises the block producer internals and returns an in-process [`BlockProducerHandle`] + /// together with a future that runs the gRPC server, batch builder, and block builder. + /// + /// Use this when the caller needs to submit transactions directly to the mempool without a + /// gRPC round-trip (e.g. the embedded sequencer's RPC). + pub async fn start( + self, + ) -> anyhow::Result<(BlockProducerHandle, impl Future> + Send)> + { + info!(target: COMPONENT, endpoint=?self.block_producer_address, "Initializing embedded server"); + let store = StoreClient::new_local(self.state.clone()); + let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); + + let chain_tip = self.state.chain_tip(miden_node_store::Finality::Committed).await; + + let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); + let batch_builder = BatchBuilder::new( + store.clone(), + SERVER_NUM_BATCH_BUILDERS, + self.batch_prover_url, + self.batch_interval, + ); + let mempool = Mempool::shared( + chain_tip, + MempoolConfig { + batch_budget: BatchBudget { + transactions: self.max_txs_per_batch, + ..BatchBudget::default() + }, + block_budget: BlockBudget { batches: self.max_batches_per_block }, + tx_capacity: self.mempool_tx_capacity, + ..Default::default() + }, + ); + + let handle = BlockProducerHandle::new(mempool.clone(), store); + + let grpc_options = self.grpc_options; + let block_producer_address = self.block_producer_address; + + let serve = { + let handle = handle.clone(); + async move { + let listener = TcpListener::bind(block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Embedded server initialized"); + + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn(async move { + BlockProducerRpcServer::new(handle).serve(listener, grpc_options).await + }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + + let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await }).id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? + } + }; + + Ok((handle, serve)) + } + + /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// + /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is + /// encountered. + pub async fn serve(self) -> anyhow::Result<()> { + let (_handle, serve) = self.start().await?; + serve.await + } +} diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index f4bcd74d5b..8a25fcd161 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -1,10 +1,6 @@ -use std::collections::HashMap; -use std::net::SocketAddr; -use std::num::NonZeroUsize; use std::sync::Arc; -use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::Context; use futures::StreamExt; use miden_node_proto::domain::mempool::MempoolEvent; use miden_node_proto::generated::block_producer::api_server; @@ -23,21 +19,23 @@ use tokio::sync::{Mutex, RwLock}; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::Status; use tower_http::trace::TraceLayer; -use tracing::{debug, error, info, instrument}; -use url::Url; +use tracing::{debug, instrument}; -use crate::batch_builder::BatchBuilder; -use crate::block_builder::BlockBuilder; use crate::domain::transaction::AuthenticatedTransaction; -use crate::errors::{BlockProducerError, MempoolSubmissionError, StoreError}; -use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig, SharedMempool}; +use crate::errors::MempoolSubmissionError; +use crate::mempool::SharedMempool; use crate::store::StoreClient; -use crate::validator::BlockProducerValidatorClient; -use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BUILDERS}; +use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT}; + +pub mod embedded; +pub mod rpc; #[cfg(test)] mod tests; +pub use embedded::EmbeddedBlockProducer; +pub use rpc::BlockProducer; + // BLOCK PRODUCER HANDLE // ================================================================================================ @@ -51,13 +49,13 @@ mod tests; /// block builders equal footing with all incoming transactions combined. #[derive(Clone)] pub struct BlockProducerHandle { - mempool: Arc>, - store: StoreClient, - cached_mempool_stats: Arc>, + pub(super) mempool: Arc>, + pub(super) store: StoreClient, + pub(super) cached_mempool_stats: Arc>, } impl BlockProducerHandle { - fn new(mempool: SharedMempool, store: StoreClient) -> Self { + pub(super) fn new(mempool: SharedMempool, store: StoreClient) -> Self { Self { mempool: Arc::new(Mutex::new(mempool)), store, @@ -68,7 +66,7 @@ impl BlockProducerHandle { /// Starts a background task that periodically updates the cached mempool statistics. /// /// This prevents the need to lock the mempool for each status request. - async fn spawn_stats_updater(&self) { + pub(super) async fn spawn_stats_updater(&self) { let cached_mempool_stats = Arc::clone(&self.cached_mempool_stats); let mempool = self.mempool.lock().await.clone(); @@ -207,320 +205,27 @@ impl BlockProducerHandle { })) } - async fn subscribe(&self) -> ReceiverStream { + pub(super) async fn subscribe(&self) -> ReceiverStream { ReceiverStream::new(self.mempool.lock().await.lock().await.subscribe()) } } -// EMBEDDED BLOCK PRODUCER -// ================================================================================================ - -/// Block producer variant that uses an in-process store instead of a remote gRPC store. -pub struct EmbeddedBlockProducer { - /// The address of the block producer component. - pub block_producer_address: SocketAddr, - /// The in-process store state. - pub state: Arc, - /// The address of the validator component. - pub validator_url: Url, - /// The address of the batch prover component. - pub batch_prover_url: Option, - /// The interval at which to produce batches. - pub batch_interval: Duration, - /// The interval at which to produce blocks. - pub block_interval: Duration, - /// The maximum number of transactions per batch. - pub max_txs_per_batch: usize, - /// The maximum number of batches per block. - pub max_batches_per_block: usize, - /// Server-side gRPC options. - pub grpc_options: GrpcOptionsInternal, - /// The maximum number of inflight transactions allowed in the mempool at once. - pub mempool_tx_capacity: NonZeroUsize, -} - -impl EmbeddedBlockProducer { - /// Initialises the block producer internals and returns an in-process [`BlockProducerHandle`] - /// together with a future that runs the gRPC server, batch builder, and block builder. - /// - /// Use this when the caller needs to submit transactions directly to the mempool without a - /// gRPC round-trip (e.g. the embedded sequencer's RPC). - pub async fn start( - self, - ) -> anyhow::Result<(BlockProducerHandle, impl Future> + Send)> - { - info!(target: COMPONENT, endpoint=?self.block_producer_address, "Initializing embedded server"); - let store = StoreClient::new_local(self.state.clone()); - let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); - - let chain_tip = self.state.chain_tip(miden_node_store::Finality::Committed).await; - - let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); - let batch_builder = BatchBuilder::new( - store.clone(), - SERVER_NUM_BATCH_BUILDERS, - self.batch_prover_url, - self.batch_interval, - ); - let mempool = Mempool::shared( - chain_tip, - MempoolConfig { - batch_budget: BatchBudget { - transactions: self.max_txs_per_batch, - ..BatchBudget::default() - }, - block_budget: BlockBudget { batches: self.max_batches_per_block }, - tx_capacity: self.mempool_tx_capacity, - ..Default::default() - }, - ); - - let handle = BlockProducerHandle::new(mempool.clone(), store); - - let grpc_options = self.grpc_options; - let block_producer_address = self.block_producer_address; - - let serve = { - let handle = handle.clone(); - async move { - let listener = TcpListener::bind(block_producer_address) - .await - .context("failed to bind to block producer address")?; - - info!(target: COMPONENT, "Embedded server initialized"); - - let mut tasks = tokio::task::JoinSet::new(); - - let rpc_id = tasks - .spawn(async move { - BlockProducerRpcServer::new(handle).serve(listener, grpc_options).await - }) - .id(); - - let batch_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { - batch_builder.run(mempool).await; - Ok(()) - } - }) - .id(); - - let block_builder_id = tasks.spawn(async { block_builder.run(mempool).await }).id(); - - let task_ids = HashMap::from([ - (batch_builder_id, "batch-builder"), - (block_builder_id, "block-builder"), - (rpc_id, "rpc"), - ]); - - let task_result = tasks.join_next_with_id().await.unwrap(); - - let task_id = match &task_result { - Ok((id, _)) => *id, - Err(err) => err.id(), - }; - let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - - task_result - .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, result)| match result { - Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), - Err(source) => Err(BlockProducerError::TaskError { task, source }), - }) - .and_then(|x| x)? - } - }; - - Ok((handle, serve)) - } - - /// Serves the block-producer RPC API, the batch-builder and the block-builder. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - let (_handle, serve) = self.start().await?; - serve.await - } -} - -// BLOCK PRODUCER -// ================================================================================================ - -/// The block producer server. -/// -/// Specifies how to connect to the store, batch prover, and block prover components. -/// The connection to the store is established at startup and retried with exponential backoff -/// until the store becomes available. Once the connection is established, the block producer -/// will start serving requests. -pub struct BlockProducer { - /// The address of the block producer component. - pub block_producer_address: SocketAddr, - /// The address of the store component. - pub store_url: Url, - /// The address of the validator component. - pub validator_url: Url, - /// The address of the batch prover component. - pub batch_prover_url: Option, - /// The interval at which to produce batches. - pub batch_interval: Duration, - /// The interval at which to produce blocks. - pub block_interval: Duration, - /// The maximum number of transactions per batch. - pub max_txs_per_batch: usize, - /// The maximum number of batches per block. - pub max_batches_per_block: usize, - /// Server-side gRPC options. - pub grpc_options: GrpcOptionsInternal, - - /// The maximum number of inflight transactions allowed in the mempool at once. - pub mempool_tx_capacity: NonZeroUsize, -} - -impl BlockProducer { - /// Serves the block-producer RPC API, the batch-builder and the block-builder. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); - let store = StoreClient::new(self.store_url.clone()); - let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); - - // Retry fetching the chain tip from the store until it succeeds. - let mut retries_counter = 0; - let chain_tip = loop { - match store.latest_header().await { - Err(StoreError::GrpcClientError(err)) => { - // exponential backoff with base 500ms and max 30s - let backoff = Duration::from_millis(500) - .saturating_mul(1 << retries_counter) - .min(Duration::from_secs(30)); - - error!( - store = %self.store_url, - ?backoff, - %retries_counter, - %err, - "store connection failed while fetching chain tip, retrying" - ); - - retries_counter += 1; - tokio::time::sleep(backoff).await; - }, - Ok(header) => break header.block_num(), - Err(e) => { - error!(target: COMPONENT, %e, "failed to fetch chain tip from store"); - return Err(e.into()); - }, - } - }; - - let listener = TcpListener::bind(self.block_producer_address) - .await - .context("failed to bind to block producer address")?; - - info!(target: COMPONENT, "Server initialized"); - - let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); - let batch_builder = BatchBuilder::new( - store.clone(), - SERVER_NUM_BATCH_BUILDERS, - self.batch_prover_url, - self.batch_interval, - ); - let mempool = Mempool::shared( - chain_tip, - MempoolConfig { - batch_budget: BatchBudget { - transactions: self.max_txs_per_batch, - ..BatchBudget::default() - }, - block_budget: BlockBudget { batches: self.max_batches_per_block }, - tx_capacity: self.mempool_tx_capacity, - ..Default::default() - }, - ); - - let handle = BlockProducerHandle::new(mempool.clone(), store); - - // Spawn rpc server and batch and block provers. - // - // These communicate indirectly via a shared mempool. - // - // These should run forever, so we combine them into a joinset so that if - // any complete or fail, we can shutdown the rest (somewhat) gracefully. - let mut tasks = tokio::task::JoinSet::new(); - - let rpc_id = tasks - .spawn(async move { - BlockProducerRpcServer::new(handle).serve(listener, self.grpc_options).await - }) - .id(); - - let batch_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { - batch_builder.run(mempool).await; - Ok(()) - } - }) - .id(); - let block_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { block_builder.run(mempool).await } - }) - .id(); - - let task_ids = HashMap::from([ - (batch_builder_id, "batch-builder"), - (block_builder_id, "block-builder"), - (rpc_id, "rpc"), - ]); - - // Wait for any task to end. They should run indefinitely, so this is an unexpected result. - // - // SAFETY: The JoinSet is definitely not empty. - let task_result = tasks.join_next_with_id().await.unwrap(); - - let task_id = match &task_result { - Ok((id, _)) => *id, - Err(err) => err.id(), - }; - let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - - // We could abort the other tasks here, but not much point as we're probably crashing the - // node. - task_result - .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, result)| match result { - Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), - Err(source) => Err(BlockProducerError::TaskError { task, source }), - }) - .and_then(|x| x)? - } -} - // BLOCK PRODUCER RPC SERVER // ================================================================================================ /// Serves the block producer's RPC [api](api_server::Api). /// /// A thin gRPC adapter over [`BlockProducerHandle`] — all business logic lives on the handle. -struct BlockProducerRpcServer { +pub(super) struct BlockProducerRpcServer { handle: BlockProducerHandle, } impl BlockProducerRpcServer { - fn new(handle: BlockProducerHandle) -> Self { + pub(super) fn new(handle: BlockProducerHandle) -> Self { Self { handle } } - async fn serve( + pub(super) async fn serve( self, listener: TcpListener, grpc_options: GrpcOptionsInternal, @@ -583,7 +288,7 @@ impl api_server::Api for BlockProducerRpcServer { // MEMPOOL SUBSCRIPTION // ================================================================================================ -struct MempoolEventSubscription { +pub(super) struct MempoolEventSubscription { inner: ReceiverStream, } @@ -605,7 +310,7 @@ impl tokio_stream::Stream for MempoolEventSubscription { /// Mempool statistics that are updated periodically to avoid locking the mempool. #[derive(Clone, Copy, Default)] -struct MempoolStats { +pub(super) struct MempoolStats { /// The mempool's current view of the chain tip height. chain_tip: BlockNumber, /// Number of transactions currently in the mempool waiting to be batched. diff --git a/crates/block-producer/src/server/rpc.rs b/crates/block-producer/src/server/rpc.rs new file mode 100644 index 0000000000..4944d7370f --- /dev/null +++ b/crates/block-producer/src/server/rpc.rs @@ -0,0 +1,177 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use anyhow::Context; +use miden_node_utils::clap::GrpcOptionsInternal; +use tokio::net::TcpListener; +use tracing::{error, info}; +use url::Url; + +use super::{BlockProducerHandle, BlockProducerRpcServer}; +use crate::batch_builder::BatchBuilder; +use crate::block_builder::BlockBuilder; +use crate::errors::{BlockProducerError, StoreError}; +use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig}; +use crate::store::StoreClient; +use crate::validator::BlockProducerValidatorClient; +use crate::{COMPONENT, SERVER_NUM_BATCH_BUILDERS}; + +// BLOCK PRODUCER +// ================================================================================================ + +/// The block producer server. +/// +/// Specifies how to connect to the store, batch prover, and block prover components. +/// The connection to the store is established at startup and retried with exponential backoff +/// until the store becomes available. Once the connection is established, the block producer +/// will start serving requests. +pub struct BlockProducer { + /// The address of the block producer component. + pub block_producer_address: SocketAddr, + /// The address of the store component. + pub store_url: Url, + /// The address of the validator component. + pub validator_url: Url, + /// The address of the batch prover component. + pub batch_prover_url: Option, + /// The interval at which to produce batches. + pub batch_interval: Duration, + /// The interval at which to produce blocks. + pub block_interval: Duration, + /// The maximum number of transactions per batch. + pub max_txs_per_batch: usize, + /// The maximum number of batches per block. + pub max_batches_per_block: usize, + /// Server-side gRPC options. + pub grpc_options: GrpcOptionsInternal, + /// The maximum number of inflight transactions allowed in the mempool at once. + pub mempool_tx_capacity: NonZeroUsize, +} + +impl BlockProducer { + /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// + /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is + /// encountered. + pub async fn serve(self) -> anyhow::Result<()> { + info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); + let store = StoreClient::new(self.store_url.clone()); + let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); + + // Retry fetching the chain tip from the store until it succeeds. + let mut retries_counter = 0; + let chain_tip = loop { + match store.latest_header().await { + Err(StoreError::GrpcClientError(err)) => { + // exponential backoff with base 500ms and max 30s + let backoff = Duration::from_millis(500) + .saturating_mul(1 << retries_counter) + .min(Duration::from_secs(30)); + + error!( + store = %self.store_url, + ?backoff, + %retries_counter, + %err, + "store connection failed while fetching chain tip, retrying" + ); + + retries_counter += 1; + tokio::time::sleep(backoff).await; + }, + Ok(header) => break header.block_num(), + Err(e) => { + error!(target: COMPONENT, %e, "failed to fetch chain tip from store"); + return Err(e.into()); + }, + } + }; + + let listener = TcpListener::bind(self.block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Server initialized"); + + let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); + let batch_builder = BatchBuilder::new( + store.clone(), + SERVER_NUM_BATCH_BUILDERS, + self.batch_prover_url, + self.batch_interval, + ); + let mempool = Mempool::shared( + chain_tip, + MempoolConfig { + batch_budget: BatchBudget { + transactions: self.max_txs_per_batch, + ..BatchBudget::default() + }, + block_budget: BlockBudget { batches: self.max_batches_per_block }, + tx_capacity: self.mempool_tx_capacity, + ..Default::default() + }, + ); + + let handle = BlockProducerHandle::new(mempool.clone(), store); + + // Spawn rpc server and batch and block provers. + // + // These communicate indirectly via a shared mempool. + // + // These should run forever, so we combine them into a joinset so that if + // any complete or fail, we can shutdown the rest (somewhat) gracefully. + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn(async move { + BlockProducerRpcServer::new(handle).serve(listener, self.grpc_options).await + }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + let block_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { block_builder.run(mempool).await } + }) + .id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + // Wait for any task to end. They should run indefinitely, so this is an unexpected result. + // + // SAFETY: The JoinSet is definitely not empty. + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + // We could abort the other tasks here, but not much point as we're probably crashing the + // node. + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? + } +} diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 9360d047bd..5ee8e2a87f 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -14,7 +14,8 @@ use tokio::{runtime, task}; use tonic::transport::{Channel, Endpoint}; use url::Url; -use crate::{BlockProducer, DEFAULT_MAX_BATCHES_PER_BLOCK, DEFAULT_MAX_TXS_PER_BATCH}; +use crate::server::BlockProducer; +use crate::{DEFAULT_MAX_BATCHES_PER_BLOCK, DEFAULT_MAX_TXS_PER_BATCH}; /// Tests that the block producer starts up correctly even when the store is not initially /// available. The block producer should retry with exponential backoff until the store becomes From d28393e9e7f9ed55d483322646a544b33bd9def0 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:34:59 +1200 Subject: [PATCH 13/15] Rename to remote --- bin/node/src/commands/block_producer.rs | 4 +- .../block-producer/src/block_builder/mod.rs | 2 +- crates/block-producer/src/lib.rs | 2 +- crates/block-producer/src/server/mod.rs | 4 +- crates/block-producer/src/server/rpc.rs | 177 ------------------ crates/block-producer/src/server/tests.rs | 4 +- 6 files changed, 8 insertions(+), 185 deletions(-) delete mode 100644 crates/block-producer/src/server/rpc.rs diff --git a/bin/node/src/commands/block_producer.rs b/bin/node/src/commands/block_producer.rs index 64d7168747..2187b6d3c2 100644 --- a/bin/node/src/commands/block_producer.rs +++ b/bin/node/src/commands/block_producer.rs @@ -4,11 +4,11 @@ use std::time::Duration; use anyhow::Context; use miden_node_block_producer::{ - BlockProducer, DEFAULT_BATCH_INTERVAL, DEFAULT_BLOCK_INTERVAL, DEFAULT_MAX_BATCHES_PER_BLOCK, DEFAULT_MAX_TXS_PER_BATCH, + RemoteBlockProducer, }; use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_string}; use url::Url; @@ -84,7 +84,7 @@ impl BlockProducerCommand { ); } - BlockProducer { + RemoteBlockProducer { block_producer_address, store_url, validator_url, diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 08ebd38f5d..b464d9f47b 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -159,7 +159,7 @@ impl BlockBuilder { let unauthenticated_notes_iter = batch_iter.clone().flat_map(|batch| { // Note: .cloned() shouldn't be necessary but not having it produces an odd lifetime - // error in BlockProducer::serve. Not sure if there's a better fix. Error: + // error in RemoteBlockProducer::serve. Not sure if there's a better fix. Error: // implementation of `FnOnce` is not general enough closure with signature // `fn(&InputNoteCommitment) -> miden_protocol::note::NoteId` must implement // `FnOnce<(&InputNoteCommitment,)>` ...but it actually implements diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 3244b0c8a2..9f6144e3e5 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -18,7 +18,7 @@ pub mod errors; mod errors; pub mod server; -pub use server::{BlockProducer, BlockProducerHandle, EmbeddedBlockProducer}; +pub use server::{BlockProducerHandle, EmbeddedBlockProducer, RemoteBlockProducer}; // CONSTANTS // ================================================================================================= diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 8a25fcd161..d6ecc7f792 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -28,13 +28,13 @@ use crate::store::StoreClient; use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT}; pub mod embedded; -pub mod rpc; +pub mod remote; #[cfg(test)] mod tests; pub use embedded::EmbeddedBlockProducer; -pub use rpc::BlockProducer; +pub use remote::RemoteBlockProducer; // BLOCK PRODUCER HANDLE // ================================================================================================ diff --git a/crates/block-producer/src/server/rpc.rs b/crates/block-producer/src/server/rpc.rs deleted file mode 100644 index 4944d7370f..0000000000 --- a/crates/block-producer/src/server/rpc.rs +++ /dev/null @@ -1,177 +0,0 @@ -use std::collections::HashMap; -use std::net::SocketAddr; -use std::num::NonZeroUsize; -use std::time::Duration; - -use anyhow::Context; -use miden_node_utils::clap::GrpcOptionsInternal; -use tokio::net::TcpListener; -use tracing::{error, info}; -use url::Url; - -use super::{BlockProducerHandle, BlockProducerRpcServer}; -use crate::batch_builder::BatchBuilder; -use crate::block_builder::BlockBuilder; -use crate::errors::{BlockProducerError, StoreError}; -use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig}; -use crate::store::StoreClient; -use crate::validator::BlockProducerValidatorClient; -use crate::{COMPONENT, SERVER_NUM_BATCH_BUILDERS}; - -// BLOCK PRODUCER -// ================================================================================================ - -/// The block producer server. -/// -/// Specifies how to connect to the store, batch prover, and block prover components. -/// The connection to the store is established at startup and retried with exponential backoff -/// until the store becomes available. Once the connection is established, the block producer -/// will start serving requests. -pub struct BlockProducer { - /// The address of the block producer component. - pub block_producer_address: SocketAddr, - /// The address of the store component. - pub store_url: Url, - /// The address of the validator component. - pub validator_url: Url, - /// The address of the batch prover component. - pub batch_prover_url: Option, - /// The interval at which to produce batches. - pub batch_interval: Duration, - /// The interval at which to produce blocks. - pub block_interval: Duration, - /// The maximum number of transactions per batch. - pub max_txs_per_batch: usize, - /// The maximum number of batches per block. - pub max_batches_per_block: usize, - /// Server-side gRPC options. - pub grpc_options: GrpcOptionsInternal, - /// The maximum number of inflight transactions allowed in the mempool at once. - pub mempool_tx_capacity: NonZeroUsize, -} - -impl BlockProducer { - /// Serves the block-producer RPC API, the batch-builder and the block-builder. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); - let store = StoreClient::new(self.store_url.clone()); - let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); - - // Retry fetching the chain tip from the store until it succeeds. - let mut retries_counter = 0; - let chain_tip = loop { - match store.latest_header().await { - Err(StoreError::GrpcClientError(err)) => { - // exponential backoff with base 500ms and max 30s - let backoff = Duration::from_millis(500) - .saturating_mul(1 << retries_counter) - .min(Duration::from_secs(30)); - - error!( - store = %self.store_url, - ?backoff, - %retries_counter, - %err, - "store connection failed while fetching chain tip, retrying" - ); - - retries_counter += 1; - tokio::time::sleep(backoff).await; - }, - Ok(header) => break header.block_num(), - Err(e) => { - error!(target: COMPONENT, %e, "failed to fetch chain tip from store"); - return Err(e.into()); - }, - } - }; - - let listener = TcpListener::bind(self.block_producer_address) - .await - .context("failed to bind to block producer address")?; - - info!(target: COMPONENT, "Server initialized"); - - let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); - let batch_builder = BatchBuilder::new( - store.clone(), - SERVER_NUM_BATCH_BUILDERS, - self.batch_prover_url, - self.batch_interval, - ); - let mempool = Mempool::shared( - chain_tip, - MempoolConfig { - batch_budget: BatchBudget { - transactions: self.max_txs_per_batch, - ..BatchBudget::default() - }, - block_budget: BlockBudget { batches: self.max_batches_per_block }, - tx_capacity: self.mempool_tx_capacity, - ..Default::default() - }, - ); - - let handle = BlockProducerHandle::new(mempool.clone(), store); - - // Spawn rpc server and batch and block provers. - // - // These communicate indirectly via a shared mempool. - // - // These should run forever, so we combine them into a joinset so that if - // any complete or fail, we can shutdown the rest (somewhat) gracefully. - let mut tasks = tokio::task::JoinSet::new(); - - let rpc_id = tasks - .spawn(async move { - BlockProducerRpcServer::new(handle).serve(listener, self.grpc_options).await - }) - .id(); - - let batch_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { - batch_builder.run(mempool).await; - Ok(()) - } - }) - .id(); - let block_builder_id = tasks - .spawn({ - let mempool = mempool.clone(); - async { block_builder.run(mempool).await } - }) - .id(); - - let task_ids = HashMap::from([ - (batch_builder_id, "batch-builder"), - (block_builder_id, "block-builder"), - (rpc_id, "rpc"), - ]); - - // Wait for any task to end. They should run indefinitely, so this is an unexpected result. - // - // SAFETY: The JoinSet is definitely not empty. - let task_result = tasks.join_next_with_id().await.unwrap(); - - let task_id = match &task_result { - Ok((id, _)) => *id, - Err(err) => err.id(), - }; - let task = task_ids.get(&task_id).unwrap_or(&"unknown"); - - // We could abort the other tasks here, but not much point as we're probably crashing the - // node. - task_result - .map_err(|source| BlockProducerError::JoinError { task, source }) - .map(|(_, result)| match result { - Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), - Err(source) => Err(BlockProducerError::TaskError { task, source }), - }) - .and_then(|x| x)? - } -} diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 5ee8e2a87f..241ece3a19 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -14,7 +14,7 @@ use tokio::{runtime, task}; use tonic::transport::{Channel, Endpoint}; use url::Url; -use crate::server::BlockProducer; +use crate::server::RemoteBlockProducer; use crate::{DEFAULT_MAX_BATCHES_PER_BLOCK, DEFAULT_MAX_TXS_PER_BATCH}; /// Tests that the block producer starts up correctly even when the store is not initially @@ -66,7 +66,7 @@ async fn block_producer_startup_is_robust_to_network_failures() { let validator_url = Url::parse(&format!("http://{validator_addr}")).expect("Failed to parse validator URL"); task::spawn(async move { - BlockProducer { + RemoteBlockProducer { block_producer_address: block_producer_addr, store_url, validator_url, From 72e1efd90521cea7a70aedf7df528c2abf1266b7 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:35:06 +1200 Subject: [PATCH 14/15] Add missing mod --- crates/block-producer/src/server/remote.rs | 177 +++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 crates/block-producer/src/server/remote.rs diff --git a/crates/block-producer/src/server/remote.rs b/crates/block-producer/src/server/remote.rs new file mode 100644 index 0000000000..9dbe2f7444 --- /dev/null +++ b/crates/block-producer/src/server/remote.rs @@ -0,0 +1,177 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use anyhow::Context; +use miden_node_utils::clap::GrpcOptionsInternal; +use tokio::net::TcpListener; +use tracing::{error, info}; +use url::Url; + +use super::{BlockProducerHandle, BlockProducerRpcServer}; +use crate::batch_builder::BatchBuilder; +use crate::block_builder::BlockBuilder; +use crate::errors::{BlockProducerError, StoreError}; +use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig}; +use crate::store::StoreClient; +use crate::validator::BlockProducerValidatorClient; +use crate::{COMPONENT, SERVER_NUM_BATCH_BUILDERS}; + +// BLOCK PRODUCER +// ================================================================================================ + +/// The block producer server. +/// +/// Specifies how to connect to the store, batch prover, and block prover components. +/// The connection to the store is established at startup and retried with exponential backoff +/// until the store becomes available. Once the connection is established, the block producer +/// will start serving requests. +pub struct RemoteBlockProducer { + /// The address of the block producer component. + pub block_producer_address: SocketAddr, + /// The address of the store component. + pub store_url: Url, + /// The address of the validator component. + pub validator_url: Url, + /// The address of the batch prover component. + pub batch_prover_url: Option, + /// The interval at which to produce batches. + pub batch_interval: Duration, + /// The interval at which to produce blocks. + pub block_interval: Duration, + /// The maximum number of transactions per batch. + pub max_txs_per_batch: usize, + /// The maximum number of batches per block. + pub max_batches_per_block: usize, + /// Server-side gRPC options. + pub grpc_options: GrpcOptionsInternal, + /// The maximum number of inflight transactions allowed in the mempool at once. + pub mempool_tx_capacity: NonZeroUsize, +} + +impl RemoteBlockProducer { + /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// + /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is + /// encountered. + pub async fn serve(self) -> anyhow::Result<()> { + info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); + let store = StoreClient::new(self.store_url.clone()); + let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); + + // Retry fetching the chain tip from the store until it succeeds. + let mut retries_counter = 0; + let chain_tip = loop { + match store.latest_header().await { + Err(StoreError::GrpcClientError(err)) => { + // exponential backoff with base 500ms and max 30s + let backoff = Duration::from_millis(500) + .saturating_mul(1 << retries_counter) + .min(Duration::from_secs(30)); + + error!( + store = %self.store_url, + ?backoff, + %retries_counter, + %err, + "store connection failed while fetching chain tip, retrying" + ); + + retries_counter += 1; + tokio::time::sleep(backoff).await; + }, + Ok(header) => break header.block_num(), + Err(e) => { + error!(target: COMPONENT, %e, "failed to fetch chain tip from store"); + return Err(e.into()); + }, + } + }; + + let listener = TcpListener::bind(self.block_producer_address) + .await + .context("failed to bind to block producer address")?; + + info!(target: COMPONENT, "Server initialized"); + + let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); + let batch_builder = BatchBuilder::new( + store.clone(), + SERVER_NUM_BATCH_BUILDERS, + self.batch_prover_url, + self.batch_interval, + ); + let mempool = Mempool::shared( + chain_tip, + MempoolConfig { + batch_budget: BatchBudget { + transactions: self.max_txs_per_batch, + ..BatchBudget::default() + }, + block_budget: BlockBudget { batches: self.max_batches_per_block }, + tx_capacity: self.mempool_tx_capacity, + ..Default::default() + }, + ); + + let handle = BlockProducerHandle::new(mempool.clone(), store); + + // Spawn rpc server and batch and block provers. + // + // These communicate indirectly via a shared mempool. + // + // These should run forever, so we combine them into a joinset so that if + // any complete or fail, we can shutdown the rest (somewhat) gracefully. + let mut tasks = tokio::task::JoinSet::new(); + + let rpc_id = tasks + .spawn(async move { + BlockProducerRpcServer::new(handle).serve(listener, self.grpc_options).await + }) + .id(); + + let batch_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { + batch_builder.run(mempool).await; + Ok(()) + } + }) + .id(); + let block_builder_id = tasks + .spawn({ + let mempool = mempool.clone(); + async { block_builder.run(mempool).await } + }) + .id(); + + let task_ids = HashMap::from([ + (batch_builder_id, "batch-builder"), + (block_builder_id, "block-builder"), + (rpc_id, "rpc"), + ]); + + // Wait for any task to end. They should run indefinitely, so this is an unexpected result. + // + // SAFETY: The JoinSet is definitely not empty. + let task_result = tasks.join_next_with_id().await.unwrap(); + + let task_id = match &task_result { + Ok((id, _)) => *id, + Err(err) => err.id(), + }; + let task = task_ids.get(&task_id).unwrap_or(&"unknown"); + + // We could abort the other tasks here, but not much point as we're probably crashing the + // node. + task_result + .map_err(|source| BlockProducerError::JoinError { task, source }) + .map(|(_, result)| match result { + Ok(_) => Err(BlockProducerError::UnexpectedTaskCompletion { task }), + Err(source) => Err(BlockProducerError::TaskError { task, source }), + }) + .and_then(|x| x)? + } +} From 370d4f93850bcf11594c9f4ae3be24e7759a9c75 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 21 May 2026 15:38:06 +1200 Subject: [PATCH 15/15] new_remote --- bin/stress-test/src/seeding/mod.rs | 2 +- crates/block-producer/src/server/remote.rs | 2 +- crates/block-producer/src/store/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index bed3d5d34c..a40d97f79c 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -133,7 +133,7 @@ pub async fn seed_store( // start the store let (_, store_url) = start_store(data_directory.clone()).await; - let store_client = StoreClient::new(store_url); + let store_client = StoreClient::new_remote(store_url); // start generating blocks let accounts_filepath = data_directory.join(ACCOUNTS_FILENAME); diff --git a/crates/block-producer/src/server/remote.rs b/crates/block-producer/src/server/remote.rs index 9dbe2f7444..76b8a769ae 100644 --- a/crates/block-producer/src/server/remote.rs +++ b/crates/block-producer/src/server/remote.rs @@ -57,7 +57,7 @@ impl RemoteBlockProducer { /// encountered. pub async fn serve(self) -> anyhow::Result<()> { info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); - let store = StoreClient::new(self.store_url.clone()); + let store = StoreClient::new_remote(self.store_url.clone()); let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); // Retry fetching the chain tip from the store until it succeeds. diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index a8f842c81c..0eee7c7fc4 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -423,7 +423,7 @@ pub enum StoreClient { impl StoreClient { /// Creates a gRPC-backed store client with a lazy connection. - pub fn new(store_url: Url) -> Self { + pub fn new_remote(store_url: Url) -> Self { Self::Remote(RemoteStoreClient::new(store_url).into()) }