add metrics and fix checkpoint sync backfill bugs

lean_client/containers/src/state.rs

@@ -544,6 +544,14 @@ impl State {
                     let old_finalized_slot = finalized_slot;
                     latest_finalized = source;
                     finalized_slot = latest_finalized.slot;
+
+                    // Record successful finalization
+                    METRICS.get().map(|metrics| {
+                        metrics
+                            .lean_finalizations_total
+                            .with_label_values(&["success"])
+                            .inc();
+                    });
                     let delta = finalized_slot.0.checked_sub(old_finalized_slot.0);
 
                     if let Some(delta) = delta

lean_client/fork_choice/src/handlers.rs

@@ -129,6 +129,11 @@ pub fn on_gossip_attestation(
         .gossip_signatures
         .insert(sig_key, signed_attestation.signature);
 
+    // Update gossip signatures gauge
+    METRICS.get().map(|metrics| {
+        metrics.lean_gossip_signatures.set(store.gossip_signatures.len() as i64);
+    });
+
     // Store attestation data indexed by hash for aggregation lookup
     store
         .attestation_data_by_root
@@ -274,7 +279,11 @@ pub fn on_aggregated_attestation(
         metrics
             .lean_attestations_valid_total
             .with_label_values(&["aggregation"])
-            .inc()
+            .inc();
+        // Update gauge for new aggregated payloads count
+        metrics
+            .lean_latest_new_aggregated_payloads
+            .set(store.latest_new_aggregated_payloads.len() as i64);
     });
 
     Ok(())
@@ -484,6 +493,13 @@ fn process_block_internal(
         }
     }
 
+    // Update gauge for known aggregated payloads count
+    METRICS.get().map(|metrics| {
+        metrics
+            .lean_latest_known_aggregated_payloads
+            .set(store.latest_known_aggregated_payloads.len() as i64);
+    });
+
     // Process each aggregated attestation's validators for fork choice
     // Signature verification is done in verify_signatures() before on_block()
     // Per Devnet-2, we process attestation data directly (not SignedAttestation)

lean_client/fork_choice/src/store.rs

@@ -5,7 +5,7 @@ use containers::{
     AggregatedSignatureProof, Attestation, AttestationData, Block, BlockHeader, Checkpoint, Config,
     SignatureKey, SignedBlockWithAttestation, Slot, State,
 };
-use metrics::set_gauge_u64;
+use metrics::{set_gauge_u64, METRICS};
 use ssz::{H256, SszHash};
 use xmss::Signature;
 
@@ -295,6 +295,8 @@ pub fn get_latest_justified(states: &HashMap<H256, State>) -> Option<&Checkpoint
 }
 
 pub fn update_head(store: &mut Store) {
+    let old_head = store.head;
+
     // Compute new head using LMD-GHOST from latest justified root
     let new_head = get_fork_choice_head(
         store,
@@ -304,6 +306,48 @@ pub fn update_head(store: &mut Store) {
     );
     store.head = new_head;
 
+    // Detect reorg if head changed and new head's parent is not old head
+    if new_head != old_head && !old_head.is_zero() {
+        if let Some(new_head_block) = store.blocks.get(&new_head) {
+            if new_head_block.parent_root != old_head {
+                let mut depth = 0u64;
+                let mut current = old_head;
+
+
+                while !current.is_zero() && depth < 100 {
+                    if let Some(block) = store.blocks.get(&current) {
+                        // Check if new head descends from this block
+                        let mut check = new_head;
+                        while !check.is_zero() {
+                            if check == current {
+                                // Found common ancestor
+                                break;
+                            }
+                            if let Some(b) = store.blocks.get(&check) {
+                                check = b.parent_root;
+                            } else {
+                                break;
+                            }
+                        }
+                        if check == current {
+                            break;
+                        }
+                        depth += 1;
+                        current = block.parent_root;
+                    } else {
+                        break;
+                    }
+                }
+
+                // Record reorg metrics
+                METRICS.get().map(|metrics| {
+                    metrics.lean_fork_choice_reorgs_total.inc();
+                    metrics.lean_fork_choice_reorg_depth.observe(depth as f64);
+                });
+            }
+        }
+    }
+
     set_gauge_u64(
         |m| &m.lean_head_slot,
         || {

lean_client/metrics/src/metrics.rs

@@ -17,6 +17,15 @@ pub struct Metrics {
     lean_node_start_time_seconds: IntGauge,
 
     // PQ Signature metrics
+    /// Total number of individual attestation signatures
+    pub lean_pq_sig_attestation_signatures_total: IntCounter,
+
+    /// Total number of valid individual attestation signatures
+    pub lean_pq_sig_attestation_signatures_valid_total: IntCounter,
+
+    /// Total number of invalid individual attestation signatures
+    pub lean_pq_sig_attestation_signatures_invalid_total: IntCounter,
+
     /// Time taken to sign an attestation
     pub lean_pq_sig_attestation_signing_time_seconds: Histogram,
 
@@ -30,7 +39,7 @@ pub struct Metrics {
     pub lean_pq_sig_attestations_in_aggregated_signatures_total: IntCounter,
 
     /// Time taken to build an aggregated attestation signature
-    pub lean_pq_sig_attestation_signatures_building_time_seconds: Histogram,
+    pub lean_pq_sig_aggregated_signatures_building_time_seconds: Histogram,
 
     /// Time taken to verify an aggregated attestation signature
     pub lean_pq_sig_aggregated_signatures_verification_time_seconds: Histogram,
@@ -64,10 +73,10 @@ pub struct Metrics {
     pub lean_attestation_validation_time_seconds: Histogram,
 
     /// Total number of fork choice reorgs
-    lean_fork_choice_reorgs_total: IntCounter,
+    pub lean_fork_choice_reorgs_total: IntCounter,
 
     /// Depth of fork choice reorgs (in blocks)
-    lean_fork_choice_reorg_depth: Histogram,
+    pub lean_fork_choice_reorg_depth: Histogram,
 
     // State Transition Metrics
     /// Latest justified slot
@@ -77,7 +86,7 @@ pub struct Metrics {
     pub lean_latest_finalized_slot: IntGauge,
 
     /// Total number of finalization attempts
-    lean_finalizations_total: IntCounterVec,
+    pub lean_finalizations_total: IntCounterVec,
 
     /// Time to process state transition
     pub lean_state_transition_time_seconds: Histogram,
@@ -110,6 +119,27 @@ pub struct Metrics {
 
     /// Total number of peer disconnection events
     lean_peer_disconnection_events_total: IntCounterVec,
+
+    /// Number of gossip signatures in fork-choice store
+    pub lean_gossip_signatures: IntGauge,
+
+    /// Number of new aggregated payload items
+    pub lean_latest_new_aggregated_payloads: IntGauge,
+
+    /// Number of known aggregated payload items
+    pub lean_latest_known_aggregated_payloads: IntGauge,
+
+    /// Time taken to aggregate committee signatures
+    pub lean_committee_signatures_aggregation_time_seconds: Histogram,
+
+    /// Validator's is_aggregator status (1=true, 0=false)
+    pub lean_is_aggregator: IntGauge,
+
+    /// Node's attestation committee subnet
+    pub lean_attestation_committee_subnet: IntGauge,
+
+    /// Number of attestation committees (ATTESTATION_COMMITTEE_COUNT)
+    pub lean_attestation_committee_count: IntGauge,
 }
 
 impl Metrics {
@@ -125,6 +155,18 @@ impl Metrics {
             )?,
 
             // PQ Signature metrics
+            lean_pq_sig_attestation_signatures_total: IntCounter::new(
+                "lean_pq_sig_attestation_signatures_total",
+                "Total number of individual attestation signatures",
+            )?,
+            lean_pq_sig_attestation_signatures_valid_total: IntCounter::new(
+                "lean_pq_sig_attestation_signatures_valid_total",
+                "Total number of valid individual attestation signatures",
+            )?,
+            lean_pq_sig_attestation_signatures_invalid_total: IntCounter::new(
+                "lean_pq_sig_attestation_signatures_invalid_total",
+                "Total number of invalid individual attestation signatures",
+            )?,
             lean_pq_sig_attestation_signing_time_seconds: Histogram::with_opts(histogram_opts!(
                 "lean_pq_sig_attestation_signing_time_seconds",
                 "Time taken to sign an attestation",
@@ -145,23 +187,23 @@ impl Metrics {
                 "lean_pq_sig_attestations_in_aggregated_signatures_total",
                 "Total number of attestations included into aggregated signatures",
             )?,
-            lean_pq_sig_attestation_signatures_building_time_seconds: Histogram::with_opts(
+            lean_pq_sig_aggregated_signatures_building_time_seconds: Histogram::with_opts(
                 histogram_opts!(
-                    "lean_pq_sig_attestation_signatures_building_time_seconds",
-                    "Time taken to verify an aggregated attestation signature",
-                    vec![0.005, 0.01, 0.025, 0.05, 0.1, 1.0]
+                    "lean_pq_sig_aggregated_signatures_building_time_seconds",
+                    "Time taken to build an aggregated attestation signature",
+                    vec![0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0, 4.0]
                 ),
             )?,
             lean_pq_sig_aggregated_signatures_verification_time_seconds: Histogram::with_opts(
                 histogram_opts!(
                     "lean_pq_sig_aggregated_signatures_verification_time_seconds",
                     "Time taken to verify an aggregated attestation signature",
-                    vec![0.005, 0.01, 0.025, 0.05, 0.1, 1.0]
+                    vec![0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0, 4.0]
                 ),
             )?,
             lean_pq_sig_aggregated_signatures_valid_total: IntCounter::new(
                 "lean_pq_sig_aggregated_signatures_valid_total",
-                "On validate aggregated signature",
+                "Total number of valid aggregated signatures",
             )?,
             lean_pq_sig_aggregated_signatures_invalid_total: IntCounter::new(
                 "lean_pq_sig_aggregated_signatures_invalid_total",
@@ -178,7 +220,7 @@ impl Metrics {
             lean_fork_choice_block_processing_time_seconds: Histogram::with_opts(histogram_opts!(
                 "lean_fork_choice_block_processing_time_seconds",
                 "Time taken to process block",
-                vec![0.005, 0.01, 0.025, 0.05, 0.1, 1.0]
+                vec![0.005, 0.01, 0.025, 0.05, 0.1, 1.0, 1.25, 1.5, 2.0, 4.0]
             ))?,
             lean_attestations_valid_total: IntCounterVec::new(
                 opts!(
@@ -285,6 +327,40 @@ impl Metrics {
                 ),
                 &["direction", "reason"],
             )?,
+
+            lean_gossip_signatures: IntGauge::new(
+                "lean_gossip_signatures",
+                "Number of gossip signatures in fork-choice store",
+            )?,
+            lean_latest_new_aggregated_payloads: IntGauge::new(
+                "lean_latest_new_aggregated_payloads",
+                "Number of new aggregated payload items",
+            )?,
+            lean_latest_known_aggregated_payloads: IntGauge::new(
+                "lean_latest_known_aggregated_payloads",
+                "Number of known aggregated payload items",
+            )?,
+            lean_committee_signatures_aggregation_time_seconds: Histogram::with_opts(
+                histogram_opts!(
+                    "lean_committee_signatures_aggregation_time_seconds",
+                    "Time taken to aggregate committee signatures",
+                    vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0]
+                ),
+            )?,
+
+            lean_is_aggregator: IntGauge::new(
+                "lean_is_aggregator",
+                "Validator's is_aggregator status (1=true, 0=false)",
+            )?,
+
+            lean_attestation_committee_subnet: IntGauge::new(
+                "lean_attestation_committee_subnet",
+                "Node's attestation committee subnet",
+            )?,
+            lean_attestation_committee_count: IntGauge::new(
+                "lean_attestation_committee_count",
+                "Number of attestation committees (ATTESTATION_COMMITTEE_COUNT)",
+            )?,
         })
     }
 
@@ -293,6 +369,15 @@ impl Metrics {
 
         default_registry.register(Box::new(self.lean_node_info.clone()))?;
         default_registry.register(Box::new(self.lean_node_start_time_seconds.clone()))?;
+        default_registry.register(Box::new(
+            self.lean_pq_sig_attestation_signatures_total.clone(),
+        ))?;
+        default_registry.register(Box::new(
+            self.lean_pq_sig_attestation_signatures_valid_total.clone(),
+        ))?;
+        default_registry.register(Box::new(
+            self.lean_pq_sig_attestation_signatures_invalid_total.clone(),
+        ))?;
         default_registry.register(Box::new(
             self.lean_pq_sig_attestation_signing_time_seconds.clone(),
         ))?;
@@ -308,7 +393,7 @@ impl Metrics {
                 .clone(),
         ))?;
         default_registry.register(Box::new(
-            self.lean_pq_sig_attestation_signatures_building_time_seconds
+            self.lean_pq_sig_aggregated_signatures_building_time_seconds
                 .clone(),
         ))?;
         default_registry.register(Box::new(
@@ -362,6 +447,19 @@ impl Metrics {
         default_registry.register(Box::new(self.lean_peer_connection_events_total.clone()))?;
         default_registry.register(Box::new(self.lean_peer_disconnection_events_total.clone()))?;
 
+        // Additional Fork-Choice Metrics
+        default_registry.register(Box::new(self.lean_gossip_signatures.clone()))?;
+        default_registry.register(Box::new(self.lean_latest_new_aggregated_payloads.clone()))?;
+        default_registry.register(Box::new(self.lean_latest_known_aggregated_payloads.clone()))?;
+        default_registry.register(Box::new(
+            self.lean_committee_signatures_aggregation_time_seconds.clone(),
+        ))?;
+
+        default_registry.register(Box::new(self.lean_is_aggregator.clone()))?;
+
+        default_registry.register(Box::new(self.lean_attestation_committee_subnet.clone()))?;
+        default_registry.register(Box::new(self.lean_attestation_committee_count.clone()))?;
+
         Ok(())
     }
 

lean_client/networking/src/gossipsub/tests/config.rs

@@ -1,5 +1,5 @@
 use crate::gossipsub::config::GossipsubConfig;
-use crate::gossipsub::topic::{ATTESTATION_SUBNET_COUNT, GossipsubKind, get_topics};
+use crate::gossipsub::topic::{ATTESTATION_SUBNET_COUNT, GossipsubKind, get_subscription_topics};
 
 #[test]
 fn test_default_parameters() {
@@ -39,7 +39,8 @@ fn test_default_parameters() {
 #[test]
 fn test_set_topics() {
     let mut config = GossipsubConfig::new();
-    let topics = get_topics("genesis".to_string());
+    // Use aggregator mode to get all subnets for this test
+    let topics = get_subscription_topics("genesis".to_string(), Some(0), true);
 
     config.set_topics(topics.clone());