Draft: HTTPS outcalls dark launch tracker

rs/https_outcalls/client/src/client.rs

@@ -65,6 +65,7 @@ pub struct CanisterHttpAdapterClientImpl {
     rx: Receiver<(CanisterHttpResponse, CanisterHttpPaymentReceipt)>,
     query_service: TransformExecutionService,
     metrics: Metrics,
+    pricing_factory: PricingFactory,
     log: ReplicaLogger,
 }
 
@@ -79,13 +80,15 @@ impl CanisterHttpAdapterClientImpl {
     ) -> Self {
         let (tx, rx) = channel(inflight_requests);
         let metrics = Metrics::new(&metrics_registry);
+        let pricing_factory = PricingFactory::new(&metrics_registry, log.clone());
         Self {
             rt_handle,
             grpc_channel,
             tx,
             rx,
             query_service,
             metrics,
+            pricing_factory,
             log,
         }
     }
@@ -121,6 +124,7 @@ impl NonBlockingChannel<CanisterHttpRequest> for CanisterHttpAdapterClientImpl {
         let mut http_adapter_client = HttpsOutcallsServiceClient::new(self.grpc_channel.clone());
         let query_handler = self.query_service.clone();
         let metrics = self.metrics.clone();
+        let pricing_factory = self.pricing_factory.clone();
         let log = self.log.clone();
 
         // Spawn an async task that sends the canister http request to the adapter and awaits the response.
@@ -133,7 +137,7 @@ impl NonBlockingChannel<CanisterHttpRequest> for CanisterHttpAdapterClientImpl {
                 socks_proxy_addrs,
             } = canister_http_request;
 
-            let mut budget = PricingFactory::new_tracker(&request_context);
+            let mut budget = pricing_factory.new_tracker(&request_context);
             let request_size = request_context.variable_parts_size();
 
             let CanisterHttpRequestContext {

rs/https_outcalls/pricing/BUILD.bazel

@@ -8,8 +8,14 @@ rust_library(
     crate_name = "ic_https_outcalls_pricing",
     version = "0.1.0",
     deps = [
+        # Keep sorted.
         "//rs/config",
+        "//rs/monitoring/logger",
+        "//rs/monitoring/metrics",
+        "//rs/types/cycles",
         "//rs/types/types",
+        "@crate_index//:prometheus",
+        "@crate_index//:slog",
     ],
 )
 

rs/https_outcalls/pricing/Cargo.toml

@@ -8,4 +8,9 @@ documentation.workspace = true
 
 [dependencies]
 ic-config = { path = "../../config" }
-ic-types = { path = "../../types/types" }
+ic-logger = { path = "../../monitoring/logger" }
+ic-metrics = { path = "../../monitoring/metrics" }
+ic-types = { path = "../../types/types" }
+ic-types-cycles = { path = "../../types/cycles" }
+prometheus = { workspace = true }
+slog = { workspace = true }

rs/https_outcalls/pricing/src/dark_launch.rs

@@ -0,0 +1,269 @@
+use ic_logger::{ReplicaLogger, warn};
+use ic_types::{CanisterId, NumBytes, NumInstructions, canister_http::CanisterHttpPaymentReceipt};
+
+use crate::{AdapterLimits, BudgetTracker, NetworkUsage, PricingError, metrics::PricingMetrics};
+
+/// A [`BudgetTracker`] that runs two child trackers side by side: a `real`
+/// tracker whose results are the only ones returned (and therefore the only
+/// ones that affect observable behaviour), and a `shadow` tracker whose results
+/// are merely compared against the real one.
+///
+/// Whenever the shadow tracker disagrees with the real tracker (e.g. it returns
+/// a pricing error where the real tracker succeeded), the divergence is counted
+/// in a metric and logged together with the canister id, so we can measure what
+/// fraction of requests would not be backwards compatible under the shadow
+/// pricing and which canisters would break.
+pub struct DarkLaunchTracker {
+    real: Box<dyn BudgetTracker>,
+    shadow: Box<dyn BudgetTracker>,
+    canister_id: CanisterId,
+    metrics: PricingMetrics,
+    log: ReplicaLogger,
+    /// Whether an incompatibility has already been recorded for this request.
+    /// Ensures we count and log at most once per request.
+    reported: bool,
+}
+
+impl DarkLaunchTracker {
+    pub fn new(
+        real: Box<dyn BudgetTracker>,
+        shadow: Box<dyn BudgetTracker>,
+        canister_id: CanisterId,
+        metrics: PricingMetrics,
+        log: ReplicaLogger,
+    ) -> Self {
+        Self {
+            real,
+            shadow,
+            canister_id,
+            metrics,
+            log,
+            reported: false,
+        }
+    }
+
+    /// Compares the results of the real and shadow trackers for a given
+    /// accounting `step` and records a divergence if they disagree.
+    fn compare(
+        &mut self,
+        step: &str,
+        real: &Result<(), PricingError>,
+        shadow: &Result<(), PricingError>,
+    ) {
+        if real.is_ok() == shadow.is_ok() {
+            return;
+        }
+        if self.reported {
+            return;
+        }
+        self.reported = true;
+        self.metrics
+            .shadow_incompatible_total
+            .with_label_values(&[step])
+            .inc();
+        warn!(
+            self.log,
+            "Canister http request would not be backwards compatible under shadow pricing: \
+             canister_id {}, step {}, real_result {:?}, shadow_result {:?}",
+            self.canister_id,
+            step,
+            real,
+            shadow,
+        );
+    }
+}
+
+impl BudgetTracker for DarkLaunchTracker {
+    fn get_adapter_limits(&self) -> AdapterLimits {
+        // Only the real tracker drives observable behaviour.
+        self.real.get_adapter_limits()
+    }
+
+    fn subtract_network_usage(&mut self, network_usage: NetworkUsage) -> Result<(), PricingError> {
+        let real = self.real.subtract_network_usage(network_usage);
+        let shadow = self.shadow.subtract_network_usage(network_usage);
+        self.compare("network_usage", &real, &shadow);
+        real
+    }
+
+    fn get_transform_limit(&self) -> NumInstructions {
+        self.real.get_transform_limit()
+    }
+
+    fn subtract_transform_usage(&mut self, usage: NumInstructions) -> Result<(), PricingError> {
+        let real = self.real.subtract_transform_usage(usage);
+        let shadow = self.shadow.subtract_transform_usage(usage);
+        self.compare("transform_usage", &real, &shadow);
+        real
+    }
+
+    fn subtract_transformed_response_usage(
+        &mut self,
+        transformed_response_size: NumBytes,
+    ) -> Result<(), PricingError> {
+        let real = self
+            .real
+            .subtract_transformed_response_usage(transformed_response_size);
+        let shadow = self
+            .shadow
+            .subtract_transformed_response_usage(transformed_response_size);
+        self.compare("transformed_response_usage", &real, &shadow);
+        real
+    }
+
+    fn create_payment_receipt(&self) -> CanisterHttpPaymentReceipt {
+        // Count every request that reaches the final accounting step so the
+        // incompatible counter can be expressed as a fraction of the total.
+        self.metrics.shadow_requests_total.inc();
+        self.real.create_payment_receipt()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ic_logger::no_op_logger;
+    use ic_metrics::MetricsRegistry;
+    use std::time::Duration;
+
+    /// A [`BudgetTracker`] whose accounting steps return preconfigured results.
+    struct FakeTracker {
+        network: Result<(), PricingError>,
+        transform: Result<(), PricingError>,
+        transformed: Result<(), PricingError>,
+    }
+
+    impl FakeTracker {
+        fn ok() -> Self {
+            Self {
+                network: Ok(()),
+                transform: Ok(()),
+                transformed: Ok(()),
+            }
+        }
+    }
+
+    impl BudgetTracker for FakeTracker {
+        fn get_adapter_limits(&self) -> AdapterLimits {
+            AdapterLimits {
+                max_response_size: NumBytes::from(0),
+                max_response_time: Duration::ZERO,
+            }
+        }
+        fn subtract_network_usage(&mut self, _: NetworkUsage) -> Result<(), PricingError> {
+            self.network
+        }
+        fn get_transform_limit(&self) -> NumInstructions {
+            NumInstructions::from(0)
+        }
+        fn subtract_transform_usage(&mut self, _: NumInstructions) -> Result<(), PricingError> {
+            self.transform
+        }
+        fn subtract_transformed_response_usage(&mut self, _: NumBytes) -> Result<(), PricingError> {
+            self.transformed
+        }
+        fn create_payment_receipt(&self) -> CanisterHttpPaymentReceipt {
+            CanisterHttpPaymentReceipt::default()
+        }
+    }
+
+    fn dark_launch(
+        real: FakeTracker,
+        shadow: FakeTracker,
+        metrics: PricingMetrics,
+    ) -> DarkLaunchTracker {
+        DarkLaunchTracker::new(
+            Box::new(real),
+            Box::new(shadow),
+            CanisterId::from_u64(7),
+            metrics,
+            no_op_logger(),
+        )
+    }
+
+    fn network_usage() -> NetworkUsage {
+        NetworkUsage {
+            response_size: NumBytes::from(0),
+            response_time: Duration::ZERO,
+        }
+    }
+
+    fn incompatible_count(metrics: &PricingMetrics) -> u64 {
+        [
+            "network_usage",
+            "transform_usage",
+            "transformed_response_usage",
+        ]
+        .iter()
+        .map(|step| {
+            metrics
+                .shadow_incompatible_total
+                .with_label_values(&[*step])
+                .get()
+        })
+        .sum()
+    }
+
+    #[test]
+    fn returns_real_result_and_counts_divergence() {
+        let metrics = PricingMetrics::new(&MetricsRegistry::new());
+        let shadow = FakeTracker {
+            network: Err(PricingError::InsufficientCycles),
+            ..FakeTracker::ok()
+        };
+        let mut tracker = dark_launch(FakeTracker::ok(), shadow, metrics.clone());
+
+        // The real (always-Ok) result is returned even though the shadow fails.
+        assert_eq!(tracker.subtract_network_usage(network_usage()), Ok(()));
+        assert_eq!(
+            metrics
+                .shadow_incompatible_total
+                .with_label_values(&["network_usage"])
+                .get(),
+            1
+        );
+
+        let _ = tracker.create_payment_receipt();
+        assert_eq!(metrics.shadow_requests_total.get(), 1);
+    }
+
+    #[test]
+    fn counts_divergence_at_most_once_per_request() {
+        let metrics = PricingMetrics::new(&MetricsRegistry::new());
+        let shadow = FakeTracker {
+            network: Err(PricingError::InsufficientCycles),
+            transform: Err(PricingError::InsufficientCycles),
+            transformed: Err(PricingError::InsufficientCycles),
+        };
+        let mut tracker = dark_launch(FakeTracker::ok(), shadow, metrics.clone());
+
+        assert_eq!(tracker.subtract_network_usage(network_usage()), Ok(()));
+        assert_eq!(
+            tracker.subtract_transform_usage(NumInstructions::from(0)),
+            Ok(())
+        );
+        assert_eq!(
+            tracker.subtract_transformed_response_usage(NumBytes::from(0)),
+            Ok(())
+        );
+
+        // Only the first divergence is recorded for the request.
+        assert_eq!(incompatible_count(&metrics), 1);
+    }
+
+    #[test]
+    fn no_divergence_when_results_agree() {
+        let metrics = PricingMetrics::new(&MetricsRegistry::new());
+        let mut tracker = dark_launch(FakeTracker::ok(), FakeTracker::ok(), metrics.clone());
+
+        assert_eq!(tracker.subtract_network_usage(network_usage()), Ok(()));
+        assert_eq!(
+            tracker.subtract_transform_usage(NumInstructions::from(0)),
+            Ok(())
+        );
+        let _ = tracker.create_payment_receipt();
+
+        assert_eq!(incompatible_count(&metrics), 0);
+        assert_eq!(metrics.shadow_requests_total.get(), 1);
+    }
+}

rs/https_outcalls/pricing/src/legacy.rs

@@ -45,6 +45,13 @@ impl BudgetTracker for LegacyTracker {
         Ok(())
     }
 
+    fn subtract_transformed_response_usage(
+        &mut self,
+        _transformed_response_size: NumBytes,
+    ) -> Result<(), PricingError> {
+        Ok(())
+    }
+
     fn create_payment_receipt(&self) -> CanisterHttpPaymentReceipt {
         // Legacy pricing does not perform cycles accounting, so no cycles
         // are ever refunded.