From 12a788ca05146af294db1e75843854bffbe8afea Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 23 Jun 2026 13:21:35 +0000 Subject: [PATCH 1/2] fix: deflake xnet_slo_120_subnets_staging_test by relaxing send rate threshold The colocated 120-subnet XNet SLO test flakes on the "Send rate below 0.3" check. With 120 single-node subnets sharing performance hardware, the per-subnet block production rate varies between runs: on loaded runs 2-3 straggler subnets dip just below the 0.3 threshold (0.28-0.297), while on retries the minimum jumps to 0.43-0.45 with no subnets below threshold. Add a with_send_rate_threshold builder to the shared Config and lower this test's threshold to 0.2 to absorb the hardware variance while still catching systemic XNet regressions (the median send rate is ~0.7). --- .../xnet/slo_test_lib/xnet_slo_test_lib.rs | 6 ++++++ .../xnet/xnet_slo_120_subnets_staging_test.rs | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs b/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs index a8545a5297ae..c1d1009609bd 100644 --- a/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs +++ b/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs @@ -135,6 +135,12 @@ impl Config { config } + pub fn with_send_rate_threshold(self, send_rate_threshold: f64) -> Self { + let mut config = self.clone(); + config.send_rate_threshold = send_rate_threshold; + config + } + pub fn with_payload_bytes(self, payload_size_bytes: u64) -> Self { let mut config = self.clone(); config.request_payload_size_bytes = payload_size_bytes; diff --git a/rs/tests/message_routing/xnet/xnet_slo_120_subnets_staging_test.rs b/rs/tests/message_routing/xnet/xnet_slo_120_subnets_staging_test.rs index 1b82fc1694d5..3b1abcb22533 100644 --- a/rs/tests/message_routing/xnet/xnet_slo_120_subnets_staging_test.rs +++ b/rs/tests/message_routing/xnet/xnet_slo_120_subnets_staging_test.rs @@ -16,7 +16,14 @@ fn main() -> Result<()> { let config = Config::new(SUBNETS, NODES_PER_SUBNET, RUNTIME, REQUEST_RATE) // Only best-effort calls with 30 seconds timeout, so the test doesn't hang for // minutes in case we can't connect to some replica/subnet to pull. - .with_call_timeouts(&[Some(30)]); + .with_call_timeouts(&[Some(30)]) + // With 120 single-node subnets colocated on shared performance hardware, the + // per-subnet block production rate varies between runs. On loaded runs a few + // "straggler" subnets dip just below the default 0.3 send rate threshold (down + // to ~0.28), while on unloaded runs the minimum is ~0.43. Lower the threshold to + // 0.2 to absorb this hardware variance while still catching systemic XNet + // regressions (the median send rate is ~0.7). + .with_send_rate_threshold(0.2); let test = config.clone().test(); SystemTestGroup::new() From 10ef99cca7a6f6202e0554698ce61c4f853ad437 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 23 Jun 2026 13:27:59 +0000 Subject: [PATCH 2/2] Mutate owned self in with_send_rate_threshold instead of cloning Addresses Copilot review: the builder consumes self, so cloning the whole Config is unnecessary. Mutate the owned self directly, consistent with with_resource_overrides. --- .../message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs b/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs index c1d1009609bd..8a1acb4bd544 100644 --- a/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs +++ b/rs/tests/message_routing/xnet/slo_test_lib/xnet_slo_test_lib.rs @@ -135,10 +135,9 @@ impl Config { config } - pub fn with_send_rate_threshold(self, send_rate_threshold: f64) -> Self { - let mut config = self.clone(); - config.send_rate_threshold = send_rate_threshold; - config + pub fn with_send_rate_threshold(mut self, send_rate_threshold: f64) -> Self { + self.send_rate_threshold = send_rate_threshold; + self } pub fn with_payload_bytes(self, payload_size_bytes: u64) -> Self {