chore: Stabilize benchmarks for better signal

Cargo.toml

@@ -30,6 +30,7 @@ license = "MIT OR Apache-2.0"
 rust-version = "1.90.0"
 
 [workspace.dependencies]
+criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false }
 enum-map = { version = "2.7", default-features = false }
 enumset = { version = "1.1", default-features = false }
 hex = { version = "0.4", default-features = false }

neqo-bin/Cargo.toml

@@ -54,10 +54,9 @@ thiserror = { workspace = true }
 tokio = { version = "1", default-features = false, features = ["net", "time", "macros", "rt"] }
 
 [dev-dependencies]
-criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false, features = [
-        "async_tokio",
-] }
+criterion = { workspace = true, features = ["async_tokio"] }
 neqo-bin = { path = ".", features = ["draft-29"] }
+test-fixture = { path = "../test-fixture" }
 tokio = { version = "1", default-features = false, features = ["sync"] }
 
 [features]
@@ -66,7 +65,7 @@ fast-apple-datapath = ["quinn-udp/fast-apple-datapath"]
 draft-29 = ["neqo-http3/draft-29", "neqo-transport/draft-29"]
 
 [package.metadata.cargo-machete]
-ignored = ["criterion"]
+ignored = ["criterion", "test-fixture"]
 
 [lib]
 # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options

neqo-bin/benches/main.rs

@@ -14,6 +14,7 @@ use std::{env, hint::black_box, net::SocketAddr, path::PathBuf, str::FromStr as
 
 use criterion::{BatchSize, Criterion, Throughput, criterion_group, criterion_main};
 use neqo_bin::{client, server};
+use test_fixture::bench::{self, NOISE_THRESHOLD};
 use tokio::runtime::Builder;
 
 struct Benchmark {
@@ -62,6 +63,7 @@ fn transfer(c: &mut Criterion) {
             .as_ref()
             .map_or_else(|| name.to_string(), |suffix| format!("{name}{suffix}"));
         let mut group = c.benchmark_group("transfer");
+        group.noise_threshold(NOISE_THRESHOLD);
         group.throughput(if num_requests == 1 {
             Throughput::Bytes((upload_size + download_size) as u64)
         } else {
@@ -87,7 +89,7 @@ fn transfer(c: &mut Criterion) {
                             server_handle.send(()).unwrap();
                         })
                     },
-                    BatchSize::PerIteration,
+                    BatchSize::LargeInput,
                 );
         });
         group.finish();
@@ -122,5 +124,9 @@ fn spawn_server() -> (tokio::sync::oneshot::Sender<()>, SocketAddr) {
     (done_sender, addr_receiver.recv().unwrap())
 }
 
-criterion_group!(benches, transfer);
+criterion_group! {
+    name = benches;
+    config = bench::config_walltime();
+    targets = transfer
+}
 criterion_main!(benches);

neqo-common/Cargo.toml

@@ -30,7 +30,7 @@ thiserror = { workspace = true }
 windows = { workspace = true, features = ["Win32_Media"] }
 
 [dev-dependencies]
-criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false }
+criterion = { workspace = true }
 test-fixture = { path = "../test-fixture" }
 
 [features]

neqo-common/benches/decoder.rs

@@ -13,6 +13,7 @@ use std::hint::black_box;
 
 use criterion::{Criterion, criterion_group, criterion_main};
 use neqo_common::Decoder;
+use test_fixture::bench;
 
 /// Fill the buffer with sequentially increasing values, wrapping at 255.
 fn fill_buffer(n: usize, mask: u8) -> Vec<u8> {
@@ -47,5 +48,9 @@ fn benchmark_decoder(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, benchmark_decoder);
+criterion_group! {
+    name = benches;
+    config = bench::config_fast();
+    targets = benchmark_decoder
+}
 criterion_main!(benches);

neqo-http3/Cargo.toml

@@ -33,7 +33,7 @@ strum = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false }
+criterion = { workspace = true }
 neqo-http3 = { path = ".", features = ["draft-29"] }
 test-fixture = { path = "../test-fixture" }
 

neqo-http3/benches/common.rs

@@ -6,8 +6,9 @@
 
 use std::time::Duration;
 
-use criterion::{BenchmarkGroup, Criterion};
+use criterion::{BenchmarkGroup, Criterion, measurement::WallTime};
 use test_fixture::{
+    bench::NOISE_THRESHOLD,
     boxed, fixture_init,
     sim::{
         ReadySimulator, Simulator,
@@ -40,11 +41,12 @@ pub fn setup(streams: usize, data_size: usize) -> ReadySimulator {
 /// benchmark to define its own measurement approach.
 pub fn benchmark<M>(c: &mut Criterion, mut measure: M)
 where
-    M: FnMut(&mut BenchmarkGroup<'_, criterion::measurement::WallTime>, usize, usize),
+    M: FnMut(&mut BenchmarkGroup<'_, WallTime>, usize, usize),
 {
     fixture_init();
 
     let mut group = c.benchmark_group("streams");
+    group.noise_threshold(NOISE_THRESHOLD);
     for (streams, data_size) in BENCHMARK_PARAMS {
         measure(&mut group, streams, data_size);
     }

neqo-http3/benches/streams_simulated.rs

@@ -16,6 +16,7 @@
 use std::time::Duration;
 
 use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use test_fixture::bench;
 
 #[path = "common.rs"]
 mod common;
@@ -36,5 +37,9 @@ fn benchmark(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, benchmark);
+criterion_group! {
+    name = benches;
+    config = bench::config_simulation();
+    targets = benchmark
+}
 criterion_main!(benches);

neqo-http3/benches/streams_walltime.rs

@@ -14,6 +14,7 @@
 use std::hint::black_box;
 
 use criterion::{Criterion, criterion_group, criterion_main};
+use test_fixture::bench;
 
 #[path = "common.rs"]
 mod common;
@@ -31,5 +32,9 @@ fn benchmark(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, benchmark);
+criterion_group! {
+    name = benches;
+    config = bench::config_walltime();
+    targets = benchmark
+}
 criterion_main!(benches);

neqo-transport/Cargo.toml

@@ -33,7 +33,7 @@ strum = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false }
+criterion = { workspace = true }
 neqo-transport = { path = ".", features = ["draft-29"] }
 test-fixture = { path = "../test-fixture" }
 

neqo-transport/benches/range_tracker.rs

@@ -13,6 +13,7 @@ use std::hint::black_box;
 
 use criterion::{Criterion, criterion_group, criterion_main};
 use neqo_transport::send_stream::RangeTracker;
+use test_fixture::bench;
 
 const CHUNK: u64 = 1000;
 const END: u64 = 100_000;
@@ -55,5 +56,9 @@ fn benchmark_coalesce(c: &mut Criterion) {
     coalesce(c, 1000);
 }
 
-criterion_group!(benches, benchmark_coalesce);
+criterion_group! {
+    name = benches;
+    config = bench::config_fast();
+    targets = benchmark_coalesce
+}
 criterion_main!(benches);

neqo-transport/benches/rx_stream_orderer.rs

@@ -13,6 +13,7 @@ use std::hint::black_box;
 
 use criterion::{Criterion, criterion_group, criterion_main};
 use neqo_transport::recv_stream::RxStreamOrderer;
+use test_fixture::bench;
 
 fn rx_stream_orderer() {
     let mut rx = RxStreamOrderer::new();
@@ -29,5 +30,9 @@ fn criterion_benchmark(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, criterion_benchmark);
+criterion_group! {
+    name = benches;
+    config = bench::config_fast();
+    targets = criterion_benchmark
+}
 criterion_main!(benches);

neqo-transport/benches/sent_packets.rs

@@ -16,6 +16,7 @@ use neqo_transport::{
     packet,
     recovery::{self, sent},
 };
+use test_fixture::bench;
 
 fn sent_packets() -> sent::Packets {
     let mut pkts = sent::Packets::default();
@@ -51,5 +52,9 @@ fn take_ranges(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, take_ranges);
+criterion_group! {
+    name = benches;
+    config = bench::config_fast();
+    targets = take_ranges
+}
 criterion_main!(benches);

neqo-transport/benches/transfer_common.rs

@@ -9,6 +9,7 @@ use std::time::Duration;
 use criterion::{BenchmarkGroup, Criterion};
 use neqo_transport::{ConnectionParameters, State};
 use test_fixture::{
+    bench::NOISE_THRESHOLD,
     boxed,
     sim::{
         ReadySimulator, Simulator,
@@ -62,27 +63,10 @@ pub fn benchmark<M>(c: &mut Criterion, mut measure: M)
 where
     M: FnMut(&mut BenchmarkGroup<'_, criterion::measurement::WallTime>, &str, Option<&str>, bool),
 {
-    // Handle SIMULATION_SEED environment variable for varying-seeds config
-    let env_seed = std::env::var("SIMULATION_SEED").ok();
-    let configs: [(&str, Option<&str>); 2] = [
-        ("varying-seeds", env_seed.as_deref()),
-        ("same-seed", Some(FIXED_SEED)),
-    ];
-
     let mut group = c.benchmark_group("transfer");
-    group.noise_threshold(0.03);
-    for (label, seed) in configs {
-        for pacing in [false, true] {
-            measure(&mut group, label, seed, pacing);
-        }
+    group.noise_threshold(NOISE_THRESHOLD);
+    for pacing in [false, true] {
+        measure(&mut group, "fixed-seed", Some(FIXED_SEED), pacing);
     }
     group.finish();
 }
-
-/// Returns the criterion configuration for transfer benchmarks.
-#[must_use]
-pub fn criterion_config() -> Criterion {
-    Criterion::default()
-        .warm_up_time(Duration::from_secs(5))
-        .measurement_time(Duration::from_secs(15))
-}

neqo-transport/benches/transfer_simulated.rs

@@ -16,6 +16,7 @@
 use std::time::Duration;
 
 use criterion::{Throughput, criterion_group, criterion_main};
+use test_fixture::bench;
 
 #[path = "transfer_common.rs"]
 mod common;
@@ -38,7 +39,7 @@ fn benchmark(c: &mut criterion::Criterion) {
 
 criterion_group! {
     name = transfer;
-    config = common::criterion_config();
+    config = bench::config_simulation();
     targets = benchmark
 }
 criterion_main!(transfer);

neqo-transport/benches/transfer_walltime.rs

@@ -14,6 +14,7 @@
 use std::hint::black_box;
 
 use criterion::{BatchSize::SmallInput, criterion_group, criterion_main};
+use test_fixture::bench;
 
 #[path = "transfer_common.rs"]
 mod common;
@@ -33,7 +34,7 @@ fn benchmark(c: &mut criterion::Criterion) {
 
 criterion_group! {
     name = transfer;
-    config = common::criterion_config();
+    config = bench::config_walltime();
     targets = benchmark
 }
 criterion_main!(transfer);

test-fixture/Cargo.toml

@@ -16,6 +16,7 @@ readme.workspace = true
 workspace = true
 
 [dependencies]
+criterion = { workspace = true }
 log = { workspace = true }
 neqo-common = { path = "../neqo-common", features = ["test-fixture"] }
 neqo-crypto = { path = "../neqo-crypto" }

test-fixture/src/bench.rs

@@ -0,0 +1,65 @@
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Common configuration for Criterion benchmarks.
+
+use std::time::Duration;
+
+use criterion::Criterion;
+
+/// Set to 1% to detect small but meaningful performance changes.
+pub const NOISE_THRESHOLD: f64 = 0.01;
+
+/// Sample size for fast benchmarks (pure compute).
+const SAMPLE_SIZE_FAST: usize = 1000;
+
+/// Sample size for walltime benchmarks.
+const SAMPLE_SIZE_WALLTIME: usize = 500;
+
+/// Creates a Criterion base configuration.
+fn base_config(
+    sample_size: usize,
+    warm_up_time: Option<Duration>,
+    measurement_time: Option<Duration>,
+) -> Criterion {
+    let mut c = Criterion::default()
+        .sample_size(sample_size)
+        .significance_level(0.01)
+        .confidence_level(0.99);
+    if let Some(dur) = warm_up_time {
+        c = c.warm_up_time(dur);
+    }
+    if let Some(dur) = measurement_time {
+        c = c.measurement_time(dur);
+    }
+    c
+}
+
+/// Configuration for fast, pure-compute benchmarks.
+#[must_use]
+pub fn config_fast() -> Criterion {
+    base_config(SAMPLE_SIZE_FAST, None, None)
+}
+
+/// Configuration for simulation benchmarks.
+#[must_use]
+pub fn config_simulation() -> Criterion {
+    base_config(
+        SAMPLE_SIZE_FAST,
+        Some(Duration::from_secs(5)),
+        Some(Duration::from_secs(30)),
+    )
+}
+
+/// Configuration for walltime benchmarks.
+#[must_use]
+pub fn config_walltime() -> Criterion {
+    base_config(
+        SAMPLE_SIZE_WALLTIME,
+        Some(Duration::from_secs(10)),
+        Some(Duration::from_secs(60)),
+    )
+}