From cc10330599216271ff217733fa9a19aed9b45fbc Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Tue, 28 Jan 2025 11:55:01 +0100 Subject: [PATCH 1/3] fix: use time per iteration instead of total round time in stats it doesn't affect existing data since having sub 100ns time in python is quite difficult --- src/pytest_codspeed/instruments/walltime.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index e107ca3..9002e6a 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -69,13 +69,14 @@ class BenchmarkStats: @classmethod def from_list( cls, - times_ns: list[float], + times_per_round_ns: list[float], *, rounds: int, iter_per_round: int, warmup_iters: int, total_time: float, ) -> BenchmarkStats: + times_ns = [t / iter_per_round for t in times_per_round_ns] stdev_ns = stdev(times_ns) if len(times_ns) > 1 else 0 mean_ns = mean(times_ns) if len(times_ns) > 1: @@ -129,20 +130,20 @@ def run_benchmark( out = fn(*args, **kwargs) # Warmup - times_ns: list[float] = [] + times_per_round_ns: list[float] = [] warmup_start = start = perf_counter_ns() while True: start = perf_counter_ns() fn(*args, **kwargs) end = perf_counter_ns() - times_ns.append(end - start) + times_per_round_ns.append(end - start) if end - warmup_start > config.warmup_time_ns: break # Round sizing - warmup_mean_ns = mean(times_ns) - warmup_iters = len(times_ns) - times_ns.clear() + warmup_mean_ns = mean(times_per_round_ns) + warmup_iters = len(times_per_round_ns) + times_per_round_ns.clear() iter_per_round = ( int(ceil(config.min_round_time_ns / warmup_mean_ns)) if warmup_mean_ns <= config.min_round_time_ns @@ -163,7 +164,7 @@ def run_benchmark( for _ in iter_range: fn(*args, **kwargs) end = perf_counter_ns() - times_ns.append(end - start) + times_per_round_ns.append(end - start) if end - run_start > config.max_time_ns: # TODO: log something @@ -172,7 +173,7 @@ def run_benchmark( total_time = (benchmark_end - run_start) / 1e9 stats = BenchmarkStats.from_list( - times_ns, + times_per_round_ns, rounds=rounds, total_time=total_time, iter_per_round=iter_per_round, @@ -240,12 +241,12 @@ def _print_benchmark_table(self) -> None: for bench in self.benchmarks: rsd = bench.stats.stdev_ns / bench.stats.mean_ns - rsd_text = Text(f"{rsd*100:.1f}%") + rsd_text = Text(f"{rsd * 100:.1f}%") if rsd > 0.1: rsd_text.stylize("red bold") table.add_row( escape(bench.name), - f"{bench.stats.min_ns/bench.stats.iter_per_round:,.0f}ns", + f"{bench.stats.min_ns / bench.stats.iter_per_round:,.0f}ns", rsd_text, f"{bench.stats.total_time:,.2f}s", f"{bench.stats.iter_per_round * bench.stats.rounds:,}", From 6e1d63e087b45c30ddce1923e4579c201d0721a9 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Tue, 28 Jan 2025 11:58:12 +0100 Subject: [PATCH 2/3] feat: increase the min round time to a bigger value (+/- 1ms) --- src/pytest_codspeed/instruments/walltime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index 9002e6a..09325a6 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -24,7 +24,7 @@ DEFAULT_WARMUP_TIME_NS = 1_000_000_000 DEFAULT_MAX_TIME_NS = 3_000_000_000 TIMER_RESOLUTION_NS = get_clock_info("perf_counter").resolution * 1e9 -DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 100 +DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 1_000_000 @dataclass From 4333c1f7984a46636b9f1ce4cbd3d2200328eafc Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Tue, 28 Jan 2025 12:05:48 +0100 Subject: [PATCH 3/3] refactor: replace hardcoded outlier factor for improved readability --- src/pytest_codspeed/instruments/walltime.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index 09325a6..dcbe25b 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -26,6 +26,9 @@ TIMER_RESOLUTION_NS = get_clock_info("perf_counter").resolution * 1e9 DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 1_000_000 +IQR_OUTLIER_FACTOR = 1.5 +STDEV_OUTLIER_FACTOR = 3 + @dataclass class BenchmarkConfig: @@ -89,12 +92,16 @@ def from_list( ) iqr_ns = q3_ns - q1_ns iqr_outlier_rounds = sum( - 1 for t in times_ns if t < q1_ns - 1.5 * iqr_ns or t > q3_ns + 1.5 * iqr_ns + 1 + for t in times_ns + if t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns + or t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns ) stdev_outlier_rounds = sum( 1 for t in times_ns - if t < mean_ns - 3 * stdev_ns or t > mean_ns + 3 * stdev_ns + if t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns + or t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns ) return cls(