From a20fd6a3affc7271f0c5decb4bab03a9e6067ad7 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Tue, 23 Jun 2026 14:35:11 +0200 Subject: [PATCH 1/2] chore(benchmark): refresh CI baseline to latest gateway main Re-measured the footprint and scaling lanes on the same host as the previous baseline (Intel i7-10700K, 16 cores, glibc) against gateway main bdcc4ed, and stored the result as the ci baseline. - footprint USS median: 90.8 -> 53.0 MiB - footprint threads: 53 -> 31 - footprint CPU-cores: 0.195 -> 0.246 - scaling exponent: 0.456 -> 0.293 (95% CI [0.03, 0.55]) - gateway_sha now recorded for all lanes including footprint (the old footprint entry was 'unknown') compare passes against the new baseline (no regressions). --- benchmark/baseline/ci.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark/baseline/ci.json b/benchmark/baseline/ci.json index 331d481..dea0d4b 100644 --- a/benchmark/baseline/ci.json +++ b/benchmark/baseline/ci.json @@ -1,19 +1,19 @@ { - "_comment": "Seeded from scaling run 20260618-181019 + footprint run 20260618-185748 on Intel i7-10700K 16-core. gateway_sha for footprint was 'unknown' (demo image built before SHA capture; will be real on next footprint build). Scaling gateway_sha: 8569f213e7a1e4b4fd2f699546eda9a11e78dcf1.", - "gateway_sha": "8569f213e7a1e4b4fd2f699546eda9a11e78dcf1", + "_comment": "Baseline seeded from run: 20260622-220119. gateway_sha may be 'unknown' for footprint lane if the turtlebot3 demo image was built before SHA capture was added.", + "gateway_sha": "bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8", "host": { "cpu_model": "Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz", "nproc": 16, - "mem_total_kb": 32824960 + "mem_total_kb": 32824968 }, "footprint": { - "footprint.uss_kib_median": 92960.0, - "footprint.cpu_cores_median": 0.19535913385165726, - "footprint.num_threads_median": 53.0 + "footprint.uss_kib_median": 54248.0, + "footprint.cpu_cores_median": 0.24552932830433832, + "footprint.num_threads_median": 31.0 }, "scaling": { - "scaling.exponent": 0.4556986360933056, - "scaling.ci_lo": 0.26119683252603965, - "scaling.ci_hi": 0.6502004396605715 + "scaling.exponent": 0.2930011978650536, + "scaling.ci_lo": 0.03479966730245926, + "scaling.ci_hi": 0.551202728427648 } } From 16bf7272b724a75e25026478c916f58fae9f26e0 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Tue, 23 Jun 2026 15:29:13 +0200 Subject: [PATCH 2/2] chore(benchmark): derive baseline comment from per-lane gateway_sha update-baseline wrote a fixed comment claiming the footprint gateway_sha may be unknown, regardless of the actual run. Collect gateway_sha from every lane's run_metadata and write a comment that states the real source: one measured commit when all lanes agree, or the per-lane values otherwise. Take the top-level gateway_sha from the first non-unknown lane. Regenerate ci.json: the comment now names the measured commit; the metrics are unchanged. --- benchmark/baseline/ci.json | 2 +- benchmark/benchmark.py | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/benchmark/baseline/ci.json b/benchmark/baseline/ci.json index dea0d4b..72b10ef 100644 --- a/benchmark/baseline/ci.json +++ b/benchmark/baseline/ci.json @@ -1,5 +1,5 @@ { - "_comment": "Baseline seeded from run: 20260622-220119. gateway_sha may be 'unknown' for footprint lane if the turtlebot3 demo image was built before SHA capture was added.", + "_comment": "Baseline from run 20260622-220119. gateway_sha bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8 is the measured gateway commit for every lane in this run.", "gateway_sha": "bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8", "host": { "cpu_model": "Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz", diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 8c541f7..5ee139e 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -882,13 +882,37 @@ def cmd_update_baseline(args): run_meta = json.loads(meta_path.read_text()) break + # gateway_sha can differ per lane (synthetic image vs demo image), so collect + # every lane's value and describe the real source instead of a generic guess. + lane_shas = {} + for lane_dir in sorted(run_dir.iterdir()): + meta_path = lane_dir / "run_metadata.json" + if lane_dir.is_dir() and meta_path.exists(): + lane_shas[lane_dir.name] = json.loads( + meta_path.read_text()).get("gateway_sha", "unknown") + real_shas = sorted({s for s in lane_shas.values() if s != "unknown"}) + gateway_sha = real_shas[0] if real_shas else "unknown" + if lane_shas and all(s != "unknown" for s in lane_shas.values()) and len(real_shas) == 1: + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha {gateway_sha} is the " + "measured gateway commit for every lane in this run." + ) + elif lane_shas: + per_lane = ", ".join(f"{k}={v}" for k, v in sorted(lane_shas.items())) + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha per lane: {per_lane}. " + "Top-level gateway_sha is the first measured commit; 'unknown' means " + "that lane's image predated SHA capture." + ) + else: + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha unknown " + "(no lane run_metadata.json found)." + ) + baseline: dict = { - "_comment": ( - "Baseline seeded from run: " + run_dir.name + - ". gateway_sha may be 'unknown' for footprint lane if " - "the turtlebot3 demo image was built before SHA capture was added." - ), - "gateway_sha": run_meta.get("gateway_sha", "unknown"), + "_comment": comment, + "gateway_sha": gateway_sha, "host": { "cpu_model": run_meta.get("cpu_model", "unknown"), "nproc": run_meta.get("nproc", 0),