diff --git a/benchmark/baseline/ci.json b/benchmark/baseline/ci.json index 331d481..72b10ef 100644 --- a/benchmark/baseline/ci.json +++ b/benchmark/baseline/ci.json @@ -1,19 +1,19 @@ { - "_comment": "Seeded from scaling run 20260618-181019 + footprint run 20260618-185748 on Intel i7-10700K 16-core. gateway_sha for footprint was 'unknown' (demo image built before SHA capture; will be real on next footprint build). Scaling gateway_sha: 8569f213e7a1e4b4fd2f699546eda9a11e78dcf1.", - "gateway_sha": "8569f213e7a1e4b4fd2f699546eda9a11e78dcf1", + "_comment": "Baseline from run 20260622-220119. gateway_sha bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8 is the measured gateway commit for every lane in this run.", + "gateway_sha": "bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8", "host": { "cpu_model": "Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz", "nproc": 16, - "mem_total_kb": 32824960 + "mem_total_kb": 32824968 }, "footprint": { - "footprint.uss_kib_median": 92960.0, - "footprint.cpu_cores_median": 0.19535913385165726, - "footprint.num_threads_median": 53.0 + "footprint.uss_kib_median": 54248.0, + "footprint.cpu_cores_median": 0.24552932830433832, + "footprint.num_threads_median": 31.0 }, "scaling": { - "scaling.exponent": 0.4556986360933056, - "scaling.ci_lo": 0.26119683252603965, - "scaling.ci_hi": 0.6502004396605715 + "scaling.exponent": 0.2930011978650536, + "scaling.ci_lo": 0.03479966730245926, + "scaling.ci_hi": 0.551202728427648 } } diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 8c541f7..5ee139e 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -882,13 +882,37 @@ def cmd_update_baseline(args): run_meta = json.loads(meta_path.read_text()) break + # gateway_sha can differ per lane (synthetic image vs demo image), so collect + # every lane's value and describe the real source instead of a generic guess. + lane_shas = {} + for lane_dir in sorted(run_dir.iterdir()): + meta_path = lane_dir / "run_metadata.json" + if lane_dir.is_dir() and meta_path.exists(): + lane_shas[lane_dir.name] = json.loads( + meta_path.read_text()).get("gateway_sha", "unknown") + real_shas = sorted({s for s in lane_shas.values() if s != "unknown"}) + gateway_sha = real_shas[0] if real_shas else "unknown" + if lane_shas and all(s != "unknown" for s in lane_shas.values()) and len(real_shas) == 1: + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha {gateway_sha} is the " + "measured gateway commit for every lane in this run." + ) + elif lane_shas: + per_lane = ", ".join(f"{k}={v}" for k, v in sorted(lane_shas.items())) + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha per lane: {per_lane}. " + "Top-level gateway_sha is the first measured commit; 'unknown' means " + "that lane's image predated SHA capture." + ) + else: + comment = ( + f"Baseline from run {run_dir.name}. gateway_sha unknown " + "(no lane run_metadata.json found)." + ) + baseline: dict = { - "_comment": ( - "Baseline seeded from run: " + run_dir.name + - ". gateway_sha may be 'unknown' for footprint lane if " - "the turtlebot3 demo image was built before SHA capture was added." - ), - "gateway_sha": run_meta.get("gateway_sha", "unknown"), + "_comment": comment, + "gateway_sha": gateway_sha, "host": { "cpu_model": run_meta.get("cpu_model", "unknown"), "nproc": run_meta.get("nproc", 0),