Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions benchmark/baseline/ci.json
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
{
"_comment": "Seeded from scaling run 20260618-181019 + footprint run 20260618-185748 on Intel i7-10700K 16-core. gateway_sha for footprint was 'unknown' (demo image built before SHA capture; will be real on next footprint build). Scaling gateway_sha: 8569f213e7a1e4b4fd2f699546eda9a11e78dcf1.",
"gateway_sha": "8569f213e7a1e4b4fd2f699546eda9a11e78dcf1",
"_comment": "Baseline from run 20260622-220119. gateway_sha bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8 is the measured gateway commit for every lane in this run.",
"gateway_sha": "bdcc4ed08008fcb7c37e6af9b2074831f4f5abd8",
"host": {
"cpu_model": "Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz",
"nproc": 16,
"mem_total_kb": 32824960
"mem_total_kb": 32824968
},
"footprint": {
"footprint.uss_kib_median": 92960.0,
"footprint.cpu_cores_median": 0.19535913385165726,
"footprint.num_threads_median": 53.0
"footprint.uss_kib_median": 54248.0,
"footprint.cpu_cores_median": 0.24552932830433832,
"footprint.num_threads_median": 31.0
},
"scaling": {
"scaling.exponent": 0.4556986360933056,
"scaling.ci_lo": 0.26119683252603965,
"scaling.ci_hi": 0.6502004396605715
"scaling.exponent": 0.2930011978650536,
"scaling.ci_lo": 0.03479966730245926,
"scaling.ci_hi": 0.551202728427648
}
}
36 changes: 30 additions & 6 deletions benchmark/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -882,13 +882,37 @@ def cmd_update_baseline(args):
run_meta = json.loads(meta_path.read_text())
break

# gateway_sha can differ per lane (synthetic image vs demo image), so collect
# every lane's value and describe the real source instead of a generic guess.
lane_shas = {}
for lane_dir in sorted(run_dir.iterdir()):
meta_path = lane_dir / "run_metadata.json"
if lane_dir.is_dir() and meta_path.exists():
lane_shas[lane_dir.name] = json.loads(
meta_path.read_text()).get("gateway_sha", "unknown")
real_shas = sorted({s for s in lane_shas.values() if s != "unknown"})
gateway_sha = real_shas[0] if real_shas else "unknown"
if lane_shas and all(s != "unknown" for s in lane_shas.values()) and len(real_shas) == 1:
comment = (
f"Baseline from run {run_dir.name}. gateway_sha {gateway_sha} is the "
"measured gateway commit for every lane in this run."
)
elif lane_shas:
per_lane = ", ".join(f"{k}={v}" for k, v in sorted(lane_shas.items()))
comment = (
f"Baseline from run {run_dir.name}. gateway_sha per lane: {per_lane}. "
"Top-level gateway_sha is the first measured commit; 'unknown' means "
"that lane's image predated SHA capture."
)
else:
comment = (
f"Baseline from run {run_dir.name}. gateway_sha unknown "
"(no lane run_metadata.json found)."
)

baseline: dict = {
"_comment": (
"Baseline seeded from run: " + run_dir.name +
". gateway_sha may be 'unknown' for footprint lane if "
"the turtlebot3 demo image was built before SHA capture was added."
),
"gateway_sha": run_meta.get("gateway_sha", "unknown"),
"_comment": comment,
"gateway_sha": gateway_sha,
"host": {
"cpu_model": run_meta.get("cpu_model", "unknown"),
"nproc": run_meta.get("nproc", 0),
Expand Down
Loading