From 2bf0709a1ae60d149f32f660e6c9ecf10c77d9c9 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 18 May 2026 08:11:11 -0600 Subject: [PATCH 1/4] performance benchmarks --- .github/workflows/startup-benchmark.yml | 149 +++++++++++++++++++ benchmark/startup/BUILD.bazel | 10 ++ benchmark/startup/MODULE.bazel | 31 ++++ benchmark/startup/MODULE.bazel.template | 27 ++++ benchmark/startup/compare.py | 181 ++++++++++++++++++++++++ benchmark/startup/generate_module.py | 74 ++++++++++ benchmark/startup/main.py | 5 + benchmark/startup/requirements.txt | 4 + 8 files changed, 481 insertions(+) create mode 100644 .github/workflows/startup-benchmark.yml create mode 100644 benchmark/startup/BUILD.bazel create mode 100644 benchmark/startup/MODULE.bazel create mode 100644 benchmark/startup/MODULE.bazel.template create mode 100644 benchmark/startup/compare.py create mode 100644 benchmark/startup/generate_module.py create mode 100644 benchmark/startup/main.py create mode 100644 benchmark/startup/requirements.txt diff --git a/.github/workflows/startup-benchmark.yml b/.github/workflows/startup-benchmark.yml new file mode 100644 index 000000000..d7dec096f --- /dev/null +++ b/.github/workflows/startup-benchmark.yml @@ -0,0 +1,149 @@ +name: Startup Benchmark + +on: + pull_request: + paths: + - "py/private/**" + - "benchmark/startup/**" + push: + branches: [main] + +env: + BCR_VERSION: "1.11.5" + +jobs: + benchmark: + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: read + + steps: + - name: Checkout PR + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: rules_py_pr + + - name: Checkout HEAD main + uses: actions/checkout@v4 + with: + ref: main + path: rules_py_main + + - name: Install hyperfine + run: | + set -euo pipefail + HYPERFINE_VERSION="1.18.0" + DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb" + URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}" + wget -qO "$DEB" "$URL" + sudo dpkg -i "$DEB" + rm -f "$DEB" + hyperfine --version + + # ── BCR baseline ──────────────────────────────────────────────────────── + - name: Benchmark BCR ${{ env.BCR_VERSION }} + run: | + set -euo pipefail + cd rules_py_pr/benchmark/startup + python3 generate_module.py bcr --version "${BCR_VERSION}" + + OUT_BASE="/tmp/bazel-bcr" + rm -rf "$OUT_BASE" + + START=$(date +%s%N) + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + END=$(date +%s%N) + BUILD_MS=$(( (END - START) / 1000000 )) + echo "{\"build_ms\": $BUILD_MS}" > "../../../bcr-build.json" + + BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } + hyperfine --warmup 5 --runs 50 --export-json ../../../bcr.json "$BIN" + + # ── HEAD main ─────────────────────────────────────────────────────────── + - name: Benchmark HEAD main + run: | + set -euo pipefail + cd rules_py_pr/benchmark/startup + python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main" + + OUT_BASE="/tmp/bazel-main" + rm -rf "$OUT_BASE" + + START=$(date +%s%N) + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + END=$(date +%s%N) + BUILD_MS=$(( (END - START) / 1000000 )) + echo "{\"build_ms\": $BUILD_MS}" > "../../../main-build.json" + + BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } + hyperfine --warmup 5 --runs 50 --export-json ../../../main.json "$BIN" + + # ── Current commit (PR) ───────────────────────────────────────────────── + - name: Benchmark current PR + run: | + set -euo pipefail + cd rules_py_pr/benchmark/startup + python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr" + + OUT_BASE="/tmp/bazel-pr" + rm -rf "$OUT_BASE" + + START=$(date +%s%N) + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + END=$(date +%s%N) + BUILD_MS=$(( (END - START) / 1000000 )) + echo "{\"build_ms\": $BUILD_MS}" > "../../../pr-build.json" + + BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } + hyperfine --warmup 5 --runs 50 --export-json ../../../pr.json "$BIN" + + # ── Compare & gate ────────────────────────────────────────────────────── + - name: Compare results + id: compare + run: | + set -euo pipefail + for f in bcr.json main.json pr.json; do + if [[ ! -f "$f" ]]; then + echo "ERROR: missing result file: $f" + exit 1 + fi + done + python3 rules_py_pr/benchmark/startup/compare.py bcr.json main.json pr.json + + - name: Post PR comment + if: github.event_name == 'pull_request' && always() + uses: actions/github-script@v7 + with: + script: | + const header = ''; + const body = `${header}\n${process.env.TABLE}`; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body && c.body.includes(header)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + console.log(`Updated comment ${existing.id}`); + } else { + const { data: created } = await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body, + }); + console.log(`Created comment ${created.id}`); + } + env: + TABLE: ${{ steps.compare.outputs.table }} diff --git a/benchmark/startup/BUILD.bazel b/benchmark/startup/BUILD.bazel new file mode 100644 index 000000000..75a2794ed --- /dev/null +++ b/benchmark/startup/BUILD.bazel @@ -0,0 +1,10 @@ +load("@aspect_rules_py//py:defs.bzl", "py_binary") + +py_binary( + name = "bench", + srcs = ["main.py"], + main = "main.py", + deps = [ + "@pypi//cowsay", + ], +) diff --git a/benchmark/startup/MODULE.bazel b/benchmark/startup/MODULE.bazel new file mode 100644 index 000000000..834ec37ab --- /dev/null +++ b/benchmark/startup/MODULE.bazel @@ -0,0 +1,31 @@ +"Benchmark workspace for py_binary startup performance" + +module(name = "startup_benchmark") + +bazel_dep(name = "aspect_rules_py") +local_path_override( + module_name = "aspect_rules_py", + path = "../..", +) + +bazel_dep(name = "bazel_features", version = "1.38.0") +bazel_dep(name = "bazel_skylib", version = "1.4.2") +bazel_dep(name = "bazel_lib", version = "3.0.0") +bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "rules_python", version = "1.0.0") + +# Python interpreters provisioned from python-build-standalone via aspect_rules_py +interpreters = use_extension("@aspect_rules_py//py:extensions.bzl", "python_interpreters") +interpreters.toolchain( + is_default = True, + python_version = "3.11", +) + +# Pip dependencies for the benchmark target +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pypi", + python_version = "3.11", + requirements_lock = "//:requirements.txt", +) +use_repo(pip, "pypi") diff --git a/benchmark/startup/MODULE.bazel.template b/benchmark/startup/MODULE.bazel.template new file mode 100644 index 000000000..4803b1d16 --- /dev/null +++ b/benchmark/startup/MODULE.bazel.template @@ -0,0 +1,27 @@ +"Benchmark workspace for py_binary startup performance" + +module(name = "startup_benchmark") + +{{RULES_PY_DECLARATION}} + +bazel_dep(name = "bazel_features", version = "1.38.0") +bazel_dep(name = "bazel_skylib", version = "1.4.2") +bazel_dep(name = "bazel_lib", version = "3.0.0") +bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "rules_python", version = "1.0.0") + +# Python interpreters provisioned from python-build-standalone via aspect_rules_py +interpreters = use_extension("@aspect_rules_py//py:extensions.bzl", "python_interpreters") +interpreters.toolchain( + is_default = True, + python_version = "3.11", +) + +# Pip dependencies for the benchmark target +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pypi", + python_version = "3.11", + requirements_lock = "//:requirements.txt", +) +use_repo(pip, "pypi") diff --git a/benchmark/startup/compare.py b/benchmark/startup/compare.py new file mode 100644 index 000000000..455a9d54e --- /dev/null +++ b/benchmark/startup/compare.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Parse hyperfine JSON output, build a markdown table, exit 1 on regression. + +The regression gate compares PR against HEAD main (not BCR). +BCR is kept as a historical baseline for context, but gating against it is +misleading because transitive dependency versions drift between releases. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +THRESHOLD_REGRESSION_PCT = 10 # fail CI if PR is >10% slower than HEAD main + + +def write_gh_output(text: str) -> None: + """Write to GITHUB_OUTPUT if available, so sticky PR comment always has content.""" + gh_output = os.environ.get("GITHUB_OUTPUT") + if gh_output: + with open(gh_output, "a") as f: + f.write("table< dict[str, Any]: + """Load a single hyperfine JSON result.""" + p = Path(path) + if not p.exists(): + msg = f"ERROR: result file not found: {path}" + print(msg, file=sys.stderr) + write_gh_output(f"❌ {msg}") + sys.exit(2) + + with p.open() as f: + data = json.load(f) + + if "results" not in data or not data["results"]: + msg = f"ERROR: no results in {path}" + print(msg, file=sys.stderr) + write_gh_output(f"❌ {msg}") + sys.exit(2) + + r = data["results"][0] + for key in ("mean", "stddev", "min", "max", "median"): + if key not in r: + msg = f"ERROR: missing '{key}' in {path}" + print(msg, file=sys.stderr) + write_gh_output(f"❌ {msg}") + sys.exit(2) + + return { + "mean_ms": r["mean"] * 1000, + "stddev_ms": r["stddev"] * 1000, + "min_ms": r["min"] * 1000, + "max_ms": r["max"] * 1000, + "median_ms": r["median"] * 1000, + } + + +def load_build(path: str) -> dict[str, float] | None: + """Load an optional build-time JSON ({build_ms: int}).""" + p = Path(path) + if not p.exists(): + return None + with p.open() as f: + data = json.load(f) + ms = data.get("build_ms", 0) + return {"build_s": ms / 1000.0} + + +def pct(a: float, b: float) -> float: + """Percentage delta from a to b.""" + if a == 0: + return 0.0 + return (b - a) / a * 100 + + +def fmt(val: float) -> str: + """Format milliseconds with sensible precision.""" + return f"{val:.3f}" + + +def fmt_s(val: float) -> str: + """Format seconds with sensible precision.""" + return f"{val:.2f}" + + +def warn(delta: float) -> str: + """Return warning emoji if delta exceeds threshold.""" + return "⚠️" if delta > THRESHOLD_REGRESSION_PCT else "" + + +def main() -> None: + if len(sys.argv) != 4: + msg = f"Usage: {sys.argv[0]} " + print(msg, file=sys.stderr) + write_gh_output(f"❌ {msg}") + sys.exit(2) + + bcr_path, main_path, pr_path = sys.argv[1], sys.argv[2], sys.argv[3] + + bcr = load_runtime(bcr_path) + main = load_runtime(main_path) + pr = load_runtime(pr_path) + + bcr_build = load_build(bcr_path.replace(".json", "-build.json")) + main_build = load_build(main_path.replace(".json", "-build.json")) + pr_build = load_build(pr_path.replace(".json", "-build.json")) + + main_vs_bcr = pct(bcr["mean_ms"], main["mean_ms"]) + pr_vs_bcr = pct(bcr["mean_ms"], pr["mean_ms"]) + pr_vs_main = pct(main["mean_ms"], pr["mean_ms"]) + + has_build = bcr_build is not None or main_build is not None or pr_build is not None + + table = "## py_binary startup benchmark\n\n" + if has_build: + table += "| Version | Mean (ms) | Median (ms) | ± stddev | vs BCR | vs main | Build (s) |\n" + table += "|---------|-----------|-------------|----------|--------|---------|-----------|\n" + else: + table += "| Version | Mean (ms) | Median (ms) | ± stddev | vs BCR | vs main |\n" + table += "|---------|-----------|-------------|----------|--------|---------|\n" + + def row(label: str, d: dict[str, Any], d_build: dict[str, float] | None, vs_bcr: str, vs_main: str) -> str: + line = ( + f"| {label} | {fmt(d['mean_ms'])} | {fmt(d['median_ms'])} | " + f"±{fmt(d['stddev_ms'])} | {vs_bcr} | {vs_main}" + ) + if has_build: + b = fmt_s(d_build["build_s"]) if d_build else "—" + line += f" | {b}" + line += " |\n" + return line + + table += row( + "BCR 1.11.5 (baseline)", bcr, bcr_build, "—", "—" + ) + table += row( + "HEAD main", main, main_build, + f"{main_vs_bcr:+.1f}% {warn(main_vs_bcr)}", "—" + ) + table += row( + "This PR", pr, pr_build, + f"{pr_vs_bcr:+.1f}% {warn(pr_vs_bcr)}", + f"{pr_vs_main:+.1f}% {warn(pr_vs_main)}" + ) + + table += ( + f"\n> Measured with `hyperfine --warmup 5 --runs 50` on " + f"`{os.environ.get('RUNNER_OS', 'local')}`\n" + ) + table += ( + f"> **Gate**: PR vs HEAD main (threshold: {THRESHOLD_REGRESSION_PCT}%). " + f"BCR is shown only as a historical baseline.\n" + ) + if has_build: + table += ( + "> **Build time**: cold `bazel build //:bench` with isolated output base, no disk cache.\n" + ) + + write_gh_output(table) + print(table) + + if pr_vs_main > THRESHOLD_REGRESSION_PCT: + print( + f"\n❌ REGRESSION: PR is {pr_vs_main:.1f}% slower than HEAD main " + f"(threshold: {THRESHOLD_REGRESSION_PCT}%)" + ) + sys.exit(1) + + print(f"\n✅ No regression detected (PR is {pr_vs_main:+.1f}% vs HEAD main)") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/benchmark/startup/generate_module.py b/benchmark/startup/generate_module.py new file mode 100644 index 000000000..84c6d2000 --- /dev/null +++ b/benchmark/startup/generate_module.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Generate MODULE.bazel for the benchmark workspace from a template. + +Replaces fragile sed-based mutation with explicit, validated generation. +""" + +import argparse +import sys +from pathlib import Path + +TEMPLATE = Path(__file__).with_name("MODULE.bazel.template") +OUTPUT = Path(__file__).with_name("MODULE.bazel") + + +def generate(declaration: str) -> str: + """Substitute {{RULES_PY_DECLARATION}} in the template.""" + if not TEMPLATE.exists(): + print(f"ERROR: template not found: {TEMPLATE}", file=sys.stderr) + sys.exit(1) + + content = TEMPLATE.read_text() + if "{{RULES_PY_DECLARATION}}" not in content: + print("ERROR: template missing {{RULES_PY_DECLARATION}} placeholder", file=sys.stderr) + sys.exit(1) + + return content.replace("{{RULES_PY_DECLARATION}}", declaration) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate MODULE.bazel for benchmark") + parser.add_argument( + "mode", + choices=["bcr", "local"], + help="'bcr' pins to a BCR release; 'local' uses local_path_override", + ) + parser.add_argument( + "--version", + default="1.11.5", + help="BCR version to pin when mode=bcr (default: 1.11.5)", + ) + parser.add_argument( + "--path", + default="../..", + help="Local path for local_path_override when mode=local (default: ../..)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print to stdout instead of writing MODULE.bazel", + ) + args = parser.parse_args() + + if args.mode == "bcr": + declaration = f'bazel_dep(name = "aspect_rules_py", version = "{args.version}")' + else: + declaration = ( + f'bazel_dep(name = "aspect_rules_py")\n' + f'local_path_override(\n' + f' module_name = "aspect_rules_py",\n' + f' path = "{args.path}",\n' + f')' + ) + + result = generate(declaration) + + if args.dry_run: + print(result) + else: + OUTPUT.write_text(result) + print(f"Wrote {OUTPUT}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/startup/main.py b/benchmark/startup/main.py new file mode 100644 index 000000000..01a8181a0 --- /dev/null +++ b/benchmark/startup/main.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +import cowsay + +cowsay.cow("bench") diff --git a/benchmark/startup/requirements.txt b/benchmark/startup/requirements.txt new file mode 100644 index 000000000..5179ddf01 --- /dev/null +++ b/benchmark/startup/requirements.txt @@ -0,0 +1,4 @@ +--index-url https://pypi.org/simple + +cowsay==6.1 \ + --hash=sha256:274b1e6fc1b966d53976333eb90ac94cb07a450a700b455af9fbdf882244b30a From 4738f0b791d6d7e4c36381289f89875863773213 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Tue, 26 May 2026 10:36:24 -0600 Subject: [PATCH 2/4] bump actions checkout --- .github/workflows/startup-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/startup-benchmark.yml b/.github/workflows/startup-benchmark.yml index d7dec096f..7fd0dcfac 100644 --- a/.github/workflows/startup-benchmark.yml +++ b/.github/workflows/startup-benchmark.yml @@ -20,13 +20,13 @@ jobs: steps: - name: Checkout PR - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 path: rules_py_pr - name: Checkout HEAD main - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: main path: rules_py_main From 0bcd6e58c14759631a47d38225c2f63159fb0ced Mon Sep 17 00:00:00 2001 From: xangcastle Date: Thu, 28 May 2026 09:46:35 -0600 Subject: [PATCH 3/4] improving the beanchmarks --- .github/workflows/startup-benchmark.yml | 12 +++++++ benchmark/startup/BUILD.bazel | 9 ++++++ benchmark/startup/MODULE.bazel | 2 +- benchmark/startup/compare.py | 42 +++++++++++++++++++++++++ benchmark/startup/syspath_probe.py | 35 +++++++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 benchmark/startup/syspath_probe.py diff --git a/.github/workflows/startup-benchmark.yml b/.github/workflows/startup-benchmark.yml index 7fd0dcfac..406b00166 100644 --- a/.github/workflows/startup-benchmark.yml +++ b/.github/workflows/startup-benchmark.yml @@ -62,6 +62,10 @@ jobs: test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } hyperfine --warmup 5 --runs 50 --export-json ../../../bcr.json "$BIN" + BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } + "$BIN_SP" "$GITHUB_WORKSPACE/bcr-syspath.json" + # ── HEAD main ─────────────────────────────────────────────────────────── - name: Benchmark HEAD main run: | @@ -82,6 +86,10 @@ jobs: test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } hyperfine --warmup 5 --runs 50 --export-json ../../../main.json "$BIN" + BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } + "$BIN_SP" "$GITHUB_WORKSPACE/main-syspath.json" + # ── Current commit (PR) ───────────────────────────────────────────────── - name: Benchmark current PR run: | @@ -102,6 +110,10 @@ jobs: test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } hyperfine --warmup 5 --runs 50 --export-json ../../../pr.json "$BIN" + BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) + test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } + "$BIN_SP" "$GITHUB_WORKSPACE/pr-syspath.json" + # ── Compare & gate ────────────────────────────────────────────────────── - name: Compare results id: compare diff --git a/benchmark/startup/BUILD.bazel b/benchmark/startup/BUILD.bazel index 75a2794ed..fd00a442f 100644 --- a/benchmark/startup/BUILD.bazel +++ b/benchmark/startup/BUILD.bazel @@ -8,3 +8,12 @@ py_binary( "@pypi//cowsay", ], ) + +py_binary( + name = "bench_syspath", + srcs = ["syspath_probe.py"], + main = "syspath_probe.py", + deps = [ + "@pypi//cowsay", + ], +) diff --git a/benchmark/startup/MODULE.bazel b/benchmark/startup/MODULE.bazel index 834ec37ab..b1b978cab 100644 --- a/benchmark/startup/MODULE.bazel +++ b/benchmark/startup/MODULE.bazel @@ -5,7 +5,7 @@ module(name = "startup_benchmark") bazel_dep(name = "aspect_rules_py") local_path_override( module_name = "aspect_rules_py", - path = "../..", + path = "/Users/abel/code/rules_py", ) bazel_dep(name = "bazel_features", version = "1.38.0") diff --git a/benchmark/startup/compare.py b/benchmark/startup/compare.py index 455a9d54e..10da7b57a 100644 --- a/benchmark/startup/compare.py +++ b/benchmark/startup/compare.py @@ -73,6 +73,20 @@ def load_build(path: str) -> dict[str, float] | None: return {"build_s": ms / 1000.0} +def load_syspath(path: str) -> dict[str, int] | None: + """Load an optional sys.path quality JSON from syspath_probe.py.""" + p = Path(path) + if not p.exists(): + return None + with p.open() as f: + data = json.load(f) + return { + "total_entries": data.get("total_entries", 0), + "distinct_sp_roots": data.get("distinct_sp_roots", 0), + "dupe_realpaths": data.get("dupe_realpaths", 0), + } + + def pct(a: float, b: float) -> float: """Percentage delta from a to b.""" if a == 0: @@ -112,11 +126,16 @@ def main() -> None: main_build = load_build(main_path.replace(".json", "-build.json")) pr_build = load_build(pr_path.replace(".json", "-build.json")) + bcr_syspath = load_syspath(bcr_path.replace(".json", "-syspath.json")) + main_syspath = load_syspath(main_path.replace(".json", "-syspath.json")) + pr_syspath = load_syspath(pr_path.replace(".json", "-syspath.json")) + main_vs_bcr = pct(bcr["mean_ms"], main["mean_ms"]) pr_vs_bcr = pct(bcr["mean_ms"], pr["mean_ms"]) pr_vs_main = pct(main["mean_ms"], pr["mean_ms"]) has_build = bcr_build is not None or main_build is not None or pr_build is not None + has_syspath = bcr_syspath is not None or main_syspath is not None or pr_syspath is not None table = "## py_binary startup benchmark\n\n" if has_build: @@ -163,6 +182,29 @@ def row(label: str, d: dict[str, Any], d_build: dict[str, float] | None, vs_bcr: "> **Build time**: cold `bazel build //:bench` with isolated output base, no disk cache.\n" ) + if has_syspath: + table += "\n### sys.path quality\n\n" + table += "| Version | sys.path entries | distinct site-packages roots | duplicate realpaths |\n" + table += "|---------|-----------------|------------------------------|---------------------|\n" + + def syspath_row(label: str, sp: dict[str, int] | None) -> str: + if sp is None: + return f"| {label} | — | — | — |\n" + dupe_flag = " ⚠️" if sp["dupe_realpaths"] > 0 else "" + return ( + f"| {label} | {sp['total_entries']} | {sp['distinct_sp_roots']} " + f"| {sp['dupe_realpaths']}{dupe_flag} |\n" + ) + + table += syspath_row("BCR 1.11.5 (baseline)", bcr_syspath) + table += syspath_row("HEAD main", main_syspath) + table += syspath_row("This PR", pr_syspath) + table += ( + "\n> **sys.path quality** measured by `bench_syspath` inside the assembled venv. " + "Duplicate realpaths indicate symlink redundancy; many distinct site-packages roots " + "suggest an inefficient venv layout.\n" + ) + write_gh_output(table) print(table) diff --git a/benchmark/startup/syspath_probe.py b/benchmark/startup/syspath_probe.py new file mode 100644 index 000000000..9008c28b9 --- /dev/null +++ b/benchmark/startup/syspath_probe.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Output sys.path quality metrics as JSON. + +Adapted from e2e-perf/venv_build_test.py: measures structural venv efficiency +rather than wall-clock timing. A high dupe_realpaths count or many distinct +site-packages roots indicates unnecessary overhead in the assembled venv. +""" + +import json +import os +import sys +from pathlib import Path + + +def main() -> None: + entries = [p for p in sys.path if p] + sp_roots = {p for p in entries if "site-packages" in p} + realpaths = [os.path.realpath(p) for p in entries] + dupe_realpaths = len(realpaths) - len(set(realpaths)) + + metrics = { + "total_entries": len(entries), + "distinct_sp_roots": len(sp_roots), + "dupe_realpaths": dupe_realpaths, + } + + out = sys.argv[1] if len(sys.argv) > 1 else None + if out: + Path(out).write_text(json.dumps(metrics)) + else: + print(json.dumps(metrics)) + + +if __name__ == "__main__": + main() \ No newline at end of file From 1416720be3a47ac29f5ab5f205ea6cc2dd1ca142 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Thu, 28 May 2026 09:54:27 -0600 Subject: [PATCH 4/4] improving the beanchmarks --- .github/workflows/startup-benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/startup-benchmark.yml b/.github/workflows/startup-benchmark.yml index 406b00166..6c5160713 100644 --- a/.github/workflows/startup-benchmark.yml +++ b/.github/workflows/startup-benchmark.yml @@ -53,7 +53,7 @@ jobs: rm -rf "$OUT_BASE" START=$(date +%s%N) - bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath END=$(date +%s%N) BUILD_MS=$(( (END - START) / 1000000 )) echo "{\"build_ms\": $BUILD_MS}" > "../../../bcr-build.json" @@ -77,7 +77,7 @@ jobs: rm -rf "$OUT_BASE" START=$(date +%s%N) - bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath END=$(date +%s%N) BUILD_MS=$(( (END - START) / 1000000 )) echo "{\"build_ms\": $BUILD_MS}" > "../../../main-build.json" @@ -101,7 +101,7 @@ jobs: rm -rf "$OUT_BASE" START=$(date +%s%N) - bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench + bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath END=$(date +%s%N) BUILD_MS=$(( (END - START) / 1000000 )) echo "{\"build_ms\": $BUILD_MS}" > "../../../pr-build.json"