From ae552cdc0f8477b408844ea19ef322198207a3e1 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 6 Jul 2026 14:47:25 +0800 Subject: [PATCH] refactor(board): one source file + one PR per submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public leaderboard was a single site/results.json updated by one aggregate PR, so all models were coupled — you couldn't accept, hold, or revert one model's result without the whole board, and triggering the scorer twice opened duplicate PRs (no dedup). Make each submission its own reviewable unit: - Source of truth is now one file per submission, site/results/.json, where slug = ----, derived deterministically from the scored file (backend_score.board_slug/board_entry/write_board_entries). Test submissions produce no public file. - score-from-r2.yml opens ONE PR per new/changed entry on branch bot/leaderboard/. The slug is stable, so re-scoring the same submission force-updates the SAME branch/PR (idempotent — no duplicates); a different submission is its own independently mergeable/revertible PR. - site/results.json becomes a generated artifact: publish-on-merge aggregates site/results/*.json (best run per model) via `backend_score --build-board`, guards each entry + the built board, and deploys. It is gitignored; `make board` / `make serve` build it locally. - check_aggregate accepts a single-entry object (per-submission file) as well as the list, and allows the timestamp/submission_id provenance tags. - Also adds a reset_results workflow input (supersedes PR #34) and fixes the root-anchored /results/ gitignore so it no longer shadows site/results/. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_018xw6L5yUrz33UrQpanbwnH --- .github/scripts/check_aggregate.py | 14 +++- .github/workflows/publish-on-merge.yml | 24 +++--- .github/workflows/score-from-r2.yml | 86 ++++++++++++++------- .gitignore | 9 ++- Makefile | 38 +++++---- benchmark/backend_score.py | 103 ++++++++++++++++++++++++- benchmark/tests/test_backend_score.py | 67 ++++++++++++++++ site/results.json | 1 - site/results/README.md | 10 +++ 9 files changed, 288 insertions(+), 64 deletions(-) delete mode 100644 site/results.json create mode 100644 site/results/README.md diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py index 01c50fd..df7d5ba 100644 --- a/.github/scripts/check_aggregate.py +++ b/.github/scripts/check_aggregate.py @@ -18,6 +18,8 @@ "model", "library_commit", "budget_cap", "bugs_found", "rules_tested", "total_cost_usd", "total_tokens_k", "efficiency_bugs_per_ktok", "efficiency_bugs_per_dollar", "submitted_by", "placeholder", + # per-submission entry files (site/results/.json) also carry provenance tags + "timestamp", "submission_id", } # Substrings that must never appear anywhere in the serialized aggregate — a belt behind # the per-entry allowlist, in case the shape ever changes. @@ -36,10 +38,16 @@ def check(path: Path) -> list[str]: data = json.loads(raw) except json.JSONDecodeError as e: return [f"invalid JSON: {e}"] - if not isinstance(data, list): - return ["top-level value must be a list of leaderboard entries"] + # Accept either the built board (a list of entries) or one per-submission entry (an + # object) — both are guarded the same way, key by key. + if isinstance(data, dict): + entries = [data] + elif isinstance(data, list): + entries = data + else: + return ["top-level value must be a leaderboard entry object or a list of them"] - for i, entry in enumerate(data): + for i, entry in enumerate(entries): if not isinstance(entry, dict): problems.append(f"entry {i}: not an object") continue diff --git a/.github/workflows/publish-on-merge.yml b/.github/workflows/publish-on-merge.yml index ffcbd40..3daaf54 100644 --- a/.github/workflows/publish-on-merge.yml +++ b/.github/workflows/publish-on-merge.yml @@ -1,14 +1,10 @@ name: Publish leaderboard (GitHub Pages) # Publishes the static leaderboard site to GitHub Pages. The public repo holds ONLY the -# aggregate leaderboard (site/results.json: counts / cost / tokens / efficiency / Pareto — -# never certificates or buggy-rule identities). Scoring happens OFF the public repo: -# * self-run: `make publish-local` scores your local (gitignored) submissions and -# commits the refreshed site/results.json here. -# * external: the R2 scoring worker (score-from-r2.yml) re-verifies submissions pulled -# from private R2 and opens a PR that updates site/results.json. -# So this workflow never rebuilds from submissions/ — it just deploys site/, after guarding -# that no answer-key field leaked into the aggregate. +# aggregate leaderboard — never certificates or buggy-rule identities. The source of truth is +# one file per submission under site/results/.json (each merged via its own PR by +# score-from-r2.yml). This workflow builds the deployed site/results.json by aggregating them +# (best run per model), guards that nothing leaked, and deploys site/. # # One-time repo setup: Settings → Pages → Source = "GitHub Actions". @@ -37,8 +33,16 @@ jobs: with: python-version: '3.12' - - name: Egress guard — the published aggregate must carry no certificates / rule identities - run: python .github/scripts/check_aggregate.py site/results.json + - name: Build the deployed board from the per-submission entries + run: | + # Aggregate site/results/.json (best run per model) → the deployed + # site/results.json. Guard every source entry AND the built board. + for f in site/results/*.json; do + [ -e "$f" ] || continue + python .github/scripts/check_aggregate.py "$f" + done + python -m benchmark.backend_score --build-board site/results site/results.json + python .github/scripts/check_aggregate.py site/results.json - uses: actions/configure-pages@v6 - uses: actions/upload-pages-artifact@v5 diff --git a/.github/workflows/score-from-r2.yml b/.github/workflows/score-from-r2.yml index e038cc9..2611ed0 100644 --- a/.github/workflows/score-from-r2.yml +++ b/.github/workflows/score-from-r2.yml @@ -15,7 +15,12 @@ name: Score submissions from R2 (private → aggregate) # See intake/cloudflare-worker/README.md. on: - workflow_dispatch: {} # manual trigger (maintainer) + workflow_dispatch: # manual trigger (maintainer) + inputs: + reset_results: + description: "Wipe R2 results/ before scoring (clear stale scored files)" + type: boolean + default: false schedule: - cron: "0 16 * * *" # daily sweep of R2 for new submissions (16:00 UTC). @@ -61,8 +66,14 @@ jobs: - name: Pull pending submissions + prior scored detail from R2 run: | mkdir -p incoming results/scored + # reset_results (manual): wipe prior scored detail so the board is rebuilt from + # scratch — clears stale/test entries that predate a rule change. + if [ "${{ github.event.inputs.reset_results }}" = "true" ]; then + echo "reset_results: wiping s3://$BUCKET/results" + aws s3 rm "s3://$BUCKET/results" --recursive --endpoint-url "$R2_ENDPOINT" || true + fi # incoming/ = raw submissions to score; results/ = all prior scored detail, so the - # rebuilt leaderboard reflects the FULL history, not just this batch. + # rebuilt board reflects the FULL history, not just this batch. aws s3 sync "s3://$BUCKET/incoming" ./incoming --endpoint-url "$R2_ENDPOINT" aws s3 sync "s3://$BUCKET/results" ./results/scored --endpoint-url "$R2_ENDPOINT" echo "pending: $(ls -1 incoming/*.json 2>/dev/null | wc -l) file(s)" @@ -75,43 +86,58 @@ jobs: --entrypoint python prb-scoring:latest \ -m benchmark.backend_score --local /app/submissions /app/results/scored - - name: Stage the aggregate → site, and GUARD it (no certs / rule identities) - run: | - test -f results/scored/leaderboard.json - cp results/scored/leaderboard.json site/results.json - python .github/scripts/check_aggregate.py site/results.json - - name: Persist scored detail privately (R2), archive processed submissions run: | # Scored detail carries certificates — it goes back to PRIVATE R2, never to git. aws s3 sync ./results/scored "s3://$BUCKET/results" --endpoint-url "$R2_ENDPOINT" - # Move the now-scored raw submissions out of the queue. This step only runs when - # the scoring step exited 0 (no FAILED submissions), so we never archive an - # un-scored item. Limitation: the archive is coarse (whole incoming/ at once), so a - # permanently-malformed submission FAILs every run and jams the queue until it is - # deleted from R2 incoming/ by hand. Follow-up: archive per-file by FINISHED status. + # Move the now-scored raw submissions out of the queue. Runs only when the scoring + # step exited 0 (no FAILED), so we never archive an un-scored item. Limitation: the + # archive is coarse (whole incoming/ at once), so a permanently-malformed submission + # FAILs every run and jams the queue until deleted from R2 incoming/ by hand. aws s3 mv "s3://$BUCKET/incoming" "s3://$BUCKET/processed" \ --recursive --endpoint-url "$R2_ENDPOINT" || true - - name: Open a PR with the refreshed aggregate (main is protected) + - name: Open ONE PR per new/changed submission entry (main is protected) env: GH_TOKEN: ${{ github.token }} run: | - if git diff --quiet -- site/results.json; then - echo "No leaderboard change — nothing to publish."; exit 0 - fi + # backend_score wrote one PUBLIC entry file per non-test submission into + # results/scored/board/.json (slug = model--time--id). Each becomes its own + # PR that adds site/results/.json — independently reviewed, merged, reverted. + # The slug is deterministic, so re-scoring the same submission force-updates the SAME + # branch/PR (idempotent — no duplicate PRs); a different submission is its own PR. + shopt -s nullglob git config user.name "prb-bot" git config user.email "prb-bot@users.noreply.github.com" - # Summarize the new board for the PR title: how many models, how many total bugs. - MODELS=$(jq 'length' site/results.json) - BUGS=$(jq '[.[].bugs_found] | add // 0' site/results.json) - DATE=$(date -u +%Y-%m-%d) - TITLE="Update leaderboard — ${MODELS} models, ${BUGS} verified bugs (${DATE})" - BR="bot/leaderboard-${{ github.run_id }}" - git checkout -b "$BR" - git add site/results.json - git commit -m "$TITLE" - git push origin "$BR" - gh pr create --base main --head "$BR" \ - --title "$TITLE" \ - --body "Automated refresh of \`site/results.json\` from newly scored submissions in private R2. Certificates and rule identities stay private in R2 — only the aggregate counts change here (guarded by \`check_aggregate.py\`). Merging deploys the public site via publish-on-merge." + BASE="$(git rev-parse HEAD)" + mkdir -p site/results + opened=0 + for f in results/scored/board/*.json; do + slug="$(basename "$f" .json)" + dest="site/results/${slug}.json" + # Skip if this exact entry is already published on the base branch. + if git cat-file -e "$BASE:$dest" 2>/dev/null && git show "$BASE:$dest" | cmp -s - "$f"; then + echo "unchanged, skip: $slug"; continue + fi + MODEL="$(jq -r .model "$f")"; BUGS="$(jq -r '.bugs_found // 0' "$f")" + TS="$(jq -r '.timestamp // "?"' "$f")" + TITLE="leaderboard: ${MODEL} — ${BUGS} bug(s) @ ${TS}" + BR="bot/leaderboard/${slug}" + git checkout -B "$BR" "$BASE" >/dev/null + cp "$f" "$dest" + python .github/scripts/check_aggregate.py "$dest" # no certs / rule identities + git add "$dest" + git commit -q -m "$TITLE" + git push -f origin "$BR" + EXISTING="$(gh pr list --head "$BR" --state open --json number -q '.[0].number')" + if [ -n "$EXISTING" ]; then + gh pr edit "$EXISTING" --title "$TITLE" + echo "updated PR #$EXISTING ($slug)" + else + gh pr create --base "${{ github.ref_name }}" --head "$BR" --title "$TITLE" \ + --body "One submission's public entry (\`$dest\`) — aggregate only (counts / cost / tokens / efficiency), no certificates or rule identities (guarded by \`check_aggregate.py\`). The full certificate stays private in R2. Merging rebuilds \`site/results.json\` from all entries and deploys via publish-on-merge. This PR is scoped to a single submission; merge/close/revert it independently." + fi + opened=$((opened+1)) + done + echo "opened/updated $opened submission PR(s)" + git checkout -q "$BASE" 2>/dev/null || true diff --git a/.gitignore b/.gitignore index d9cb747..ae6bacc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -# Generated HF Dataset bundle (rebuild with `build_dataset.py`) -dataset/ +# Local scoring output (backend_score / score-local writes here). Root-anchored so it does +# NOT match the tracked per-submission board under site/results/. +/results/ -# Local scoring output (backend_score / score-local writes here) -results/ +# Deployed board — generated from site/results/*.json at publish (build-board); not committed. +/site/results.json # R2 sync scratch (score-from-r2 workflow / local scoring pulls here) — holds raw # submissions with certificates; never commit. diff --git a/Makefile b/Makefile index 860bd0f..510e578 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ SUBS_DIR ?= submissions SCORED ?= results/scored ENV_FILE ?= submission.env -.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help runner-build preflight run score-local publish-local serve +.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help runner-build preflight run score-local board publish-local serve ## Run the full test suite (unit + integration tests that need real repo). test: @@ -63,24 +63,33 @@ run: echo "No $(ENV_FILE) — copy submission.env.example and fill it in (then: make preflight)"; exit 1; fi mkdir -p out docker run --rm --env-file "$(ENV_FILE)" -v "$(PWD)/out:/out" $(IMAGE) - @echo "Wrote ./out/submission.json — now submit it via a GitHub PR (see CONTRIBUTING.md)." + @echo "Wrote ./out/submission.json — now submit it with 'python -m benchmark.submit' (see CONTRIBUTING.md)." -## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred). -## Writes scored results + leaderboard.json into SCORED. +## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred). Writes scored +## results + leaderboard.json into SCORED, and one public entry per submission into SCORED/board. score-local: python -m benchmark.backend_score --local $(SUBS_DIR) $(SCORED) -## Refresh the public site's aggregate from local scored submissions. SUBS_DIR holds the -## answer key (cert + trajectory) and is gitignored — it NEVER leaves your machine. This -## scores it, copies ONLY the aggregate leaderboard into site/results.json, and guards that -## no certificate / rule identity leaked. Commit site/results.json; SUBS_DIR stays local. -publish-local: score-local - cp $(SCORED)/leaderboard.json site/results.json +## Build the deployed board (site/results.json) from the per-submission entries in +## site/results/*.json (best run per model), then guard it. Generated — not committed. +board: + python -m benchmark.backend_score --build-board site/results site/results.json python .github/scripts/check_aggregate.py site/results.json - @echo "Updated site/results.json (aggregate only). Commit it — SUBS_DIR stays local." + @echo "Built site/results.json from site/results/*.json (aggregate only)." + +## Self-run publish: score SUBS_DIR (gitignored answer key, stays local), stage each +## submission's public entry into site/results/, and rebuild the deployed board. Commit the +## new site/results/.json files; SUBS_DIR stays local. +publish-local: score-local + mkdir -p site/results + for f in $(SCORED)/board/*.json; do \ + python .github/scripts/check_aggregate.py "$$f" && cp "$$f" site/results/; done + $(MAKE) board + @echo "Staged site/results/.json + rebuilt site/results.json. Commit the entries." -## Preview the leaderboard site locally (it's published to GitHub Pages on merge). -serve: +## Preview the leaderboard site locally (published to GitHub Pages on merge). Builds the +## board first so results.json (gitignored) exists for the preview. +serve: board @echo "Serving site/ at http://localhost:8000 (Ctrl-C to stop)" cd site && python3 -m http.server 8000 @@ -101,7 +110,8 @@ help: @echo " preflight Validate submission.env (1 tiny real call) before a full run" @echo " run Run the benchmark via Docker → out/submission.json (not upload)" @echo " score-local Score SUBS_DIR submissions with the backend" - @echo " publish-local Score + refresh site/results.json (aggregate only; SUBS_DIR stays local)" + @echo " board Build site/results.json from site/results/*.json (aggregate)" + @echo " publish-local Score + stage per-submission entries + rebuild board (SUBS_DIR stays local)" @echo " serve Preview the leaderboard site locally (published via Pages on push to site/)" @echo " audit Audit pred CLI capabilities" @echo " install-deps Install Python requirements" diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index 4504244..7c27dbe 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -15,7 +15,10 @@ .github/workflows/score-from-r2.yml after it pulls pending submissions from R2. """ import argparse +import datetime +import hashlib import json +import re import shutil import sys from pathlib import Path @@ -24,6 +27,90 @@ STATUS_SUFFIX = ".status.json" +# One PUBLIC file per submission lives at site/results/.json, where the slug ties the +# file (and its PR branch) to that specific run: model + submission time + a short id. The +# slug is derived deterministically from the scored file, so re-scoring the same submission +# yields the same slug/branch/PR (idempotent — no duplicate PRs), while a different +# submission (even same model) is a distinct file reviewed and merged on its own. +_SLUG_RE = re.compile(r"[^a-z0-9._-]+") + + +def _slug(text: str) -> str: + return _SLUG_RE.sub("-", str(text).lower()).strip("-") or "model" + + +def _submission_ts(scored: dict, stem: str) -> str: + """Compact UTC timestamp for the submission. Prefer the submission's own ``created_at``; + else the intake epoch (ms) that prefixes the R2 filename stem ``-``.""" + ca = scored.get("created_at") + if ca: + return (re.sub(r"[^0-9T]", "", str(ca))[:15]) or "unknown" + epoch = stem.split("-", 1)[0] + if epoch.isdigit(): + dt = datetime.datetime.fromtimestamp(int(epoch) / 1000, tz=datetime.timezone.utc) + return dt.strftime("%Y%m%dT%H%M%S") + return "unknown" + + +def _submission_id(stem: str) -> str: + """Short id for the submission — the uuid part of ``-``, else a stem hash.""" + _, sep, uid = stem.partition("-") + return uid[:8] if sep and uid else hashlib.sha1(stem.encode()).hexdigest()[:8] + + +def board_slug(scored: dict, stem: str) -> str: + return f"{_slug(scored['model'])}--{_submission_ts(scored, stem)}--{_submission_id(stem)}" + + +def board_entry(scored: dict, stem: str) -> dict: + """The public per-submission leaderboard entry, tagged with its time + id.""" + sub_view = {"model": scored["model"], "budget_cap": scored.get("budget_cap", 20), + "submitted_by": scored.get("submitted_by")} + entry = leaderboard_entry(sub_view, scored) + entry["timestamp"] = _submission_ts(scored, stem) + entry["submission_id"] = _submission_id(stem) + return entry + + +def write_board_entries(results_dir: Path, board_dir: Path) -> list[str]: + """Write one PUBLIC entry file per NON-test scored submission into ``board_dir``. + + ``board_dir/.json`` = the aggregate-only entry (no certs / rule identities). Test + submissions are skipped, so they never produce a public file. Returns the slugs written. + """ + board_dir.mkdir(parents=True, exist_ok=True) + slugs = [] + for p in sorted(results_dir.glob("*.json")): + if p.name == "leaderboard.json": + continue + try: + scored = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + if "results" not in scored or "model" not in scored or scored.get("test"): + continue + slug = board_slug(scored, p.stem) + (board_dir / f"{slug}.json").write_text( + json.dumps(board_entry(scored, p.stem), indent=2), encoding="utf-8") + slugs.append(slug) + return slugs + + +def build_board(entries_dir: Path) -> list[dict]: + """Aggregate the per-submission entry files (site/results/*.json) into the ranked board + (best run per model). This is what the deployed site/results.json is built from.""" + entries = [] + for p in sorted(Path(entries_dir).glob("*.json")): + if p.name == "results.json": + continue + try: + e = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + if isinstance(e, dict) and "model" in e: + entries.append(e) + return _dedup_best(entries) + def _assert_pred_version() -> None: """The backend is the authoritative verifier, so its pred must be the pinned version. @@ -159,16 +246,28 @@ def process_local(subs_dir: str, results_dir: str, repo_dir: str | None = None) except Exception as e: summary.append({"submission": sub_path.name, "status": "FAILED", "error": str(e)}) aggregate_leaderboard(results) + # Public per-submission entry files (one PR each downstream); test entries are excluded. + write_board_entries(results, results / "board") return summary def main() -> None: parser = argparse.ArgumentParser(description="Backend scoring queue for submissions") - parser.add_argument("--local", nargs=2, metavar=("SUBS_DIR", "RESULTS_DIR"), - required=True, help="Score submissions from a local directory") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--local", nargs=2, metavar=("SUBS_DIR", "RESULTS_DIR"), + help="Score submissions from a local directory") + mode.add_argument("--build-board", nargs=2, metavar=("ENTRIES_DIR", "OUT_JSON"), + help="Aggregate per-submission entry files (site/results/*.json) into " + "a ranked leaderboard JSON (the deployed site/results.json)") parser.add_argument("--repo-dir", default=None, help="problem-reductions repo (default: pred on PATH)") args = parser.parse_args() + if args.build_board: + board = build_board(Path(args.build_board[0])) + Path(args.build_board[1]).write_text(json.dumps(board, indent=2), encoding="utf-8") + print(f"built {args.build_board[1]}: {len(board)} model(s)") + return + summary = process_local(args.local[0], args.local[1], args.repo_dir) for s in summary: diff --git a/benchmark/tests/test_backend_score.py b/benchmark/tests/test_backend_score.py index f1ef56b..b8e5f95 100644 --- a/benchmark/tests/test_backend_score.py +++ b/benchmark/tests/test_backend_score.py @@ -155,6 +155,73 @@ def test_test_submission_excluded_from_board(self, tmp_path): assert "anthropic/tester" not in models # test entry excluded +class TestPerSubmissionBoard: + # R2 filenames are "-.json" — the slug derives from them. + F1 = "1783317767895-dc9b2aae-c838-4c15-a5aa-2f4a25f6f82c.json" + F2 = "1783317781060-6fb1aeb1-da04-4e1b-b6c5-f0b53c0d29a3.json" + + def _score(self, tmp_path, files): + subs, results = tmp_path / "subs", tmp_path / "results" + subs.mkdir() + for name, model, test in files: + _write_submission(subs, name, [{"rule": "R", "result": "no_bug"}], + model=model, test=test) + bs.process_local(str(subs), str(results)) + return results + + def test_slug_is_deterministic_and_tagged(self): + scored = {"model": "anthropic/claude-sonnet-4-6"} + stem = self.F1[:-5] + slug = bs.board_slug(scored, stem) + assert slug == bs.board_slug(scored, stem) # deterministic + assert slug == "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae" + full = {**scored, "results": [], "bugs_found": 0, "rules_tested": 0, + "total_cost_usd": 0, "total_tokens_k": 0, + "efficiency_bugs_per_ktok": 0, "efficiency_bugs_per_dollar": 0} + e = bs.board_entry(full, stem) + assert e["timestamp"] == "20260706T060247" and e["submission_id"] == "dc9b2aae" + + def test_one_entry_file_per_nontest_submission(self, tmp_path): + results = self._score(tmp_path, [ + (self.F1, "anthropic/claude-sonnet-4-6", False), + (self.F2, "openai/gpt-5", False), + ("1783317799999-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee.json", "anthropic/claude-sonnet-4-6", True), + ]) + board_files = sorted(p.name for p in (results / "board").glob("*.json")) + assert board_files == [ + "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae.json", + "openai-gpt-5--20260706T060301--6fb1aeb1.json", + ] # two non-test entries; the test submission produced no public file + + def test_write_board_entries_is_idempotent(self, tmp_path): + results = self._score(tmp_path, [(self.F1, "anthropic/x", False)]) + before = {p.name for p in (results / "board").glob("*.json")} + bs.write_board_entries(results, results / "board") + after = {p.name for p in (results / "board").glob("*.json")} + assert before == after and len(after) == 1 + + def test_build_board_dedups_best_per_model(self, tmp_path): + d = tmp_path / "entries" + d.mkdir() + (d / "m--t1--a.json").write_text(json.dumps( + {"model": "m", "bugs_found": 1, "efficiency_bugs_per_ktok": 0.1})) + (d / "m--t2--b.json").write_text(json.dumps( + {"model": "m", "bugs_found": 3, "efficiency_bugs_per_ktok": 0.2})) + (d / "n--t1--c.json").write_text(json.dumps( + {"model": "n", "bugs_found": 2, "efficiency_bugs_per_ktok": 0.1})) + board = bs.build_board(d) + assert [(e["model"], e["bugs_found"]) for e in board] == [("m", 3), ("n", 2)] + + def test_main_build_board(self, tmp_path, monkeypatch): + d = tmp_path / "entries" + d.mkdir() + (d / "m--t--a.json").write_text(json.dumps({"model": "m", "bugs_found": 1})) + out = tmp_path / "results.json" + monkeypatch.setattr("sys.argv", ["backend_score", "--build-board", str(d), str(out)]) + bs.main() + assert json.loads(out.read_text())[0]["model"] == "m" + + @pytest.mark.integration class TestRealFixture: def test_genuine_bug_end_to_end_scores_via_real_pred(self, tmp_path): diff --git a/site/results.json b/site/results.json deleted file mode 100644 index 0637a08..0000000 --- a/site/results.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/site/results/README.md b/site/results/README.md new file mode 100644 index 0000000..5b8efab --- /dev/null +++ b/site/results/README.md @@ -0,0 +1,10 @@ +# Per-submission leaderboard entries + +Each file here is **one submission's** public leaderboard entry: +`----.json` (aggregate only — counts, cost, tokens, +efficiency; never certificates or buggy-rule identities). + +- Written by `score-from-r2.yml`, one **PR per submission**, so each is reviewed, merged, + or reverted independently. +- On merge, `publish-on-merge.yml` aggregates them (best run per model) into the deployed + `site/results.json` (generated, not committed) and publishes to GitHub Pages.