Per-submission board: one source file + one PR per submission

.github/scripts/check_aggregate.py

@@ -18,6 +18,8 @@
     "model", "library_commit", "budget_cap", "bugs_found", "rules_tested",
     "total_cost_usd", "total_tokens_k", "efficiency_bugs_per_ktok",
     "efficiency_bugs_per_dollar", "submitted_by", "placeholder",
+    # per-submission entry files (site/results/<slug>.json) also carry provenance tags
+    "timestamp", "submission_id",
 }
 # Substrings that must never appear anywhere in the serialized aggregate — a belt behind
 # the per-entry allowlist, in case the shape ever changes.
@@ -36,10 +38,16 @@ def check(path: Path) -> list[str]:
         data = json.loads(raw)
     except json.JSONDecodeError as e:
         return [f"invalid JSON: {e}"]
-    if not isinstance(data, list):
-        return ["top-level value must be a list of leaderboard entries"]
+    # Accept either the built board (a list of entries) or one per-submission entry (an
+    # object) — both are guarded the same way, key by key.
+    if isinstance(data, dict):
+        entries = [data]
+    elif isinstance(data, list):
+        entries = data
+    else:
+        return ["top-level value must be a leaderboard entry object or a list of them"]
 
-    for i, entry in enumerate(data):
+    for i, entry in enumerate(entries):
         if not isinstance(entry, dict):
             problems.append(f"entry {i}: not an object")
             continue

.github/workflows/publish-on-merge.yml

@@ -1,14 +1,10 @@
 name: Publish leaderboard (GitHub Pages)
 
 # Publishes the static leaderboard site to GitHub Pages. The public repo holds ONLY the
-# aggregate leaderboard (site/results.json: counts / cost / tokens / efficiency / Pareto —
-# never certificates or buggy-rule identities). Scoring happens OFF the public repo:
-#   * self-run:   `make publish-local` scores your local (gitignored) submissions and
-#                 commits the refreshed site/results.json here.
-#   * external:   the R2 scoring worker (score-from-r2.yml) re-verifies submissions pulled
-#                 from private R2 and opens a PR that updates site/results.json.
-# So this workflow never rebuilds from submissions/ — it just deploys site/, after guarding
-# that no answer-key field leaked into the aggregate.
+# aggregate leaderboard — never certificates or buggy-rule identities. The source of truth is
+# one file per submission under site/results/<slug>.json (each merged via its own PR by
+# score-from-r2.yml). This workflow builds the deployed site/results.json by aggregating them
+# (best run per model), guards that nothing leaked, and deploys site/.
 #
 # One-time repo setup: Settings → Pages → Source = "GitHub Actions".
 
@@ -37,8 +33,16 @@ jobs:
         with:
           python-version: '3.12'
 
-      - name: Egress guard — the published aggregate must carry no certificates / rule identities
-        run: python .github/scripts/check_aggregate.py site/results.json
+      - name: Build the deployed board from the per-submission entries
+        run: |
+          # Aggregate site/results/<slug>.json (best run per model) → the deployed
+          # site/results.json. Guard every source entry AND the built board.
+          for f in site/results/*.json; do
+            [ -e "$f" ] || continue
+            python .github/scripts/check_aggregate.py "$f"
+          done
+          python -m benchmark.backend_score --build-board site/results site/results.json
+          python .github/scripts/check_aggregate.py site/results.json
 
       - uses: actions/configure-pages@v6
       - uses: actions/upload-pages-artifact@v5

.github/workflows/score-from-r2.yml

@@ -15,7 +15,12 @@ name: Score submissions from R2 (private → aggregate)
 # See intake/cloudflare-worker/README.md.
 
 on:
-  workflow_dispatch: {}       # manual trigger (maintainer)
+  workflow_dispatch:          # manual trigger (maintainer)
+    inputs:
+      reset_results:
+        description: "Wipe R2 results/ before scoring (clear stale scored files)"
+        type: boolean
+        default: false
   schedule:
     - cron: "0 16 * * *"      # daily sweep of R2 for new submissions (16:00 UTC).
 
@@ -61,8 +66,14 @@ jobs:
       - name: Pull pending submissions + prior scored detail from R2
         run: |
           mkdir -p incoming results/scored
+          # reset_results (manual): wipe prior scored detail so the board is rebuilt from
+          # scratch — clears stale/test entries that predate a rule change.
+          if [ "${{ github.event.inputs.reset_results }}" = "true" ]; then
+            echo "reset_results: wiping s3://$BUCKET/results"
+            aws s3 rm "s3://$BUCKET/results" --recursive --endpoint-url "$R2_ENDPOINT" || true
+          fi
           # incoming/ = raw submissions to score; results/ = all prior scored detail, so the
-          # rebuilt leaderboard reflects the FULL history, not just this batch.
+          # rebuilt board reflects the FULL history, not just this batch.
           aws s3 sync "s3://$BUCKET/incoming" ./incoming --endpoint-url "$R2_ENDPOINT"
           aws s3 sync "s3://$BUCKET/results"  ./results/scored --endpoint-url "$R2_ENDPOINT"
           echo "pending: $(ls -1 incoming/*.json 2>/dev/null | wc -l) file(s)"
@@ -75,43 +86,58 @@ jobs:
             --entrypoint python prb-scoring:latest \
             -m benchmark.backend_score --local /app/submissions /app/results/scored
 
-      - name: Stage the aggregate → site, and GUARD it (no certs / rule identities)
-        run: |
-          test -f results/scored/leaderboard.json
-          cp results/scored/leaderboard.json site/results.json
-          python .github/scripts/check_aggregate.py site/results.json
-
       - name: Persist scored detail privately (R2), archive processed submissions
         run: |
           # Scored detail carries certificates — it goes back to PRIVATE R2, never to git.
           aws s3 sync ./results/scored "s3://$BUCKET/results" --endpoint-url "$R2_ENDPOINT"
-          # Move the now-scored raw submissions out of the queue. This step only runs when
-          # the scoring step exited 0 (no FAILED submissions), so we never archive an
-          # un-scored item. Limitation: the archive is coarse (whole incoming/ at once), so a
-          # permanently-malformed submission FAILs every run and jams the queue until it is
-          # deleted from R2 incoming/ by hand. Follow-up: archive per-file by FINISHED status.
+          # Move the now-scored raw submissions out of the queue. Runs only when the scoring
+          # step exited 0 (no FAILED), so we never archive an un-scored item. Limitation: the
+          # archive is coarse (whole incoming/ at once), so a permanently-malformed submission
+          # FAILs every run and jams the queue until deleted from R2 incoming/ by hand.
           aws s3 mv "s3://$BUCKET/incoming" "s3://$BUCKET/processed" \
             --recursive --endpoint-url "$R2_ENDPOINT" || true
 
-      - name: Open a PR with the refreshed aggregate (main is protected)
+      - name: Open ONE PR per new/changed submission entry (main is protected)
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          if git diff --quiet -- site/results.json; then
-            echo "No leaderboard change — nothing to publish."; exit 0
-          fi
+          # backend_score wrote one PUBLIC entry file per non-test submission into
+          # results/scored/board/<slug>.json (slug = model--time--id). Each becomes its own
+          # PR that adds site/results/<slug>.json — independently reviewed, merged, reverted.
+          # The slug is deterministic, so re-scoring the same submission force-updates the SAME
+          # branch/PR (idempotent — no duplicate PRs); a different submission is its own PR.
+          shopt -s nullglob
           git config user.name  "prb-bot"
           git config user.email "prb-bot@users.noreply.github.com"
-          # Summarize the new board for the PR title: how many models, how many total bugs.
-          MODELS=$(jq 'length' site/results.json)
-          BUGS=$(jq '[.[].bugs_found] | add // 0' site/results.json)
-          DATE=$(date -u +%Y-%m-%d)
-          TITLE="Update leaderboard — ${MODELS} models, ${BUGS} verified bugs (${DATE})"
-          BR="bot/leaderboard-${{ github.run_id }}"
-          git checkout -b "$BR"
-          git add site/results.json
-          git commit -m "$TITLE"
-          git push origin "$BR"
-          gh pr create --base main --head "$BR" \
-            --title "$TITLE" \
-            --body "Automated refresh of \`site/results.json\` from newly scored submissions in private R2. Certificates and rule identities stay private in R2 — only the aggregate counts change here (guarded by \`check_aggregate.py\`). Merging deploys the public site via publish-on-merge."
+          BASE="$(git rev-parse HEAD)"
+          mkdir -p site/results
+          opened=0
+          for f in results/scored/board/*.json; do
+            slug="$(basename "$f" .json)"
+            dest="site/results/${slug}.json"
+            # Skip if this exact entry is already published on the base branch.
+            if git cat-file -e "$BASE:$dest" 2>/dev/null && git show "$BASE:$dest" | cmp -s - "$f"; then
+              echo "unchanged, skip: $slug"; continue
+            fi
+            MODEL="$(jq -r .model "$f")"; BUGS="$(jq -r '.bugs_found // 0' "$f")"
+            TS="$(jq -r '.timestamp // "?"' "$f")"
+            TITLE="leaderboard: ${MODEL} — ${BUGS} bug(s) @ ${TS}"
+            BR="bot/leaderboard/${slug}"
+            git checkout -B "$BR" "$BASE" >/dev/null
+            cp "$f" "$dest"
+            python .github/scripts/check_aggregate.py "$dest"   # no certs / rule identities
+            git add "$dest"
+            git commit -q -m "$TITLE"
+            git push -f origin "$BR"
+            EXISTING="$(gh pr list --head "$BR" --state open --json number -q '.[0].number')"
+            if [ -n "$EXISTING" ]; then
+              gh pr edit "$EXISTING" --title "$TITLE"
+              echo "updated PR #$EXISTING ($slug)"
+            else
+              gh pr create --base "${{ github.ref_name }}" --head "$BR" --title "$TITLE" \
+                --body "One submission's public entry (\`$dest\`) — aggregate only (counts / cost / tokens / efficiency), no certificates or rule identities (guarded by \`check_aggregate.py\`). The full certificate stays private in R2. Merging rebuilds \`site/results.json\` from all entries and deploys via publish-on-merge. This PR is scoped to a single submission; merge/close/revert it independently."
+            fi
+            opened=$((opened+1))
+          done
+          echo "opened/updated $opened submission PR(s)"
+          git checkout -q "$BASE" 2>/dev/null || true

.gitignore

@@ -1,8 +1,9 @@
-# Generated HF Dataset bundle (rebuild with `build_dataset.py`)
-dataset/
+# Local scoring output (backend_score / score-local writes here). Root-anchored so it does
+# NOT match the tracked per-submission board under site/results/.
+/results/
 
-# Local scoring output (backend_score / score-local writes here)
-results/
+# Deployed board — generated from site/results/*.json at publish (build-board); not committed.
+/site/results.json
 
 # R2 sync scratch (score-from-r2 workflow / local scoring pulls here) — holds raw
 # submissions with certificates; never commit.

Makefile

@@ -22,7 +22,7 @@ SUBS_DIR ?= submissions
 SCORED   ?= results/scored
 ENV_FILE ?= submission.env
 
-.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help runner-build preflight run score-local publish-local serve
+.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help runner-build preflight run score-local board publish-local serve
 
 ## Run the full test suite (unit + integration tests that need real repo).
 test:
@@ -63,24 +63,33 @@ run:
 	  echo "No $(ENV_FILE) — copy submission.env.example and fill it in (then: make preflight)"; exit 1; fi
 	mkdir -p out
 	docker run --rm --env-file "$(ENV_FILE)" -v "$(PWD)/out:/out" $(IMAGE)
-	@echo "Wrote ./out/submission.json — now submit it via a GitHub PR (see CONTRIBUTING.md)."
+	@echo "Wrote ./out/submission.json — now submit it with 'python -m benchmark.submit' (see CONTRIBUTING.md)."
 
-## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred).
-## Writes scored results + leaderboard.json into SCORED.
+## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred). Writes scored
+## results + leaderboard.json into SCORED, and one public entry per submission into SCORED/board.
 score-local:
 	python -m benchmark.backend_score --local $(SUBS_DIR) $(SCORED)
 
-## Refresh the public site's aggregate from local scored submissions. SUBS_DIR holds the
-## answer key (cert + trajectory) and is gitignored — it NEVER leaves your machine. This
-## scores it, copies ONLY the aggregate leaderboard into site/results.json, and guards that
-## no certificate / rule identity leaked. Commit site/results.json; SUBS_DIR stays local.
-publish-local: score-local
-	cp $(SCORED)/leaderboard.json site/results.json
+## Build the deployed board (site/results.json) from the per-submission entries in
+## site/results/*.json (best run per model), then guard it. Generated — not committed.
+board:
+	python -m benchmark.backend_score --build-board site/results site/results.json
 	python .github/scripts/check_aggregate.py site/results.json
-	@echo "Updated site/results.json (aggregate only). Commit it — SUBS_DIR stays local."
+	@echo "Built site/results.json from site/results/*.json (aggregate only)."
+
+## Self-run publish: score SUBS_DIR (gitignored answer key, stays local), stage each
+## submission's public entry into site/results/, and rebuild the deployed board. Commit the
+## new site/results/<slug>.json files; SUBS_DIR stays local.
+publish-local: score-local
+	mkdir -p site/results
+	for f in $(SCORED)/board/*.json; do \
+	  python .github/scripts/check_aggregate.py "$$f" && cp "$$f" site/results/; done
+	$(MAKE) board
+	@echo "Staged site/results/<slug>.json + rebuilt site/results.json. Commit the entries."
 
-## Preview the leaderboard site locally (it's published to GitHub Pages on merge).
-serve:
+## Preview the leaderboard site locally (published to GitHub Pages on merge). Builds the
+## board first so results.json (gitignored) exists for the preview.
+serve: board
 	@echo "Serving site/ at http://localhost:8000  (Ctrl-C to stop)"
 	cd site && python3 -m http.server 8000
 
@@ -101,7 +110,8 @@ help:
 	@echo "  preflight           Validate submission.env (1 tiny real call) before a full run"
 	@echo "  run                 Run the benchmark via Docker → out/submission.json (not upload)"
 	@echo "  score-local         Score SUBS_DIR submissions with the backend"
-	@echo "  publish-local       Score + refresh site/results.json (aggregate only; SUBS_DIR stays local)"
+	@echo "  board               Build site/results.json from site/results/*.json (aggregate)"
+	@echo "  publish-local       Score + stage per-submission entries + rebuild board (SUBS_DIR stays local)"
 	@echo "  serve               Preview the leaderboard site locally (published via Pages on push to site/)"
 	@echo "  audit               Audit pred CLI capabilities"
 	@echo "  install-deps        Install Python requirements"