From c7e7363ae1faaa02975af10d86d73cbac2ffeb14 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 6 Jul 2026 13:38:52 +0800 Subject: [PATCH] refactor: tidy the test-submission marker after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup from a code-quality pass (no behavior change): - submit.py: drop the unused "test" key from the dry-run return dict (the CLI tag derives from args.test, nothing reads the result key). - verify_submission.py: bool(submission.get("test")) — the explicit False default was redundant. - submission.schema.json: move "test" into the optional-field cluster after results (next to placeholder), drop the "Optional." lead-in for house style. - score-from-r2.yml: comment the coarse whole-incoming/ archive, flagging that a permanently-FAILing submission jams the queue until deleted by hand (follow-up: archive per-file by FINISHED status). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_018xw6L5yUrz33UrQpanbwnH --- .github/workflows/score-from-r2.yml | 6 +++++- benchmark/submission.schema.json | 7 ++----- benchmark/submit.py | 3 +-- benchmark/verify_submission.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/score-from-r2.yml b/.github/workflows/score-from-r2.yml index a6282e1..e038cc9 100644 --- a/.github/workflows/score-from-r2.yml +++ b/.github/workflows/score-from-r2.yml @@ -85,7 +85,11 @@ jobs: run: | # Scored detail carries certificates — it goes back to PRIVATE R2, never to git. aws s3 sync ./results/scored "s3://$BUCKET/results" --endpoint-url "$R2_ENDPOINT" - # Move the now-scored raw submissions out of the queue. + # Move the now-scored raw submissions out of the queue. This step only runs when + # the scoring step exited 0 (no FAILED submissions), so we never archive an + # un-scored item. Limitation: the archive is coarse (whole incoming/ at once), so a + # permanently-malformed submission FAILs every run and jams the queue until it is + # deleted from R2 incoming/ by hand. Follow-up: archive per-file by FINISHED status. aws s3 mv "s3://$BUCKET/incoming" "s3://$BUCKET/processed" \ --recursive --endpoint-url "$R2_ENDPOINT" || true diff --git a/benchmark/submission.schema.json b/benchmark/submission.schema.json index f6699d2..20ddce3 100644 --- a/benchmark/submission.schema.json +++ b/benchmark/submission.schema.json @@ -43,10 +43,6 @@ "minimum": 0, "description": "Number of rules the run attempted before the budget ran out" }, - "test": { - "type": "boolean", - "description": "Optional. When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`." - }, "results": { "type": "array", "description": "Per-rule results, byte-compatible with results.schema.json items", @@ -101,6 +97,7 @@ "pred_version": {"type": "string", "description": "Version of the pred binary used (must match the pinned tag; verified server-side)"}, "created_at": {"type": "string", "description": "ISO-8601 timestamp the run finished"}, "notes": {"type": "string"}, - "placeholder": {"type": "boolean", "description": "True for demo/placeholder rows, never ranked"} + "placeholder": {"type": "boolean", "description": "True for demo/placeholder rows, never ranked"}, + "test": {"type": "boolean", "description": "When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."} } } diff --git a/benchmark/submit.py b/benchmark/submit.py index 2de8be0..2985e4d 100644 --- a/benchmark/submit.py +++ b/benchmark/submit.py @@ -116,8 +116,7 @@ def submit(path: Path, url: str, api_key: str, *, dry_run: bool = False, bugs = sum(1 for r in sub.get("results", []) if r.get("result") == "bug_found") if dry_run: return {"status": "dry-run (not sent)", "model": sub.get("model"), - "claimed_bugs": bugs, "bytes": len(path.read_bytes()), - "test": bool(sub.get("test"))} + "claimed_bugs": bugs, "bytes": len(path.read_bytes())} if not url: raise ValueError("no endpoint URL — set PRB_SUBMIT_URL or pass --url") diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py index 513f977..f768f79 100644 --- a/benchmark/verify_submission.py +++ b/benchmark/verify_submission.py @@ -119,7 +119,7 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic # Test submissions (end-to-end checks) are scored and stored privately like any # other, but aggregate_leaderboard() excludes them so they never reach the public # board. Carried through here so the flag survives in the private scored file. - "test": bool(submission.get("test", False)), + "test": bool(submission.get("test")), "bugs_found": bugs, "total_cost_usd": cost, "total_tokens_k": tokens_k,