Tidy the test-submission marker after review

.github/workflows/score-from-r2.yml

@@ -85,7 +85,11 @@ jobs:
         run: |
           # Scored detail carries certificates — it goes back to PRIVATE R2, never to git.
           aws s3 sync ./results/scored "s3://$BUCKET/results" --endpoint-url "$R2_ENDPOINT"
-          # Move the now-scored raw submissions out of the queue.
+          # Move the now-scored raw submissions out of the queue. This step only runs when
+          # the scoring step exited 0 (no FAILED submissions), so we never archive an
+          # un-scored item. Limitation: the archive is coarse (whole incoming/ at once), so a
+          # permanently-malformed submission FAILs every run and jams the queue until it is
+          # deleted from R2 incoming/ by hand. Follow-up: archive per-file by FINISHED status.
           aws s3 mv "s3://$BUCKET/incoming" "s3://$BUCKET/processed" \
             --recursive --endpoint-url "$R2_ENDPOINT" || true
 

benchmark/submission.schema.json

@@ -43,10 +43,6 @@
       "minimum": 0,
       "description": "Number of rules the run attempted before the budget ran out"
     },
-    "test": {
-      "type": "boolean",
-      "description": "Optional. When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."
-    },
     "results": {
       "type": "array",
       "description": "Per-rule results, byte-compatible with results.schema.json items",
@@ -101,6 +97,7 @@
     "pred_version": {"type": "string", "description": "Version of the pred binary used (must match the pinned tag; verified server-side)"},
     "created_at": {"type": "string", "description": "ISO-8601 timestamp the run finished"},
     "notes": {"type": "string"},
-    "placeholder": {"type": "boolean", "description": "True for demo/placeholder rows, never ranked"}
+    "placeholder": {"type": "boolean", "description": "True for demo/placeholder rows, never ranked"},
+    "test": {"type": "boolean", "description": "When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."}
   }
 }

benchmark/submit.py

@@ -116,8 +116,7 @@ def submit(path: Path, url: str, api_key: str, *, dry_run: bool = False,
     bugs = sum(1 for r in sub.get("results", []) if r.get("result") == "bug_found")
     if dry_run:
         return {"status": "dry-run (not sent)", "model": sub.get("model"),
-                "claimed_bugs": bugs, "bytes": len(path.read_bytes()),
-                "test": bool(sub.get("test"))}
+                "claimed_bugs": bugs, "bytes": len(path.read_bytes())}
 
     if not url:
         raise ValueError("no endpoint URL — set PRB_SUBMIT_URL or pass --url")

benchmark/verify_submission.py

@@ -119,7 +119,7 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic
         # Test submissions (end-to-end checks) are scored and stored privately like any
         # other, but aggregate_leaderboard() excludes them so they never reach the public
         # board. Carried through here so the flag survives in the private scored file.
-        "test": bool(submission.get("test", False)),
+        "test": bool(submission.get("test")),
         "bugs_found": bugs,
         "total_cost_usd": cost,
         "total_tokens_k": tokens_k,