Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions .github/workflows/score-from-r2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@ name: Score submissions from R2 (private → aggregate)
# See intake/cloudflare-worker/README.md.

on:
workflow_dispatch: {} # manual trigger (maintainer)
workflow_dispatch: # manual trigger (maintainer)
inputs:
reset_results:
description: "Wipe R2 results/ before scoring (clean the board of stale scored files)"
type: boolean
default: false
schedule:
- cron: "0 16 * * *" # daily sweep of R2 for new submissions (16:00 UTC).

Expand Down Expand Up @@ -61,6 +66,12 @@ jobs:
- name: Pull pending submissions + prior scored detail from R2
run: |
mkdir -p incoming results/scored
# reset_results (manual): wipe the prior scored detail so the board is rebuilt from
# scratch — used to clear stale/test entries that predate a rule change.
if [ "${{ github.event.inputs.reset_results }}" = "true" ]; then
echo "reset_results: wiping s3://$BUCKET/results"
aws s3 rm "s3://$BUCKET/results" --recursive --endpoint-url "$R2_ENDPOINT" || true
fi
# incoming/ = raw submissions to score; results/ = all prior scored detail, so the
# rebuilt leaderboard reflects the FULL history, not just this batch.
aws s3 sync "s3://$BUCKET/incoming" ./incoming --endpoint-url "$R2_ENDPOINT"
Expand Down Expand Up @@ -102,12 +113,22 @@ jobs:
MODELS=$(jq 'length' site/results.json)
BUGS=$(jq '[.[].bugs_found] | add // 0' site/results.json)
DATE=$(date -u +%Y-%m-%d)
TITLE="Update leaderboard — ${MODELS} models, ${BUGS} verified bugs (${DATE})"
BR="bot/leaderboard-${{ github.run_id }}"
git checkout -b "$BR"
[ "$MODELS" = 1 ] && MS="" || MS="s"
[ "$BUGS" = 1 ] && BS="" || BS="s"
TITLE="Update leaderboard — ${MODELS} model${MS}, ${BUGS} verified bug${BS} (${DATE})"
# One rolling branch → at most ONE open leaderboard PR ever. Each run force-updates
# it (and its PR) instead of opening a fresh bot/leaderboard-<run_id> PR every time.
BR="bot/leaderboard"
git checkout -B "$BR"
git add site/results.json
git commit -m "$TITLE"
git push origin "$BR"
gh pr create --base main --head "$BR" \
--title "$TITLE" \
--body "Automated refresh of \`site/results.json\` from newly scored submissions in private R2. Certificates and rule identities stay private in R2 — only the aggregate counts change here (guarded by \`check_aggregate.py\`). Merging deploys the public site via publish-on-merge."
git push -f origin "$BR"
EXISTING=$(gh pr list --head "$BR" --state open --json number -q '.[0].number')
if [ -n "$EXISTING" ]; then
gh pr edit "$EXISTING" --title "$TITLE"
echo "Updated existing leaderboard PR #$EXISTING"
else
gh pr create --base main --head "$BR" \
--title "$TITLE" \
--body "Automated refresh of \`site/results.json\` from newly scored submissions in private R2. Certificates and rule identities stay private in R2 — only the aggregate counts change here (guarded by \`check_aggregate.py\`). Merging deploys the public site via publish-on-merge. This PR is force-updated by each scoring run, so there is only ever one open leaderboard PR."
fi