From ef86b697b3e403a34ddf405426f441d808b2f2c0 Mon Sep 17 00:00:00 2001 From: Jorge Calvar Date: Tue, 9 Jun 2026 16:09:15 +0200 Subject: [PATCH 1/3] ci: post evals-monitor link comment on new PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a workflow that comments once on each opened/reopened PR with a small link to the evals-monitor PR page, where the author can start an eval. This replaces the direct CI-trigger approach (see closed #428): GitHub Actions runners can't reach the dogfood.staging workspace, so the app — which runs inside dogfood — performs the actual trigger instead. Co-authored-by: Isaac Signed-off-by: Jorge Calvar --- .github/workflows/eval-pr-comment.yml | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/eval-pr-comment.yml diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml new file mode 100644 index 000000000..dfff78e18 --- /dev/null +++ b/.github/workflows/eval-pr-comment.yml @@ -0,0 +1,48 @@ +name: Eval PR Comment + +# Posts a small comment on every new PR with a link to the evals-monitor app, +# where the author can start an eval for their PR. We post a link rather than +# triggering the job from CI because GitHub Actions runners cannot reach the +# dogfood.staging workspace (network perimeter). +on: + pull_request: + types: [opened, reopened] + branches: [main] + +permissions: + contents: read + pull-requests: write + +jobs: + post-eval-link: + name: Post eval link comment + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + steps: + - name: Post eval link comment + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const marker = ""; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const url = `https://evals-monitor-6051921418418893.staging.aws.databricksapps.com/prs/appkit/${issue_number}`; + + // Idempotent: skip if we've already commented on this PR. + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100, + }); + if (comments.some((c) => c.body?.includes(marker))) return; + + const body = `${marker}\n> 🧪  **Run evals on this PR**  ·  [**Start eval →**](${url})`; + + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body, + }); From 166311300f33085f6c8f1e6699be8a7ba3c79a68 Mon Sep 17 00:00:00 2001 From: Jorge Calvar Date: Tue, 9 Jun 2026 16:29:28 +0200 Subject: [PATCH 2/3] ci: refine eval PR comment (microscope emoji, 'Go to Evals Monitor' link) Co-authored-by: Isaac Signed-off-by: Jorge Calvar --- .github/workflows/eval-pr-comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml index dfff78e18..9f88e20a6 100644 --- a/.github/workflows/eval-pr-comment.yml +++ b/.github/workflows/eval-pr-comment.yml @@ -38,7 +38,7 @@ jobs: }); if (comments.some((c) => c.body?.includes(marker))) return; - const body = `${marker}\n> 🧪  **Run evals on this PR**  ·  [**Start eval →**](${url})`; + const body = `${marker}\n> 🔬  **Run evals on this PR**  ·  [**Go to Evals Monitor →**](${url})`; await github.rest.issues.createComment({ owner, From 713492e048e70b24d41b39071743cf01e993b2ab Mon Sep 17 00:00:00 2001 From: Jorge Calvar Date: Mon, 22 Jun 2026 17:12:19 +0200 Subject: [PATCH 3/3] ci: serialize eval PR comment per-PR and skip fork PRs Add a per-PR concurrency group (cancel-in-progress: false) so a rapid open+reopen can't race the marker check and double-post. Skip fork PRs, whose read-only GITHUB_TOKEN can't comment and for whom the internal evals-monitor link is useless anyway. Co-authored-by: Isaac Signed-off-by: Jorge Calvar --- .github/workflows/eval-pr-comment.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml index 9f88e20a6..b80ec5f0d 100644 --- a/.github/workflows/eval-pr-comment.yml +++ b/.github/workflows/eval-pr-comment.yml @@ -13,9 +13,21 @@ permissions: contents: read pull-requests: write +# Serialize per-PR so a rapid open+reopen can't race the marker check and +# double-post. cancel-in-progress: false — the queued run still executes and +# deterministically sees the prior run's comment (then skips); there's no +# "latest commit wins" semantics here, the posted link is identical either way. +concurrency: + group: eval-pr-comment-${{ github.event.pull_request.number }} + cancel-in-progress: false + jobs: post-eval-link: name: Post eval link comment + # Skip fork PRs: their GITHUB_TOKEN is read-only (createComment would 403), + # and the evals-monitor app is internal — the link is useless to external + # contributors anyway. + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest