From 133943e4afb4d8b89aba30aac5498103ebc07f13 Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Tue, 19 May 2026 19:06:38 -0700 Subject: [PATCH] fix: run full lenses for explicit review requests --- .github/scripts/evalops-pr-lens-review.rb | 11 +++++-- .github/workflows/evalops-pr-lens-review.yml | 18 ++++++++---- test/evalops_pr_lens_review_test.rb | 31 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/.github/scripts/evalops-pr-lens-review.rb b/.github/scripts/evalops-pr-lens-review.rb index 48bff3f..c0345e6 100644 --- a/.github/scripts/evalops-pr-lens-review.rb +++ b/.github/scripts/evalops-pr-lens-review.rb @@ -424,7 +424,7 @@ def pr_files_metadata(repo:, pr:) gh_api_json("repos/#{repo}/pulls/#{pr}/files?per_page=100") end - def discover_open_prs(repos:, pr_filter: nil) + def discover_open_prs(repos:, pr_filter: nil, force_lenses: nil) repos.flat_map do |repo| normalized_repo = normalize_repo(repo) prs = gh_api_json("repos/#{normalized_repo}/pulls?state=open&per_page=100") @@ -432,7 +432,7 @@ def discover_open_prs(repos:, pr_filter: nil) prs.map do |pr| files = pr_files_metadata(repo: normalized_repo, pr: Integer(pr.fetch("number"))) filenames = files.map { |file| file.fetch("filename") } - lenses = lenses_for_paths(filenames) + lenses = force_lenses || lenses_for_paths(filenames) { "repo" => normalized_repo, "repo_slug" => normalized_repo.tr("/", "-"), @@ -1359,13 +1359,18 @@ def markdown_meta_report(result) OptionParser.new do |parser| parser.on("--repos CSV") { |value| options[:repos] = EvalOpsPrLensReview.parse_list(value) } parser.on("--target-prs CSV") { |value| options[:target_prs] = value } + parser.on("--force-lenses CSV") { |value| options[:force_lenses] = EvalOpsPrLensReview.parse_list(value) } parser.on("--github-output PATH") { |value| options[:github_output] = value } parser.on("--matrix-output PATH") { |value| options[:matrix_output] = value } parser.on("--targets-output PATH") { |value| options[:targets_output] = value } end.parse! pr_filter = EvalOpsPrLensReview.parse_pr_filter(options[:target_prs], repos: options[:repos]) if options[:target_prs] - prs = EvalOpsPrLensReview.discover_open_prs(repos: options[:repos], pr_filter: pr_filter) + prs = EvalOpsPrLensReview.discover_open_prs( + repos: options[:repos], + pr_filter: pr_filter, + force_lenses: options[:force_lenses] + ) matrix = EvalOpsPrLensReview.matrix_for(prs, lenses: options[:lenses]) matrix_json = JSON.generate({ include: matrix }) diff --git a/.github/workflows/evalops-pr-lens-review.yml b/.github/workflows/evalops-pr-lens-review.yml index 854f1e7..e36e6fd 100644 --- a/.github/workflows/evalops-pr-lens-review.yml +++ b/.github/workflows/evalops-pr-lens-review.yml @@ -51,6 +51,8 @@ jobs: GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN || secrets.EVALOPS_ORG_READ_TOKEN }} TARGET_REPOS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_repos || github.event.client_payload.target_repo) || inputs.target_repos || 'evalops/platform,evalops/deploy,evalops/maestro-internal' }} TARGET_PRS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_prs || github.event.client_payload.target_pr) || inputs.target_prs || '' }} + FORCE_EXPLICIT_REVIEW_LENSES: ${{ github.event_name == 'repository_dispatch' && 'true' || 'false' }} + EXPLICIT_REVIEW_LENSES: migration-safety,nats-contract-drift,argo-manifest-skew,iam-blast-radius,generated-sdk-delta,eval-regression-risk PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -90,12 +92,18 @@ jobs: shell: bash run: | set -euo pipefail - ruby .github/scripts/evalops-pr-lens-review.rb discover \ - --repos "${TARGET_REPOS}" \ - --target-prs "${TARGET_PRS}" \ - --github-output "${GITHUB_OUTPUT}" \ - --matrix-output pr-lens-matrix.json \ + args=( + discover + --repos "${TARGET_REPOS}" + --target-prs "${TARGET_PRS}" + --github-output "${GITHUB_OUTPUT}" + --matrix-output pr-lens-matrix.json --targets-output pr-lens-targets.json + ) + if [ "${FORCE_EXPLICIT_REVIEW_LENSES}" = "true" ] && [ -n "${TARGET_PRS}" ]; then + args+=(--force-lenses "${EXPLICIT_REVIEW_LENSES}") + fi + ruby .github/scripts/evalops-pr-lens-review.rb "${args[@]}" - name: Upload discovery ledger uses: actions/upload-artifact@v4 diff --git a/test/evalops_pr_lens_review_test.rb b/test/evalops_pr_lens_review_test.rb index 1ce340f..1c8e1bd 100644 --- a/test/evalops_pr_lens_review_test.rb +++ b/test/evalops_pr_lens_review_test.rb @@ -90,6 +90,37 @@ def test_lenses_for_paths_skips_docs_only_prs assert_empty EvalOpsPrLensReview.lenses_for_paths(["README.md", "docs/runbook.md"]) end + def test_discover_open_prs_can_force_lenses_for_explicit_review_requests + pr = { + "number" => 103, + "title" => "Canary", + "html_url" => "https://github.com/evalops/.github/pull/103", + "draft" => false, + "head" => { "sha" => "head", "ref" => "evalopsbot-review-canary" }, + "base" => { "sha" => "base", "ref" => "main" } + } + api = lambda do |path, **_kwargs| + case path + when "repos/evalops/.github/pulls?state=open&per_page=100" + [pr] + when "repos/evalops/.github/pulls/103/files?per_page=100" + [{ "filename" => ".github/evalopsbot-canary/review-request.md" }] + else + flunk "unexpected gh api path #{path}" + end + end + + EvalOpsPrLensReview.stub(:gh_api_json, api) do + prs = EvalOpsPrLensReview.discover_open_prs( + repos: ["evalops/.github"], + pr_filter: { "evalops/.github" => [103] }, + force_lenses: %w[migration-safety iam-blast-radius] + ) + + assert_equal %w[migration-safety iam-blast-radius], prs.fetch(0).fetch("lenses") + end + end + def test_normalize_lens_review_drops_invalid_findings raw = { "summary" => "Found one issue",