fix: ignore superseded top-level review feedback (#120)

Author: haasonsaas

.github/scripts/check-pr-review-threads.rb

@@ -77,6 +77,7 @@ def unresolved_threads(payload, min_severity: "high", include_outdated: false)
   def top_level_feedback(payload, min_severity: "high")
     threshold = SEVERITY_RANK.fetch(min_severity)
     pull_request = payload.dig("data", "repository", "pullRequest") || {}
+    current_head_oid = pull_request["headRefOid"].to_s
     feedback = []
     Array(pull_request.dig("comments", "nodes")).each do |comment|
       next if informational_summary?(comment["body"], author: comment.dig("author", "login"))
@@ -98,6 +99,9 @@ def top_level_feedback(payload, min_severity: "high")
       detected = severity(review["body"])
       next if SEVERITY_RANK.fetch(detected) < threshold
 
+      review_commit_oid = review.dig("commit", "oid").to_s
+      next if !current_head_oid.empty? && !review_commit_oid.empty? && review_commit_oid != current_head_oid
+
       feedback << {
         kind: "pr_review",
         severity: detected,
@@ -148,6 +152,7 @@ def graphql_query
       query($owner:String!,$repo:String!,$number:Int!) {
         repository(owner:$owner, name:$repo) {
           pullRequest(number:$number) {
+            headRefOid
             comments(first:100) {
               pageInfo {
                 hasNextPage
@@ -171,6 +176,9 @@ def graphql_query
                   login
                 }
                 body
+                commit {
+                  oid
+                }
                 state
                 url
               }
@@ -239,6 +247,9 @@ def reviews_page_query
                   login
                 }
                 body
+                commit {
+                  oid
+                }
                 state
                 url
               }

.github/workflows/review-thread-guard.yml

@@ -21,7 +21,6 @@ on:
         description: "evalops/.github ref to checkout for guard scripts"
         required: false
         type: string
-        default: c02a97ba9b92c6b2ac837aab77dc3becb77f301c
       settle_seconds:
         description: "Seconds to wait before checking review feedback so bot reviews can finish before auto-merge"
         required: false
@@ -50,7 +49,6 @@ on:
         description: "evalops/.github ref to checkout for guard scripts"
         required: false
         type: string
-        default: c02a97ba9b92c6b2ac837aab77dc3becb77f301c
       settle_seconds:
         description: "Seconds to wait before checking review feedback so bot reviews can finish before auto-merge"
         required: false
@@ -69,7 +67,7 @@ jobs:
       - uses: actions/checkout@v5
         with:
           repository: evalops/.github
-          ref: ${{ inputs.guard_ref || 'c02a97ba9b92c6b2ac837aab77dc3becb77f301c' }}
+          ref: ${{ inputs.guard_ref || github.workflow_sha }}
           path: evalops-github
 
       - name: Let asynchronous review bots settle

test/check_pr_review_threads_test.rb

@@ -84,9 +84,11 @@ def test_detects_top_level_pr_comment_severity_markers
 
   def test_detects_top_level_review_body_severity_markers
     payload = payload_with(
+      head_ref_oid: "head-sha",
       reviews: [
         {
           "author" => { "login" => "reviewer" },
+          "commit" => { "oid" => "head-sha" },
           "state" => "COMMENTED",
           "body" => "**P1 Badge** paired public PR feedback is missing",
           "url" => "https://github.com/evalops/example/pull/1#pullrequestreview-1"
@@ -100,6 +102,32 @@ def test_detects_top_level_review_body_severity_markers
     assert_equal "p1", feedback.first.fetch(:severity)
   end
 
+  def test_skips_top_level_review_feedback_from_superseded_heads
+    payload = payload_with(
+      head_ref_oid: "new-head",
+      reviews: [
+        {
+          "author" => { "login" => "chatgpt-codex-connector[bot]" },
+          "commit" => { "oid" => "old-head" },
+          "state" => "COMMENTED",
+          "body" => "**P1 Badge** stale feedback already fixed on the latest head",
+          "url" => "https://github.com/evalops/example/pull/1#pullrequestreview-1"
+        },
+        {
+          "author" => { "login" => "reviewer" },
+          "commit" => { "oid" => "new-head" },
+          "state" => "COMMENTED",
+          "body" => "**High Severity** latest-head feedback still blocks",
+          "url" => "https://github.com/evalops/example/pull/1#pullrequestreview-2"
+        }
+      ]
+    )
+
+    feedback = EvalOpsReviewThreadGuard.blocking_feedback(payload, min_severity: "high")
+
+    assert_equal ["https://github.com/evalops/example/pull/1#pullrequestreview-2"], feedback.map { |item| item.fetch(:url) }
+  end
+
   def test_skips_informational_bot_pr_summaries
     payload = payload_with(
       comments: [
@@ -238,11 +266,12 @@ def test_fetch_connection_tail_uses_connection_specific_cursor
 
   private
 
-  def payload_with(comments: [], reviews: [], threads: [])
+  def payload_with(comments: [], reviews: [], threads: [], head_ref_oid: nil)
     {
       "data" => {
         "repository" => {
           "pullRequest" => {
+            "headRefOid" => head_ref_oid,
             "comments" => { "nodes" => comments },
             "reviews" => { "nodes" => reviews },
             "reviewThreads" => { "nodes" => threads }