From 602f94256c03710b9171b16b8ca14d08cd7b5334 Mon Sep 17 00:00:00 2001
From: illiabarbashov-sketch <illia.barbashov@cloudera.com>
Date: Mon, 15 Jun 2026 17:37:56 +0200
Subject: [PATCH] HIVE-29656: q.out regex fix

---
 .../apache/hadoop/hive/ql/QOutProcessor.java  | 25 ++++++++++++++++++-
 .../org/apache/hadoop/hive/ql/QTestUtil.java  | 14 ++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index b418c70538ca..2bd2bd83584e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -58,6 +58,7 @@ public class QOutProcessor {
 
   public static final String MASK_PATTERN = "#### A masked pattern was here ####";
   public static final String PARTIAL_MASK_PATTERN = "#### A PARTIAL masked pattern was here ####";
+  public static final String MASKED_VERTEX_KILLED_PATTERN = "[Masked Vertex killed due to OTHER_VERTEX_FAILURE]";
   private static final PatternReplacementPair MASK_STATS = new PatternReplacementPair(
       Pattern.compile(" Num rows: [1-9][0-9]* Data size: [1-9][0-9]*"),
       " Num rows: ###Masked### Data size: ###Masked###");
@@ -197,6 +198,7 @@ public void maskPatterns(String fname) throws Exception {
     out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
 
     boolean lastWasMasked = false;
+    boolean lastWasVertexKilled = false;
 
     while (null != (line = in.readLine())) {
       LineProcessingResult result = processLine(line);
@@ -209,10 +211,22 @@ public void maskPatterns(String fname) throws Exception {
           lastWasMasked = true;
           result.partialMaskWasMatched = false;
         }
+        lastWasVertexKilled = false;
+      } else if (result.line.equals(MASKED_VERTEX_KILLED_PATTERN)) {
+        // Deduplicate consecutive standalone vertex-killed lines — the number of sibling
+        // vertices still alive when the kill propagates is non-deterministic.
+        if (!lastWasVertexKilled) {
+          out.write(result.line);
+          out.write("\n");
+          lastWasVertexKilled = true;
+        }
+        lastWasMasked = false;
+        result.partialMaskWasMatched = false;
       } else {
         out.write(result.line);
         out.write("\n");
         lastWasMasked = false;
+        lastWasVertexKilled = false;
         result.partialMaskWasMatched = false;
       }
     }
@@ -350,7 +364,16 @@ private final static class PatternReplacementPair {
     // We do not want the test to fail because of this.
     ppm.add(new PatternReplacementPair(
         Pattern.compile("Vertex killed, vertexName=(.*?),.*\\[\\1\\] killed\\/failed due to:OTHER_VERTEX_FAILURE\\]"),
-        "[Masked Vertex killed due to OTHER_VERTEX_FAILURE]"));
+        MASKED_VERTEX_KILLED_PATTERN));
+
+    // Collapse multiple consecutive embedded [Masked Vertex killed] tokens on the same line
+    // (the long FAILED: summary line repeats one token per killed vertex).
+    ppm.add(new PatternReplacementPair(Pattern.compile("(\\Q" + MASKED_VERTEX_KILLED_PATTERN + "\\E){2,}"),
+        MASKED_VERTEX_KILLED_PATTERN));
+
+    // The number of vertices killed when a DAG fails is a scheduling race condition —
+    // depends on how many sibling vertices are still running at the moment the kill propagates.
+    ppm.add(new PatternReplacementPair(Pattern.compile("killedVertices:[0-9]+"), "killedVertices:#Masked#"));
 
     partialPlanMask = ppm.toArray(new PatternReplacementPair[ppm.size()]);
   }
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 447102f91c49..307f96baea84 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1020,7 +1020,19 @@ public QTestProcessExecResult checkCliDriverResults() throws Exception {
       qTestResultProcessor.overwriteResults(f.getPath(), outFileName);
       return QTestProcessExecResult.createWithoutOutput(0);
     } else {
-      return qTestResultProcessor.executeDiffCommand(f.getPath(), outFileName, false);
+      // Apply the same masking pipeline to a temporary copy of the reference file so that
+      // non-deterministic values (e.g. killedVertices) are normalized on both sides.
+      // This preserves backward compatibility with existing .q.out files that were written
+      // before the masking rules were introduced.
+      File maskedRef = new File(outFileName + ".masked_ref");
+      try {
+        FileUtils.copyFile(new File(outFileName), maskedRef);
+        qOutProcessor.maskPatterns(maskedRef.getPath());
+        return qTestResultProcessor.executeDiffCommand(f.getPath(), maskedRef.getPath(), false);
+      } finally {
+        maskedRef.delete();
+        new File(maskedRef.getPath() + ".orig").delete();
+      }
     }
   }