From 5a327f31910070b6732e38f0f9a5a6e90636cf1d Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Mon, 15 Jun 2026 18:43:28 +0100
Subject: [PATCH 01/15] feat(actions): add unified Playwright test health
 reporting for extension and mobile

---
 .../playwright-test-health-report/action.yml  |  91 ++++++
 .github/scripts/create-flaky-test-report.mjs  | 147 +++-------
 .../create-playwright-test-health-report.mjs  | 272 ++++++++++++++++++
 .github/scripts/lib/artifact-download.mjs     |  21 ++
 .github/scripts/lib/parse-playwright-json.mjs |  77 +++++
 .../scripts/lib/slack-test-health-blocks.mjs  |  18 ++
 .github/scripts/lib/summarize-test-health.mjs |  54 ++++
 .github/scripts/lib/workflow-runs.mjs         |  52 ++++
 8 files changed, 620 insertions(+), 112 deletions(-)
 create mode 100644 .github/actions/playwright-test-health-report/action.yml
 create mode 100644 .github/scripts/create-playwright-test-health-report.mjs
 create mode 100644 .github/scripts/lib/artifact-download.mjs
 create mode 100644 .github/scripts/lib/parse-playwright-json.mjs
 create mode 100644 .github/scripts/lib/slack-test-health-blocks.mjs
 create mode 100644 .github/scripts/lib/summarize-test-health.mjs
 create mode 100644 .github/scripts/lib/workflow-runs.mjs

diff --git a/.github/actions/playwright-test-health-report/action.yml b/.github/actions/playwright-test-health-report/action.yml
new file mode 100644
index 00000000..65b0d7d6
--- /dev/null
+++ b/.github/actions/playwright-test-health-report/action.yml
@@ -0,0 +1,91 @@
+name: Playwright Test Health Report
+description: >-
+  Aggregates Playwright JSON test reports from GitHub Actions artifacts,
+  classifies flaky and broken tests, and posts a summary to Slack.
+
+inputs:
+  repository:
+    description: Repository name (e.g. metamask-mobile, metamask-extension)
+    required: true
+  workflow-ids:
+    description: >-
+      Comma-separated workflow files to analyze (e.g. ci.yml or main.yml).
+      Runs from all listed workflows are merged into one report.
+    required: true
+  github-token:
+    description: GitHub token with repo and actions:read
+    required: true
+  slack-webhook:
+    description: Slack incoming webhook URL
+    required: true
+  owner:
+    description: GitHub org/user. Defaults to MetaMask.
+    required: false
+    default: MetaMask
+  branch:
+    description: Branch to filter workflow runs
+    required: false
+    default: main
+  lookback-days:
+    description: Number of days to look back for workflow runs
+    required: false
+    default: '1'
+  artifact-name-prefix:
+    description: >-
+      Only download artifacts whose name starts with this prefix.
+      Example: playwright-json-report
+    required: false
+    default: playwright-json-report
+  results-file-pattern:
+    description: >-
+      File name prefix inside artifact zip to parse as JSON.
+      Example: playwright-report (matches playwright-report.json,
+      playwright-report-1.json, etc.)
+    required: false
+    default: playwright-report
+  top-n:
+    description: Maximum number of tests to include in Slack report
+    required: false
+    default: '10'
+  report-title:
+    description: Slack header title override
+    required: false
+    default: Playwright Test Health Report
+  github-tools-repository:
+    required: false
+    default: ${{ github.action_repository }}
+  github-tools-ref:
+    required: false
+    default: ${{ github.action_ref }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+      with:
+        repository: ${{ inputs.github-tools-repository }}
+        ref: ${{ inputs.github-tools-ref }}
+        path: ./github-tools
+    - uses: actions/setup-node@v6
+      with:
+        node-version-file: ./github-tools/.nvmrc
+        cache: yarn
+        cache-dependency-path: ./github-tools/yarn.lock
+    - shell: bash
+      working-directory: ./github-tools
+      run: corepack enable && yarn --immutable
+    - shell: bash
+      working-directory: ./github-tools
+      env:
+        OWNER: ${{ inputs.owner }}
+        REPOSITORY: ${{ inputs.repository }}
+        WORKFLOW_IDS: ${{ inputs.workflow-ids }}
+        BRANCH: ${{ inputs.branch }}
+        LOOKBACK_DAYS: ${{ inputs.lookback-days }}
+        ARTIFACT_NAME_PREFIX: ${{ inputs.artifact-name-prefix }}
+        RESULTS_FILE_PATTERN: ${{ inputs.results-file-pattern }}
+        TOP_N: ${{ inputs.top-n }}
+        REPORT_TITLE: ${{ inputs.report-title }}
+        GITHUB_TOKEN: ${{ inputs.github-token }}
+        SLACK_WEBHOOK: ${{ inputs.slack-webhook }}
+      run: node .github/scripts/create-playwright-test-health-report.mjs
diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 7aab7582..4dec9824 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -3,8 +3,9 @@
 // Based on the original script done by @itsyoboieltr on Extension repo
 
 import { Octokit } from '@octokit/rest';
-import unzipper from 'unzipper';
-import { IncomingWebhook } from '@slack/webhook';
+import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
+import { sendSlackBatched, truncateError } from './lib/slack-test-health-blocks.mjs';
+import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
 
 const githubToken = process.env.GITHUB_TOKEN;
 if (!githubToken) throw new Error('Missing GITHUB_TOKEN env var');
@@ -23,92 +24,6 @@ const env = {
     : ['test-e2e-android-json-report', 'test-e2e-ios-json-report', 'test-e2e-chrome-report', 'test-e2e-firefox-report'],
 };
 
-function getDateRange() {
-  const today = new Date();
-  const daysAgo = new Date(today.getTime() - (env.LOOKBACK_DAYS * 24 * 60 * 60 * 1000));
-
-  const fromDisplay = daysAgo.toLocaleDateString('en-US', {
-    month: 'short',
-    day: 'numeric',
-    hour: 'numeric',
-    minute: '2-digit'
-  });
-
-  const toDisplay = today.toLocaleDateString('en-US', {
-    month: 'short',
-    day: 'numeric',
-    hour: 'numeric',
-    minute: '2-digit'
-  });
-
-  return {
-    from: daysAgo.toISOString(),
-    to: today.toISOString(),
-    display: `${fromDisplay} - ${toDisplay}`
-  };
-}
-
-async function getWorkflowRuns(github, from, to) {
-  try {
-    const runs = await github.paginate(
-      github.rest.actions.listWorkflowRuns,
-      {
-        owner: env.OWNER,
-        repo: env.REPOSITORY,
-        workflow_id: env.WORKFLOW_ID,
-        branch: env.BRANCH,
-        created: `${from}..${to}`,
-        per_page: 100,
-      }
-    );
-
-    // Filter to only completed runs from push or schedule events (excludes fork PRs)
-    const completedRuns = runs.filter(run =>
-      run.status === 'completed' &&
-      (run.event === 'push' || run.event === 'schedule')
-    );
-
-    // Sort by created date (newest first)
-    completedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
-
-    return completedRuns;
-  } catch (error) {
-    if (error.status === 404) {
-      throw new Error(`Workflow '${env.WORKFLOW_ID}' not found in ${env.OWNER}/${env.REPOSITORY}`);
-    }
-    throw error;
-  }
-}
-
-async function downloadArtifact(github, artifact) {
-  try {
-    const response = await github.rest.actions.downloadArtifact({
-      owner: env.OWNER,
-      repo: env.REPOSITORY,
-      artifact_id: artifact.id,
-      archive_format: 'zip',
-    });
-
-    const buffer = Buffer.from(response.data);
-    const zip = await unzipper.Open.buffer(buffer);
-
-    const testFile = zip.files.find(file => file.path.startsWith(env.TEST_RESULTS_FILE_PATTERN));
-    if (!testFile) {
-      console.log(`  ⚠️  No ${env.TEST_RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
-      return null;
-    }
-
-    const content = await testFile.buffer();
-    const data = JSON.parse(content.toString());
-
-    console.log(`   Parsed ${artifact.name} (${data.length} top testSuites)`);
-    return data;
-  } catch (error) {
-    console.log(`  ❌ Failed to download ${artifact.name}: ${error.message}`);
-    return null;
-  }
-}
-
 async function downloadTestArtifacts(github, runs) {
   const allTestData = [];
 
@@ -135,9 +50,25 @@ async function downloadTestArtifacts(github, runs) {
       }
 
       for (const artifact of testArtifacts) {
-        const testData = await downloadArtifact(github, artifact);
-        if (testData) {
+        try {
+          const zip = await downloadArtifactZip(github, {
+            owner: env.OWNER,
+            repo: env.REPOSITORY,
+            artifactId: artifact.id,
+          });
+          const testFiles = findFilesInZip(zip, env.TEST_RESULTS_FILE_PATTERN);
+          const testFile = testFiles[0];
+          if (!testFile) {
+            console.log(`  ⚠️  No ${env.TEST_RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
+            continue;
+          }
+
+          const content = await testFile.buffer();
+          const testData = JSON.parse(content.toString());
+          console.log(`   Parsed ${artifact.name} (${testData.length} top testSuites)`);
           allTestData.push(...testData);
+        } catch (error) {
+          console.log(`  ❌ Failed to download ${artifact.name}: ${error.message}`);
         }
       }
     } catch (error) {
@@ -333,15 +264,8 @@ async function sendSlackReport(summary, dateDisplay, workflowCount, failedCount)
 
   console.log('\n📤 Sending report to Slack...');
   try {
-    const webhook = new IncomingWebhook(env.SLACK_WEBHOOK_FLAKY_TESTS);
     const blocks = createSlackBlocks(summary, dateDisplay, workflowCount, failedCount);
-
-    // Slack has a limit of 50 blocks per message
-    const BATCH_SIZE = 50;
-    for (let i = 0; i < blocks.length; i += BATCH_SIZE) {
-      const batch = blocks.slice(i, i + BATCH_SIZE);
-      await webhook.send({ blocks: batch });
-    }
+    await sendSlackBatched(env.SLACK_WEBHOOK_FLAKY_TESTS, blocks);
 
     console.log('✅ Report sent to Slack successfully');
   } catch (slackError) {
@@ -439,13 +363,12 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount
       // Error message (if exists)
       const error = test.lastRealFailureError;
       if (error) {
-        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
         blocks.push({
           type: 'rich_text',
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `  ${errorPreview.replace(/\n/g, ' ')}`, style: { italic: true } }
+              { type: 'text', text: `  ${truncateError(error).replace(/\n/g, ' ')}`, style: { italic: true } }
             ]
           }]
         });
@@ -515,13 +438,12 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount
       // Error message (if exists)
       const error = test.flakyFailureError;
       if (error) {
-        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
         blocks.push({
           type: 'rich_text',
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}`, style: { italic: true } }
+              { type: 'text', text: `     ${truncateError(error).replace(/\n/g, ' ')}`, style: { italic: true } }
             ]
           }]
         });
@@ -570,10 +492,7 @@ function displayResults(summary, dateDisplay) {
 
       // Show error for real failures
       if (test.lastRealFailureError) {
-        const errorPreview = test.lastRealFailureError.length > 100
-          ? test.lastRealFailureError.substring(0, 100) + '...'
-          : test.lastRealFailureError;
-        console.log(`    💥 Error: ${errorPreview.replace(/\n/g, ' ')}`);
+        console.log(`    💥 Error: ${truncateError(test.lastRealFailureError, 100).replace(/\n/g, ' ')}`);
       }
     } else {
       // Flaky tests (failed initially but eventually passed)
@@ -587,10 +506,7 @@ function displayResults(summary, dateDisplay) {
 
       // Show error from initial failure
       if (test.flakyFailureError) {
-        const errorPreview = test.flakyFailureError.length > 100
-          ? test.flakyFailureError.substring(0, 100) + '...'
-          : test.flakyFailureError;
-        console.log(`    💥 Initial error: ${errorPreview.replace(/\n/g, ' ')}`);
+        console.log(`    💥 Initial error: ${truncateError(test.flakyFailureError, 100).replace(/\n/g, ' ')}`);
       }
     }
 
@@ -607,12 +523,19 @@ async function main() {
 
   console.log('🧪🧐 Flaky Test Report\n');
 
-  const dateRange = getDateRange();
+  const dateRange = getDateRange(env.LOOKBACK_DAYS);
   console.log(`Time range: ${dateRange.from} to ${dateRange.to}\n`);
 
   try {
     console.log('Fetching workflow runs...');
-    const workflowRuns = await getWorkflowRuns(github, dateRange.from, dateRange.to);
+    const workflowRuns = await getWorkflowRuns(github, {
+      owner: env.OWNER,
+      repo: env.REPOSITORY,
+      workflowId: env.WORKFLOW_ID,
+      branch: env.BRANCH,
+      from: dateRange.from,
+      to: dateRange.to,
+    });
 
     if (workflowRuns.length === 0) {
       console.log('⚠️ No workflow runs found.');
diff --git a/.github/scripts/create-playwright-test-health-report.mjs b/.github/scripts/create-playwright-test-health-report.mjs
new file mode 100644
index 00000000..386daa94
--- /dev/null
+++ b/.github/scripts/create-playwright-test-health-report.mjs
@@ -0,0 +1,272 @@
+#!/usr/bin/env node
+
+import { Octokit } from '@octokit/rest';
+import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
+import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs';
+import { sendSlackBatched, truncateError } from './lib/slack-test-health-blocks.mjs';
+import { summarizeTestHealth } from './lib/summarize-test-health.mjs';
+import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
+
+const githubToken = process.env.GITHUB_TOKEN;
+if (!githubToken) {
+  throw new Error('Missing GITHUB_TOKEN env var');
+}
+
+const env = {
+  OWNER: process.env.OWNER || 'MetaMask',
+  REPOSITORY: process.env.REPOSITORY,
+  WORKFLOW_IDS: process.env.WORKFLOW_IDS,
+  BRANCH: process.env.BRANCH || 'main',
+  LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
+  ARTIFACT_NAME_PREFIX: process.env.ARTIFACT_NAME_PREFIX || 'playwright-json-report',
+  RESULTS_FILE_PATTERN: process.env.RESULTS_FILE_PATTERN || 'playwright-report',
+  TOP_N: parseInt(process.env.TOP_N ?? '10'),
+  REPORT_TITLE: process.env.REPORT_TITLE || 'Playwright Test Health Report',
+  SLACK_WEBHOOK: process.env.SLACK_WEBHOOK || '',
+  GITHUB_TOKEN: githubToken,
+};
+
+if (!env.REPOSITORY) {
+  throw new Error('Missing REPOSITORY env var');
+}
+if (!env.WORKFLOW_IDS) {
+  throw new Error('Missing WORKFLOW_IDS env var');
+}
+
+function getWorkflowIds() {
+  return env.WORKFLOW_IDS.split(',')
+    .map(value => value.trim())
+    .filter(Boolean);
+}
+
+async function getMergedWorkflowRuns(github, dateRange) {
+  const workflowIds = getWorkflowIds();
+  const runs = [];
+
+  for (const workflowId of workflowIds) {
+    const workflowRuns = await getWorkflowRuns(github, {
+      owner: env.OWNER,
+      repo: env.REPOSITORY,
+      workflowId,
+      branch: env.BRANCH,
+      from: dateRange.from,
+      to: dateRange.to,
+    });
+    runs.push(...workflowRuns);
+  }
+
+  const dedupedRuns = Array.from(new Map(runs.map(run => [run.id, run])).values());
+  dedupedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+  return dedupedRuns;
+}
+
+async function collectFindings(github, runs) {
+  const findings = [];
+  let matchingArtifacts = 0;
+
+  for (const [index, run] of runs.entries()) {
+    console.log(`📦 Processing run ${index + 1}/${runs.length}: ${run.id}`);
+
+    const artifacts = await github.paginate(
+      github.rest.actions.listWorkflowRunArtifacts,
+      {
+        owner: env.OWNER,
+        repo: env.REPOSITORY,
+        run_id: run.id,
+      },
+    );
+
+    const matching = artifacts.filter(artifact => artifact.name.startsWith(env.ARTIFACT_NAME_PREFIX));
+    matchingArtifacts += matching.length;
+
+    if (matching.length === 0) {
+      console.log(`  ⚠️  No matching artifacts found for run ${run.id}`);
+      continue;
+    }
+
+    for (const artifact of matching) {
+      try {
+        const zip = await downloadArtifactZip(github, {
+          owner: env.OWNER,
+          repo: env.REPOSITORY,
+          artifactId: artifact.id,
+        });
+        const jsonFiles = findFilesInZip(zip, env.RESULTS_FILE_PATTERN);
+
+        if (jsonFiles.length === 0) {
+          console.log(`  ⚠️  No ${env.RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
+          continue;
+        }
+
+        for (const file of jsonFiles) {
+          try {
+            const content = await file.buffer();
+            const report = JSON.parse(content.toString());
+            findings.push(
+              ...parsePlaywrightJsonReport(report, {
+                runId: run.id,
+                runUrl: run.html_url || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${run.id}`,
+                date: run.created_at,
+              }),
+            );
+          } catch (error) {
+            console.log(`  ❌ Invalid JSON in ${artifact.name}/${file.path}: ${error.message}`);
+          }
+        }
+      } catch (error) {
+        console.log(`  ❌ Failed to process artifact ${artifact.name}: ${error.message}`);
+      }
+    }
+  }
+
+  return { findings, matchingArtifacts };
+}
+
+function createSlackBlocks(summary, dateDisplay, metadata) {
+  const topItems = summary.slice(0, env.TOP_N);
+  const broken = topItems.filter(item => item.brokenCount > 0);
+  const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
+
+  const blocks = [
+    {
+      type: 'header',
+      text: {
+        type: 'plain_text',
+        text: `${env.REPORT_TITLE} — Top ${env.TOP_N}`,
+        emoji: true,
+      },
+    },
+    {
+      type: 'context',
+      elements: [
+        {
+          type: 'mrkdwn',
+          text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Workflows: ${metadata.workflowsScanned.join(', ')} | Failed CI Runs: ${metadata.failedRunCount}/${metadata.workflowCount} from ${env.BRANCH}`,
+        },
+      ],
+    },
+    { type: 'divider' },
+  ];
+
+  if (topItems.length === 0) {
+    blocks.push({
+      type: 'section',
+      text: {
+        type: 'mrkdwn',
+        text: 'No flaky or broken tests found ✅',
+      },
+    });
+    return blocks;
+  }
+
+  if (broken.length > 0) {
+    blocks.push({
+      type: 'section',
+      text: { type: 'mrkdwn', text: '*❌ Broken (failed all retries)*' },
+    });
+
+    broken.forEach((test, index) => {
+      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
+      const runUrl = test.lastBrokenRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastBrokenRunId}`;
+      blocks.push({
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — broken: *${test.brokenCount}*, flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
+        },
+      });
+      blocks.push({
+        type: 'section',
+        text: { type: 'mrkdwn', text: `_${truncateError(test.lastBrokenError)}_` },
+      });
+    });
+  }
+
+  if (broken.length > 0 && flaky.length > 0) {
+    blocks.push({ type: 'divider' });
+  }
+
+  if (flaky.length > 0) {
+    blocks.push({
+      type: 'section',
+      text: { type: 'mrkdwn', text: '*🟡 Flaky (passed on retry)*' },
+    });
+
+    flaky.forEach((test, index) => {
+      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
+      const runUrl = test.lastFlakyRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastFlakyRunId}`;
+      blocks.push({
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
+        },
+      });
+      blocks.push({
+        type: 'section',
+        text: { type: 'mrkdwn', text: `_${truncateError(test.lastFlakyError)}_` },
+      });
+    });
+  }
+
+  return blocks;
+}
+
+async function sendSlackReport(summary, dateDisplay, metadata) {
+  if (!env.SLACK_WEBHOOK || !env.SLACK_WEBHOOK.startsWith('https://')) {
+    console.log('Skipping Slack notification');
+    return;
+  }
+
+  const blocks = createSlackBlocks(summary, dateDisplay, metadata);
+  await sendSlackBatched(env.SLACK_WEBHOOK, blocks);
+  console.log('✅ Report sent to Slack successfully');
+}
+
+async function main() {
+  const github = new Octokit({ auth: env.GITHUB_TOKEN });
+  const dateRange = getDateRange(env.LOOKBACK_DAYS);
+  const workflowsScanned = getWorkflowIds();
+
+  console.log('🧪 Playwright Test Health Report\n');
+  console.log(`Time range: ${dateRange.from} to ${dateRange.to}`);
+  console.log(`Workflows: ${workflowsScanned.join(', ')}\n`);
+
+  try {
+    const workflowRuns = await getMergedWorkflowRuns(github, dateRange);
+
+    if (workflowRuns.length === 0) {
+      console.log('⚠️ No workflow runs found in lookback window.');
+      return;
+    }
+
+    const failedRunCount = workflowRuns.filter(run => run.conclusion === 'failure').length;
+    const { findings, matchingArtifacts } = await collectFindings(github, workflowRuns);
+
+    if (matchingArtifacts === 0) {
+      console.log('⚠️ No matching artifacts found.');
+      return;
+    }
+
+    const summary = summarizeTestHealth(findings);
+    await sendSlackReport(summary, dateRange.display, {
+      workflowCount: workflowRuns.length,
+      failedRunCount,
+      workflowsScanned,
+    });
+  } catch (error) {
+    console.error('❌ Error:', error.message);
+    if (error.status === 401) {
+      console.log('\n💡 GitHub token is unauthorized. Ensure it has repo and actions:read permissions.');
+    }
+    if (error.status === 404) {
+      console.log('\n💡 One or more workflows were not found. Check WORKFLOW_IDS values.');
+    }
+    process.exit(1);
+  }
+}
+
+main().catch(error => {
+  console.error('\n❌ Unexpected error:', error);
+  process.exit(1);
+});
diff --git a/.github/scripts/lib/artifact-download.mjs b/.github/scripts/lib/artifact-download.mjs
new file mode 100644
index 00000000..9e11987a
--- /dev/null
+++ b/.github/scripts/lib/artifact-download.mjs
@@ -0,0 +1,21 @@
+import path from 'path';
+import unzipper from 'unzipper';
+
+export async function downloadArtifactZip(github, { owner, repo, artifactId }) {
+  const response = await github.rest.actions.downloadArtifact({
+    owner,
+    repo,
+    artifact_id: artifactId,
+    archive_format: 'zip',
+  });
+
+  const buffer = Buffer.from(response.data);
+  return unzipper.Open.buffer(buffer);
+}
+
+export function findFilesInZip(zip, filePattern) {
+  return zip.files.filter(file => {
+    const fileName = path.basename(file.path);
+    return file.path.startsWith(filePattern) || fileName.startsWith(filePattern);
+  });
+}
diff --git a/.github/scripts/lib/parse-playwright-json.mjs b/.github/scripts/lib/parse-playwright-json.mjs
new file mode 100644
index 00000000..bbcecdce
--- /dev/null
+++ b/.github/scripts/lib/parse-playwright-json.mjs
@@ -0,0 +1,77 @@
+function isFailureResult(result) {
+  return result.status === 'failed' || result.status === 'timedOut';
+}
+
+function classifyTest(test) {
+  if (test.status === 'unexpected') {
+    return 'broken';
+  }
+  if (test.status === 'flaky') {
+    return 'flaky';
+  }
+  if (test.status === 'expected' || test.status === 'skipped') {
+    return null;
+  }
+
+  const results = Array.isArray(test.results) ? test.results : [];
+  if (results.length <= 1) {
+    return null;
+  }
+
+  const lastResult = results[results.length - 1];
+  const anyFailed = results.some(isFailureResult);
+  if (!anyFailed) {
+    return null;
+  }
+
+  return lastResult?.status === 'passed' ? 'flaky' : 'broken';
+}
+
+function extractFirstFailureError(test) {
+  const results = Array.isArray(test.results) ? test.results : [];
+  const firstFailure = results.find(isFailureResult);
+  if (!firstFailure) {
+    return 'No error details';
+  }
+  return firstFailure?.error?.message ?? firstFailure?.errors?.[0]?.message ?? 'No error details';
+}
+
+function walkSuites(suites, currentFile, findings, metadata) {
+  for (const suite of suites ?? []) {
+    const suiteFile = suite.file || currentFile;
+
+    for (const spec of suite.specs ?? []) {
+      for (const test of spec.tests ?? []) {
+        const classification = classifyTest(test);
+        if (!classification) {
+          continue;
+        }
+
+        const projectName = test.projectName || 'default';
+        const key = `${projectName}::${spec.file || suiteFile || 'unknown-file'}::${test.title}`;
+        const results = Array.isArray(test.results) ? test.results : [];
+
+        findings.push({
+          key,
+          name: test.title,
+          path: spec.file || suiteFile || 'unknown-file',
+          projectName,
+          classification,
+          retries: Math.max(0, results.length - 1),
+          error: extractFirstFailureError(test),
+          runId: metadata.runId,
+          runUrl: metadata.runUrl,
+          date: new Date(metadata.date),
+        });
+      }
+    }
+
+    walkSuites(suite.suites, suiteFile, findings, metadata);
+  }
+}
+
+export function parsePlaywrightJsonReport(report, metadata) {
+  const findings = [];
+  walkSuites(report?.suites ?? [], undefined, findings, metadata);
+  return findings;
+}
diff --git a/.github/scripts/lib/slack-test-health-blocks.mjs b/.github/scripts/lib/slack-test-health-blocks.mjs
new file mode 100644
index 00000000..ccd7b16a
--- /dev/null
+++ b/.github/scripts/lib/slack-test-health-blocks.mjs
@@ -0,0 +1,18 @@
+import { IncomingWebhook } from '@slack/webhook';
+
+export function truncateError(message, maxLength = 150) {
+  if (!message) {
+    return 'No error details';
+  }
+  return message.length > maxLength ? `${message.substring(0, maxLength)}...` : message;
+}
+
+export async function sendSlackBatched(webhookUrl, blocks) {
+  const webhook = new IncomingWebhook(webhookUrl);
+  const batchSize = 50;
+
+  for (let i = 0; i < blocks.length; i += batchSize) {
+    const batch = blocks.slice(i, i + batchSize);
+    await webhook.send({ blocks: batch });
+  }
+}
diff --git a/.github/scripts/lib/summarize-test-health.mjs b/.github/scripts/lib/summarize-test-health.mjs
new file mode 100644
index 00000000..1c90f343
--- /dev/null
+++ b/.github/scripts/lib/summarize-test-health.mjs
@@ -0,0 +1,54 @@
+export function summarizeTestHealth(findings) {
+  const summary = new Map();
+
+  for (const finding of findings) {
+    const existing = summary.get(finding.key);
+
+    if (existing) {
+      if (finding.classification === 'broken') {
+        existing.brokenCount += 1;
+        existing.lastBrokenRunId = finding.runId;
+        existing.lastBrokenRunUrl = finding.runUrl;
+        existing.lastBrokenError = finding.error;
+      } else if (finding.classification === 'flaky') {
+        existing.flakyCount += 1;
+        existing.lastFlakyRunId = finding.runId;
+        existing.lastFlakyRunUrl = finding.runUrl;
+        existing.lastFlakyError = finding.error;
+      }
+
+      existing.totalRetries += finding.retries;
+      if (finding.date > existing.lastSeen) {
+        existing.lastSeen = finding.date;
+      }
+      continue;
+    }
+
+    summary.set(finding.key, {
+      key: finding.key,
+      name: finding.name,
+      path: finding.path,
+      projectName: finding.projectName,
+      brokenCount: finding.classification === 'broken' ? 1 : 0,
+      flakyCount: finding.classification === 'flaky' ? 1 : 0,
+      totalRetries: finding.retries,
+      lastSeen: finding.date,
+      lastBrokenRunId: finding.classification === 'broken' ? finding.runId : undefined,
+      lastBrokenRunUrl: finding.classification === 'broken' ? finding.runUrl : undefined,
+      lastBrokenError: finding.classification === 'broken' ? finding.error : undefined,
+      lastFlakyRunId: finding.classification === 'flaky' ? finding.runId : undefined,
+      lastFlakyRunUrl: finding.classification === 'flaky' ? finding.runUrl : undefined,
+      lastFlakyError: finding.classification === 'flaky' ? finding.error : undefined,
+    });
+  }
+
+  return Array.from(summary.values()).sort((a, b) => {
+    if (a.brokenCount !== b.brokenCount) {
+      return b.brokenCount - a.brokenCount;
+    }
+    if (a.flakyCount !== b.flakyCount) {
+      return b.flakyCount - a.flakyCount;
+    }
+    return b.totalRetries - a.totalRetries;
+  });
+}
diff --git a/.github/scripts/lib/workflow-runs.mjs b/.github/scripts/lib/workflow-runs.mjs
new file mode 100644
index 00000000..2536f792
--- /dev/null
+++ b/.github/scripts/lib/workflow-runs.mjs
@@ -0,0 +1,52 @@
+export function getDateRange(lookbackDays = 1) {
+  const today = new Date();
+  const daysAgo = new Date(today.getTime() - (lookbackDays * 24 * 60 * 60 * 1000));
+
+  const fromDisplay = daysAgo.toLocaleDateString('en-US', {
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit',
+  });
+
+  const toDisplay = today.toLocaleDateString('en-US', {
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit',
+  });
+
+  return {
+    from: daysAgo.toISOString(),
+    to: today.toISOString(),
+    display: `${fromDisplay} - ${toDisplay}`,
+  };
+}
+
+export async function getWorkflowRuns(github, { owner, repo, workflowId, branch, from, to }) {
+  try {
+    const runs = await github.paginate(
+      github.rest.actions.listWorkflowRuns,
+      {
+        owner,
+        repo,
+        workflow_id: workflowId,
+        branch,
+        created: `${from}..${to}`,
+        per_page: 100,
+      },
+    );
+
+    const completedRuns = runs.filter(
+      run => run.status === 'completed' && (run.event === 'push' || run.event === 'schedule'),
+    );
+    completedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+    return completedRuns;
+  } catch (error) {
+    if (error.status === 404) {
+      throw new Error(`Workflow '${workflowId}' not found in ${owner}/${repo}`);
+    }
+    throw error;
+  }
+}

From 66ac7d75bccbe9e5a8af53bd7df5bae856bfbf09 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Mon, 15 Jun 2026 22:18:00 +0100
Subject: [PATCH 02/15] feat(parse-playwright-json): improve test key
 generation with clearer variable usage

---
 .github/scripts/lib/parse-playwright-json.mjs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/lib/parse-playwright-json.mjs b/.github/scripts/lib/parse-playwright-json.mjs
index bbcecdce..5b86de81 100644
--- a/.github/scripts/lib/parse-playwright-json.mjs
+++ b/.github/scripts/lib/parse-playwright-json.mjs
@@ -48,13 +48,15 @@ function walkSuites(suites, currentFile, findings, metadata) {
         }
 
         const projectName = test.projectName || 'default';
-        const key = `${projectName}::${spec.file || suiteFile || 'unknown-file'}::${test.title}`;
+        const specFile = spec.file || suiteFile || 'unknown-file';
+        const testTitle = test.title || spec.title || 'unknown-test';
+        const key = `${projectName}::${specFile}::${testTitle}`;
         const results = Array.isArray(test.results) ? test.results : [];
 
         findings.push({
           key,
-          name: test.title,
-          path: spec.file || suiteFile || 'unknown-file',
+          name: testTitle,
+          path: specFile,
           projectName,
           classification,
           retries: Math.max(0, results.length - 1),

From aaf10ac4b47e503ea2223c51e566757030ec8b7b Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Tue, 16 Jun 2026 13:24:56 +0100
Subject: [PATCH 03/15] move scripts to actions folder

---
 .../playwright-test-health-report/action.yml  |   2 +-
 .../create-playwright-test-health-report.mjs  |   0
 .../lib/artifact-download.mjs                 |   0
 .../lib/parse-playwright-json.mjs             |   0
 .../lib/slack-test-health-blocks.mjs          |   0
 .../lib/summarize-test-health.mjs             |   0
 .../lib/workflow-runs.mjs                     |   0
 .github/scripts/create-flaky-test-report.mjs  | 147 +++++++++++++-----
 8 files changed, 113 insertions(+), 36 deletions(-)
 rename .github/{scripts => actions/playwright-test-health-report}/create-playwright-test-health-report.mjs (100%)
 rename .github/{scripts => actions/playwright-test-health-report}/lib/artifact-download.mjs (100%)
 rename .github/{scripts => actions/playwright-test-health-report}/lib/parse-playwright-json.mjs (100%)
 rename .github/{scripts => actions/playwright-test-health-report}/lib/slack-test-health-blocks.mjs (100%)
 rename .github/{scripts => actions/playwright-test-health-report}/lib/summarize-test-health.mjs (100%)
 rename .github/{scripts => actions/playwright-test-health-report}/lib/workflow-runs.mjs (100%)

diff --git a/.github/actions/playwright-test-health-report/action.yml b/.github/actions/playwright-test-health-report/action.yml
index 65b0d7d6..d1fbf9cf 100644
--- a/.github/actions/playwright-test-health-report/action.yml
+++ b/.github/actions/playwright-test-health-report/action.yml
@@ -88,4 +88,4 @@ runs:
         REPORT_TITLE: ${{ inputs.report-title }}
         GITHUB_TOKEN: ${{ inputs.github-token }}
         SLACK_WEBHOOK: ${{ inputs.slack-webhook }}
-      run: node .github/scripts/create-playwright-test-health-report.mjs
+      run: node .github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
diff --git a/.github/scripts/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
similarity index 100%
rename from .github/scripts/create-playwright-test-health-report.mjs
rename to .github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
diff --git a/.github/scripts/lib/artifact-download.mjs b/.github/actions/playwright-test-health-report/lib/artifact-download.mjs
similarity index 100%
rename from .github/scripts/lib/artifact-download.mjs
rename to .github/actions/playwright-test-health-report/lib/artifact-download.mjs
diff --git a/.github/scripts/lib/parse-playwright-json.mjs b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
similarity index 100%
rename from .github/scripts/lib/parse-playwright-json.mjs
rename to .github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
diff --git a/.github/scripts/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
similarity index 100%
rename from .github/scripts/lib/slack-test-health-blocks.mjs
rename to .github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
diff --git a/.github/scripts/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
similarity index 100%
rename from .github/scripts/lib/summarize-test-health.mjs
rename to .github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
diff --git a/.github/scripts/lib/workflow-runs.mjs b/.github/actions/playwright-test-health-report/lib/workflow-runs.mjs
similarity index 100%
rename from .github/scripts/lib/workflow-runs.mjs
rename to .github/actions/playwright-test-health-report/lib/workflow-runs.mjs
diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 4dec9824..7aab7582 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -3,9 +3,8 @@
 // Based on the original script done by @itsyoboieltr on Extension repo
 
 import { Octokit } from '@octokit/rest';
-import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
-import { sendSlackBatched, truncateError } from './lib/slack-test-health-blocks.mjs';
-import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
+import unzipper from 'unzipper';
+import { IncomingWebhook } from '@slack/webhook';
 
 const githubToken = process.env.GITHUB_TOKEN;
 if (!githubToken) throw new Error('Missing GITHUB_TOKEN env var');
@@ -24,6 +23,92 @@ const env = {
     : ['test-e2e-android-json-report', 'test-e2e-ios-json-report', 'test-e2e-chrome-report', 'test-e2e-firefox-report'],
 };
 
+function getDateRange() {
+  const today = new Date();
+  const daysAgo = new Date(today.getTime() - (env.LOOKBACK_DAYS * 24 * 60 * 60 * 1000));
+
+  const fromDisplay = daysAgo.toLocaleDateString('en-US', {
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit'
+  });
+
+  const toDisplay = today.toLocaleDateString('en-US', {
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit'
+  });
+
+  return {
+    from: daysAgo.toISOString(),
+    to: today.toISOString(),
+    display: `${fromDisplay} - ${toDisplay}`
+  };
+}
+
+async function getWorkflowRuns(github, from, to) {
+  try {
+    const runs = await github.paginate(
+      github.rest.actions.listWorkflowRuns,
+      {
+        owner: env.OWNER,
+        repo: env.REPOSITORY,
+        workflow_id: env.WORKFLOW_ID,
+        branch: env.BRANCH,
+        created: `${from}..${to}`,
+        per_page: 100,
+      }
+    );
+
+    // Filter to only completed runs from push or schedule events (excludes fork PRs)
+    const completedRuns = runs.filter(run =>
+      run.status === 'completed' &&
+      (run.event === 'push' || run.event === 'schedule')
+    );
+
+    // Sort by created date (newest first)
+    completedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+    return completedRuns;
+  } catch (error) {
+    if (error.status === 404) {
+      throw new Error(`Workflow '${env.WORKFLOW_ID}' not found in ${env.OWNER}/${env.REPOSITORY}`);
+    }
+    throw error;
+  }
+}
+
+async function downloadArtifact(github, artifact) {
+  try {
+    const response = await github.rest.actions.downloadArtifact({
+      owner: env.OWNER,
+      repo: env.REPOSITORY,
+      artifact_id: artifact.id,
+      archive_format: 'zip',
+    });
+
+    const buffer = Buffer.from(response.data);
+    const zip = await unzipper.Open.buffer(buffer);
+
+    const testFile = zip.files.find(file => file.path.startsWith(env.TEST_RESULTS_FILE_PATTERN));
+    if (!testFile) {
+      console.log(`  ⚠️  No ${env.TEST_RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
+      return null;
+    }
+
+    const content = await testFile.buffer();
+    const data = JSON.parse(content.toString());
+
+    console.log(`   Parsed ${artifact.name} (${data.length} top testSuites)`);
+    return data;
+  } catch (error) {
+    console.log(`  ❌ Failed to download ${artifact.name}: ${error.message}`);
+    return null;
+  }
+}
+
 async function downloadTestArtifacts(github, runs) {
   const allTestData = [];
 
@@ -50,25 +135,9 @@ async function downloadTestArtifacts(github, runs) {
       }
 
       for (const artifact of testArtifacts) {
-        try {
-          const zip = await downloadArtifactZip(github, {
-            owner: env.OWNER,
-            repo: env.REPOSITORY,
-            artifactId: artifact.id,
-          });
-          const testFiles = findFilesInZip(zip, env.TEST_RESULTS_FILE_PATTERN);
-          const testFile = testFiles[0];
-          if (!testFile) {
-            console.log(`  ⚠️  No ${env.TEST_RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
-            continue;
-          }
-
-          const content = await testFile.buffer();
-          const testData = JSON.parse(content.toString());
-          console.log(`   Parsed ${artifact.name} (${testData.length} top testSuites)`);
+        const testData = await downloadArtifact(github, artifact);
+        if (testData) {
           allTestData.push(...testData);
-        } catch (error) {
-          console.log(`  ❌ Failed to download ${artifact.name}: ${error.message}`);
         }
       }
     } catch (error) {
@@ -264,8 +333,15 @@ async function sendSlackReport(summary, dateDisplay, workflowCount, failedCount)
 
   console.log('\n📤 Sending report to Slack...');
   try {
+    const webhook = new IncomingWebhook(env.SLACK_WEBHOOK_FLAKY_TESTS);
     const blocks = createSlackBlocks(summary, dateDisplay, workflowCount, failedCount);
-    await sendSlackBatched(env.SLACK_WEBHOOK_FLAKY_TESTS, blocks);
+
+    // Slack has a limit of 50 blocks per message
+    const BATCH_SIZE = 50;
+    for (let i = 0; i < blocks.length; i += BATCH_SIZE) {
+      const batch = blocks.slice(i, i + BATCH_SIZE);
+      await webhook.send({ blocks: batch });
+    }
 
     console.log('✅ Report sent to Slack successfully');
   } catch (slackError) {
@@ -363,12 +439,13 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount
       // Error message (if exists)
       const error = test.lastRealFailureError;
       if (error) {
+        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
         blocks.push({
           type: 'rich_text',
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `  ${truncateError(error).replace(/\n/g, ' ')}`, style: { italic: true } }
+              { type: 'text', text: `  ${errorPreview.replace(/\n/g, ' ')}`, style: { italic: true } }
             ]
           }]
         });
@@ -438,12 +515,13 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount
       // Error message (if exists)
       const error = test.flakyFailureError;
       if (error) {
+        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
         blocks.push({
           type: 'rich_text',
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `     ${truncateError(error).replace(/\n/g, ' ')}`, style: { italic: true } }
+              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}`, style: { italic: true } }
             ]
           }]
         });
@@ -492,7 +570,10 @@ function displayResults(summary, dateDisplay) {
 
       // Show error for real failures
       if (test.lastRealFailureError) {
-        console.log(`    💥 Error: ${truncateError(test.lastRealFailureError, 100).replace(/\n/g, ' ')}`);
+        const errorPreview = test.lastRealFailureError.length > 100
+          ? test.lastRealFailureError.substring(0, 100) + '...'
+          : test.lastRealFailureError;
+        console.log(`    💥 Error: ${errorPreview.replace(/\n/g, ' ')}`);
       }
     } else {
       // Flaky tests (failed initially but eventually passed)
@@ -506,7 +587,10 @@ function displayResults(summary, dateDisplay) {
 
       // Show error from initial failure
       if (test.flakyFailureError) {
-        console.log(`    💥 Initial error: ${truncateError(test.flakyFailureError, 100).replace(/\n/g, ' ')}`);
+        const errorPreview = test.flakyFailureError.length > 100
+          ? test.flakyFailureError.substring(0, 100) + '...'
+          : test.flakyFailureError;
+        console.log(`    💥 Initial error: ${errorPreview.replace(/\n/g, ' ')}`);
       }
     }
 
@@ -523,19 +607,12 @@ async function main() {
 
   console.log('🧪🧐 Flaky Test Report\n');
 
-  const dateRange = getDateRange(env.LOOKBACK_DAYS);
+  const dateRange = getDateRange();
   console.log(`Time range: ${dateRange.from} to ${dateRange.to}\n`);
 
   try {
     console.log('Fetching workflow runs...');
-    const workflowRuns = await getWorkflowRuns(github, {
-      owner: env.OWNER,
-      repo: env.REPOSITORY,
-      workflowId: env.WORKFLOW_ID,
-      branch: env.BRANCH,
-      from: dateRange.from,
-      to: dateRange.to,
-    });
+    const workflowRuns = await getWorkflowRuns(github, dateRange.from, dateRange.to);
 
     if (workflowRuns.length === 0) {
       console.log('⚠️ No workflow runs found.');

From 4bb5ba796bad03074f968bb417f5f6429bd08c02 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Tue, 16 Jun 2026 18:34:39 +0100
Subject: [PATCH 04/15] refactor playwright test health Slack blocks and broken
 classification

---
 .../create-playwright-test-health-report.mjs  | 103 ++--------
 .../lib/slack-test-health-blocks.mjs          | 184 +++++++++++++++++-
 .../lib/summarize-test-health.mjs             |  16 +-
 3 files changed, 206 insertions(+), 97 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
index 386daa94..92d378a5 100644
--- a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
+++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
@@ -3,7 +3,7 @@
 import { Octokit } from '@octokit/rest';
 import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
 import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs';
-import { sendSlackBatched, truncateError } from './lib/slack-test-health-blocks.mjs';
+import { createSlackBlocks, sendSlackBatched } from './lib/slack-test-health-blocks.mjs';
 import { summarizeTestHealth } from './lib/summarize-test-health.mjs';
 import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
 
@@ -122,103 +122,22 @@ async function collectFindings(github, runs) {
   return { findings, matchingArtifacts };
 }
 
-function createSlackBlocks(summary, dateDisplay, metadata) {
-  const topItems = summary.slice(0, env.TOP_N);
-  const broken = topItems.filter(item => item.brokenCount > 0);
-  const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
-
-  const blocks = [
-    {
-      type: 'header',
-      text: {
-        type: 'plain_text',
-        text: `${env.REPORT_TITLE} — Top ${env.TOP_N}`,
-        emoji: true,
-      },
-    },
-    {
-      type: 'context',
-      elements: [
-        {
-          type: 'mrkdwn',
-          text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Workflows: ${metadata.workflowsScanned.join(', ')} | Failed CI Runs: ${metadata.failedRunCount}/${metadata.workflowCount} from ${env.BRANCH}`,
-        },
-      ],
-    },
-    { type: 'divider' },
-  ];
-
-  if (topItems.length === 0) {
-    blocks.push({
-      type: 'section',
-      text: {
-        type: 'mrkdwn',
-        text: 'No flaky or broken tests found ✅',
-      },
-    });
-    return blocks;
-  }
-
-  if (broken.length > 0) {
-    blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: '*❌ Broken (failed all retries)*' },
-    });
-
-    broken.forEach((test, index) => {
-      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
-      const runUrl = test.lastBrokenRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastBrokenRunId}`;
-      blocks.push({
-        type: 'section',
-        text: {
-          type: 'mrkdwn',
-          text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — broken: *${test.brokenCount}*, flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
-        },
-      });
-      blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: `_${truncateError(test.lastBrokenError)}_` },
-      });
-    });
-  }
-
-  if (broken.length > 0 && flaky.length > 0) {
-    blocks.push({ type: 'divider' });
-  }
-
-  if (flaky.length > 0) {
-    blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: '*🟡 Flaky (passed on retry)*' },
-    });
-
-    flaky.forEach((test, index) => {
-      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
-      const runUrl = test.lastFlakyRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastFlakyRunId}`;
-      blocks.push({
-        type: 'section',
-        text: {
-          type: 'mrkdwn',
-          text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
-        },
-      });
-      blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: `_${truncateError(test.lastFlakyError)}_` },
-      });
-    });
-  }
-
-  return blocks;
-}
-
 async function sendSlackReport(summary, dateDisplay, metadata) {
   if (!env.SLACK_WEBHOOK || !env.SLACK_WEBHOOK.startsWith('https://')) {
     console.log('Skipping Slack notification');
     return;
   }
 
-  const blocks = createSlackBlocks(summary, dateDisplay, metadata);
+  const blocks = createSlackBlocks(summary, dateDisplay, {
+    owner: env.OWNER,
+    repository: env.REPOSITORY,
+    branch: env.BRANCH,
+    reportTitle: env.REPORT_TITLE,
+    topN: env.TOP_N,
+    workflowsScanned: metadata.workflowsScanned,
+    failedRunCount: metadata.failedRunCount,
+    workflowCount: metadata.workflowCount,
+  });
   await sendSlackBatched(env.SLACK_WEBHOOK, blocks);
   console.log('✅ Report sent to Slack successfully');
 }
diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index ccd7b16a..2d7051d4 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -1,10 +1,190 @@
 import { IncomingWebhook } from '@slack/webhook';
 
-export function truncateError(message, maxLength = 150) {
+export function normalizeErrorForSlack(message, maxLength = 120) {
   if (!message) {
     return 'No error details';
   }
-  return message.length > maxLength ? `${message.substring(0, maxLength)}...` : message;
+
+  const withoutEmojiShortcodes = String(message).replace(/:[a-z0-9_+\-]+:/gi, ' ');
+  const firstMeaningfulLine = withoutEmojiShortcodes
+    .split(/\r?\n/)
+    .map(line => line.trim())
+    .find(Boolean);
+  const normalized = (firstMeaningfulLine || withoutEmojiShortcodes)
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  if (!normalized) {
+    return 'No error details';
+  }
+
+  return normalized.length > maxLength ? `${normalized.substring(0, maxLength - 3)}...` : normalized;
+}
+
+export function truncateError(message, maxLength = 120) {
+  return normalizeErrorForSlack(message, maxLength);
+}
+
+export function createSlackBlocks(summary, dateDisplay, options) {
+  const {
+    owner,
+    repository,
+    branch,
+    reportTitle,
+    topN,
+    workflowsScanned,
+    failedRunCount,
+    workflowCount,
+  } = options;
+
+  const topItems = summary.slice(0, topN);
+  const broken = topItems.filter(item => item.brokenCount > 0);
+  const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
+
+  const blocks = [
+    {
+      type: 'header',
+      text: {
+        type: 'plain_text',
+        text: `${reportTitle} - Top ${topN}`,
+        emoji: true,
+      },
+    },
+    {
+      type: 'context',
+      elements: [
+        {
+          type: 'mrkdwn',
+          text:
+            `Period (UTC): ${dateDisplay} | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')} | ` +
+            `Failed CI Runs: ${failedRunCount}/${workflowCount} from ${branch}` +
+            `\nFound: ${broken.length} broken, ${flaky.length} flaky`,
+        },
+      ],
+    },
+    { type: 'divider' },
+  ];
+
+  if (topItems.length === 0) {
+    blocks.push({
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [{ type: 'text', text: 'No flaky or broken tests found ✅' }],
+        },
+      ],
+    });
+    return blocks;
+  }
+
+  if (broken.length > 0) {
+    blocks.push({
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [
+            { type: 'emoji', name: 'x' },
+            { type: 'text', text: ' ' },
+            { type: 'text', text: 'Broken', style: { bold: true } },
+          ],
+        },
+      ],
+    });
+
+    broken.forEach((test, index) => {
+      const globalIndex = index + 1;
+      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
+      const runUrl =
+        test.lastBrokenRunUrl ||
+        (test.lastBrokenRunId
+          ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastBrokenRunId}`
+          : null);
+      const elements = [
+        { type: 'text', text: `  ${globalIndex}. ` },
+        { type: 'link', url: fileUrl, text: test.name },
+        { type: 'text', text: ` (${test.projectName})` },
+        { type: 'text', text: ` failed ${test.brokenCount}x`, style: { bold: true } },
+      ];
+
+      if (runUrl) {
+        elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
+      }
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [{ type: 'rich_text_section', elements }],
+      });
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [{ type: 'text', text: `  ${truncateError(test.lastBrokenError)}`, style: { italic: true } }],
+          },
+        ],
+      });
+    });
+  }
+
+  if (broken.length > 0 && flaky.length > 0) {
+    blocks.push({ type: 'divider' });
+  }
+
+  if (flaky.length > 0) {
+    blocks.push({
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [
+            { type: 'emoji', name: 'large_yellow_circle' },
+            { type: 'text', text: ' ' },
+            { type: 'text', text: 'Flaky', style: { bold: true } },
+          ],
+        },
+      ],
+    });
+
+    flaky.forEach((test, index) => {
+      const globalIndex = broken.length + index + 1;
+      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
+      const runUrl =
+        test.lastFlakyRunUrl ||
+        (test.lastFlakyRunId
+          ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastFlakyRunId}`
+          : null);
+      const elements = [
+        { type: 'text', text: `  ${globalIndex}. ` },
+        { type: 'link', url: fileUrl, text: test.name },
+        { type: 'text', text: ` (${test.projectName})` },
+        { type: 'text', text: ` flaky ${test.flakyCount}x`, style: { bold: true } },
+      ];
+
+      if (runUrl) {
+        elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
+      }
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [{ type: 'rich_text_section', elements }],
+      });
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [{ type: 'text', text: `  ${truncateError(test.lastFlakyError)}`, style: { italic: true } }],
+          },
+        ],
+      });
+    });
+  }
+
+  return blocks;
 }
 
 export async function sendSlackBatched(webhookUrl, blocks) {
diff --git a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
index 1c90f343..1788250a 100644
--- a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
+++ b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
@@ -18,8 +18,9 @@ export function summarizeTestHealth(findings) {
       }
 
       existing.totalRetries += finding.retries;
-      if (finding.date > existing.lastSeen) {
+      if (finding.date >= existing.lastSeen) {
         existing.lastSeen = finding.date;
+        existing.latestClassification = finding.classification;
       }
       continue;
     }
@@ -33,6 +34,7 @@ export function summarizeTestHealth(findings) {
       flakyCount: finding.classification === 'flaky' ? 1 : 0,
       totalRetries: finding.retries,
       lastSeen: finding.date,
+      latestClassification: finding.classification,
       lastBrokenRunId: finding.classification === 'broken' ? finding.runId : undefined,
       lastBrokenRunUrl: finding.classification === 'broken' ? finding.runUrl : undefined,
       lastBrokenError: finding.classification === 'broken' ? finding.error : undefined,
@@ -42,7 +44,15 @@ export function summarizeTestHealth(findings) {
     });
   }
 
-  return Array.from(summary.values()).sort((a, b) => {
+  return Array.from(summary.values())
+    .map(item => {
+      const latestIsBroken = item.latestClassification === 'broken';
+      return {
+        ...item,
+        brokenCount: latestIsBroken ? item.brokenCount : 0,
+      };
+    })
+    .sort((a, b) => {
     if (a.brokenCount !== b.brokenCount) {
       return b.brokenCount - a.brokenCount;
     }
@@ -50,5 +60,5 @@ export function summarizeTestHealth(findings) {
       return b.flakyCount - a.flakyCount;
     }
     return b.totalRetries - a.totalRetries;
-  });
+    });
 }

From c36584b42d28ce342b1c98946c48568a619c1261 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 11:59:25 +0100
Subject: [PATCH 05/15] use plain text section blocks for test health Slack
 message

---
 .../lib/slack-test-health-blocks.mjs          | 87 +++++--------------
 1 file changed, 20 insertions(+), 67 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index 2d7051d4..74597302 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -67,30 +67,16 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   if (topItems.length === 0) {
     blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [{ type: 'text', text: 'No flaky or broken tests found ✅' }],
-        },
-      ],
+      type: 'section',
+      text: { type: 'mrkdwn', text: 'No flaky or broken tests found ✅' },
     });
     return blocks;
   }
 
   if (broken.length > 0) {
     blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [
-            { type: 'emoji', name: 'x' },
-            { type: 'text', text: ' ' },
-            { type: 'text', text: 'Broken', style: { bold: true } },
-          ],
-        },
-      ],
+      type: 'section',
+      text: { type: 'mrkdwn', text: '*❌ Broken*' },
     });
 
     broken.forEach((test, index) => {
@@ -101,30 +87,18 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         (test.lastBrokenRunId
           ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastBrokenRunId}`
           : null);
-      const elements = [
-        { type: 'text', text: `  ${globalIndex}. ` },
-        { type: 'link', url: fileUrl, text: test.name },
-        { type: 'text', text: ` (${test.projectName})` },
-        { type: 'text', text: ` failed ${test.brokenCount}x`, style: { bold: true } },
-      ];
-
-      if (runUrl) {
-        elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
-      }
+      const line =
+        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) *failed ${test.brokenCount}x*` +
+        (runUrl ? ` - <${runUrl}|run log>` : '');
 
       blocks.push({
-        type: 'rich_text',
-        elements: [{ type: 'rich_text_section', elements }],
+        type: 'section',
+        text: { type: 'mrkdwn', text: line },
       });
 
       blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [{ type: 'text', text: `  ${truncateError(test.lastBrokenError)}`, style: { italic: true } }],
-          },
-        ],
+        type: 'section',
+        text: { type: 'mrkdwn', text: `_${truncateError(test.lastBrokenError)}_` },
       });
     });
   }
@@ -135,17 +109,8 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   if (flaky.length > 0) {
     blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [
-            { type: 'emoji', name: 'large_yellow_circle' },
-            { type: 'text', text: ' ' },
-            { type: 'text', text: 'Flaky', style: { bold: true } },
-          ],
-        },
-      ],
+      type: 'section',
+      text: { type: 'mrkdwn', text: '*🟡 Flaky*' },
     });
 
     flaky.forEach((test, index) => {
@@ -156,30 +121,18 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         (test.lastFlakyRunId
           ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastFlakyRunId}`
           : null);
-      const elements = [
-        { type: 'text', text: `  ${globalIndex}. ` },
-        { type: 'link', url: fileUrl, text: test.name },
-        { type: 'text', text: ` (${test.projectName})` },
-        { type: 'text', text: ` flaky ${test.flakyCount}x`, style: { bold: true } },
-      ];
-
-      if (runUrl) {
-        elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
-      }
+      const line =
+        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) *flaky ${test.flakyCount}x*` +
+        (runUrl ? ` - <${runUrl}|run log>` : '');
 
       blocks.push({
-        type: 'rich_text',
-        elements: [{ type: 'rich_text_section', elements }],
+        type: 'section',
+        text: { type: 'mrkdwn', text: line },
       });
 
       blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [{ type: 'text', text: `  ${truncateError(test.lastFlakyError)}`, style: { italic: true } }],
-          },
-        ],
+        type: 'section',
+        text: { type: 'mrkdwn', text: `_${truncateError(test.lastFlakyError)}_` },
       });
     });
   }

From ebd999fdc5c57d5c492093c7bdc952c684e8ad66 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 12:07:01 +0100
Subject: [PATCH 06/15] improve test health reporting using latest observed
 test state

---
 .../lib/parse-playwright-json.mjs                |  5 ++++-
 .../lib/summarize-test-health.mjs                | 16 +++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
index 5b86de81..5c9ec5da 100644
--- a/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
+++ b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
@@ -3,13 +3,16 @@ function isFailureResult(result) {
 }
 
 function classifyTest(test) {
+  if (test.status === 'expected') {
+    return 'passed';
+  }
   if (test.status === 'unexpected') {
     return 'broken';
   }
   if (test.status === 'flaky') {
     return 'flaky';
   }
-  if (test.status === 'expected' || test.status === 'skipped') {
+  if (test.status === 'skipped') {
     return null;
   }
 
diff --git a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
index 1788250a..1fc27795 100644
--- a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
+++ b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
@@ -47,18 +47,20 @@ export function summarizeTestHealth(findings) {
   return Array.from(summary.values())
     .map(item => {
       const latestIsBroken = item.latestClassification === 'broken';
+      const latestIsFlaky = item.latestClassification === 'flaky';
       return {
         ...item,
         brokenCount: latestIsBroken ? item.brokenCount : 0,
+        flakyCount: latestIsFlaky ? item.flakyCount : 0,
       };
     })
     .sort((a, b) => {
-    if (a.brokenCount !== b.brokenCount) {
-      return b.brokenCount - a.brokenCount;
-    }
-    if (a.flakyCount !== b.flakyCount) {
-      return b.flakyCount - a.flakyCount;
-    }
-    return b.totalRetries - a.totalRetries;
+      if (a.brokenCount !== b.brokenCount) {
+        return b.brokenCount - a.brokenCount;
+      }
+      if (a.flakyCount !== b.flakyCount) {
+        return b.flakyCount - a.flakyCount;
+      }
+      return b.totalRetries - a.totalRetries;
     });
 }

From a68c15db9f62c043a54959496d9fd389af19a84c Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 12:25:43 +0100
Subject: [PATCH 07/15] add classification diagnostics logging to test health
 report

---
 .../create-playwright-test-health-report.mjs  | 26 +++++++++++++++++++
 .../lib/summarize-test-health.mjs             |  2 ++
 2 files changed, 28 insertions(+)

diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
index 92d378a5..d6741829 100644
--- a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
+++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
@@ -142,6 +142,31 @@ async function sendSlackReport(summary, dateDisplay, metadata) {
   console.log('✅ Report sent to Slack successfully');
 }
 
+function logClassificationDiagnostics(summary) {
+  const totalUniqueTests = summary.length;
+  const currentlyBroken = summary.filter(test => test.brokenCount > 0);
+  const currentlyFlaky = summary.filter(test => test.brokenCount === 0 && test.flakyCount > 0);
+  const latestPassed = summary.filter(test => test.latestClassification === 'passed');
+  const resolvedFromFailure = summary.filter(
+    test =>
+      test.latestClassification === 'passed' &&
+      ((test.historicalBrokenCount ?? 0) > 0 || (test.historicalFlakyCount ?? 0) > 0),
+  );
+
+  console.log('\n🧾 Classification diagnostics');
+  console.log(`  Unique tests observed: ${totalUniqueTests}`);
+  console.log(`  Latest state -> broken: ${currentlyBroken.length}, flaky: ${currentlyFlaky.length}, passed: ${latestPassed.length}`);
+  console.log(`  Resolved since earlier runs (had broken/flaky history, latest passed): ${resolvedFromFailure.length}`);
+
+  if (resolvedFromFailure.length > 0) {
+    const preview = resolvedFromFailure
+      .slice(0, 5)
+      .map(test => `${test.name} (${test.projectName})`)
+      .join('; ');
+    console.log(`  Sample resolved tests: ${preview}`);
+  }
+}
+
 async function main() {
   const github = new Octokit({ auth: env.GITHUB_TOKEN });
   const dateRange = getDateRange(env.LOOKBACK_DAYS);
@@ -168,6 +193,7 @@ async function main() {
     }
 
     const summary = summarizeTestHealth(findings);
+    logClassificationDiagnostics(summary);
     await sendSlackReport(summary, dateRange.display, {
       workflowCount: workflowRuns.length,
       failedRunCount,
diff --git a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
index 1fc27795..bd01ae75 100644
--- a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
+++ b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
@@ -50,6 +50,8 @@ export function summarizeTestHealth(findings) {
       const latestIsFlaky = item.latestClassification === 'flaky';
       return {
         ...item,
+        historicalBrokenCount: item.brokenCount,
+        historicalFlakyCount: item.flakyCount,
         brokenCount: latestIsBroken ? item.brokenCount : 0,
         flakyCount: latestIsFlaky ? item.flakyCount : 0,
       };

From 13059055a587a2a1f99e9d80f3afbd2077d2e3ec Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 12:29:18 +0100
Subject: [PATCH 08/15] add review section for now-passing tests with failure
 history

---
 .../lib/slack-test-health-blocks.mjs          | 45 +++++++++++++++++--
 1 file changed, 42 insertions(+), 3 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index 74597302..24522797 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -37,9 +37,22 @@ export function createSlackBlocks(summary, dateDisplay, options) {
     workflowCount,
   } = options;
 
-  const topItems = summary.slice(0, topN);
+  const relevantItems = summary.filter(
+    item =>
+      item.brokenCount > 0 ||
+      item.flakyCount > 0 ||
+      (item.latestClassification === 'passed' && ((item.historicalBrokenCount ?? 0) > 0 || (item.historicalFlakyCount ?? 0) > 0)),
+  );
+  const topItems = relevantItems.slice(0, topN);
   const broken = topItems.filter(item => item.brokenCount > 0);
   const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
+  const review = topItems.filter(
+    item =>
+      item.latestClassification === 'passed' &&
+      item.brokenCount === 0 &&
+      item.flakyCount === 0 &&
+      ((item.historicalBrokenCount ?? 0) > 0 || (item.historicalFlakyCount ?? 0) > 0),
+  );
 
   const blocks = [
     {
@@ -58,7 +71,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
           text:
             `Period (UTC): ${dateDisplay} | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')} | ` +
             `Failed CI Runs: ${failedRunCount}/${workflowCount} from ${branch}` +
-            `\nFound: ${broken.length} broken, ${flaky.length} flaky`,
+            `\nFound: ${broken.length} broken, ${flaky.length} flaky, ${review.length} review`,
         },
       ],
     },
@@ -68,7 +81,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
   if (topItems.length === 0) {
     blocks.push({
       type: 'section',
-      text: { type: 'mrkdwn', text: 'No flaky or broken tests found ✅' },
+      text: { type: 'mrkdwn', text: 'No broken/flaky/review tests found ✅' },
     });
     return blocks;
   }
@@ -137,6 +150,32 @@ export function createSlackBlocks(summary, dateDisplay, options) {
     });
   }
 
+  if ((broken.length > 0 || flaky.length > 0) && review.length > 0) {
+    blocks.push({ type: 'divider' });
+  }
+
+  if (review.length > 0) {
+    blocks.push({
+      type: 'section',
+      text: { type: 'mrkdwn', text: '*🟢 Review (now passing)*' },
+    });
+
+    review.forEach((test, index) => {
+      const globalIndex = broken.length + flaky.length + index + 1;
+      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
+      const wasBroken = test.historicalBrokenCount ?? 0;
+      const wasFlaky = test.historicalFlakyCount ?? 0;
+      const line =
+        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) ` +
+        `*now passing* (was broken ${wasBroken}x, flaky ${wasFlaky}x)`;
+
+      blocks.push({
+        type: 'section',
+        text: { type: 'mrkdwn', text: line },
+      });
+    });
+  }
+
   return blocks;
 }
 

From 5cac887c91f50ac5ff08704bdf2deb3117b87a7e Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 12:36:40 +0100
Subject: [PATCH 09/15] restore rich text block formatting for slack test
 health report

---
 .../lib/slack-test-health-blocks.mjs          | 123 +++++++++++++-----
 1 file changed, 93 insertions(+), 30 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index 24522797..5cfafe64 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -80,16 +80,30 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   if (topItems.length === 0) {
     blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: 'No broken/flaky/review tests found ✅' },
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [{ type: 'text', text: 'No broken/flaky/review tests found ✅' }],
+        },
+      ],
     });
     return blocks;
   }
 
   if (broken.length > 0) {
     blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: '*❌ Broken*' },
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [
+            { type: 'emoji', name: 'x' },
+            { type: 'text', text: ' ' },
+            { type: 'text', text: 'Broken', style: { bold: true } },
+          ],
+        },
+      ],
     });
 
     broken.forEach((test, index) => {
@@ -100,18 +114,30 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         (test.lastBrokenRunId
           ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastBrokenRunId}`
           : null);
-      const line =
-        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) *failed ${test.brokenCount}x*` +
-        (runUrl ? ` - <${runUrl}|run log>` : '');
-
       blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: line },
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [
+              { type: 'text', text: `${globalIndex}. ` },
+              { type: 'link', url: fileUrl, text: test.name },
+              { type: 'text', text: ` (${test.projectName}) ` },
+              { type: 'text', text: `failed ${test.brokenCount}x`, style: { bold: true } },
+              ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []),
+            ],
+          },
+        ],
       });
 
       blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: `_${truncateError(test.lastBrokenError)}_` },
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [{ type: 'text', text: truncateError(test.lastBrokenError), style: { italic: true } }],
+          },
+        ],
       });
     });
   }
@@ -122,8 +148,17 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   if (flaky.length > 0) {
     blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: '*🟡 Flaky*' },
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [
+            { type: 'emoji', name: 'large_yellow_circle' },
+            { type: 'text', text: ' ' },
+            { type: 'text', text: 'Flaky', style: { bold: true } },
+          ],
+        },
+      ],
     });
 
     flaky.forEach((test, index) => {
@@ -134,18 +169,30 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         (test.lastFlakyRunId
           ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastFlakyRunId}`
           : null);
-      const line =
-        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) *flaky ${test.flakyCount}x*` +
-        (runUrl ? ` - <${runUrl}|run log>` : '');
-
       blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: line },
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [
+              { type: 'text', text: `${globalIndex}. ` },
+              { type: 'link', url: fileUrl, text: test.name },
+              { type: 'text', text: ` (${test.projectName}) ` },
+              { type: 'text', text: `flaky ${test.flakyCount}x`, style: { bold: true } },
+              ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []),
+            ],
+          },
+        ],
       });
 
       blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: `_${truncateError(test.lastFlakyError)}_` },
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [{ type: 'text', text: truncateError(test.lastFlakyError), style: { italic: true } }],
+          },
+        ],
       });
     });
   }
@@ -156,8 +203,17 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   if (review.length > 0) {
     blocks.push({
-      type: 'section',
-      text: { type: 'mrkdwn', text: '*🟢 Review (now passing)*' },
+      type: 'rich_text',
+      elements: [
+        {
+          type: 'rich_text_section',
+          elements: [
+            { type: 'emoji', name: 'large_green_circle' },
+            { type: 'text', text: ' ' },
+            { type: 'text', text: 'Review (now passing)', style: { bold: true } },
+          ],
+        },
+      ],
     });
 
     review.forEach((test, index) => {
@@ -165,13 +221,20 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
       const wasBroken = test.historicalBrokenCount ?? 0;
       const wasFlaky = test.historicalFlakyCount ?? 0;
-      const line =
-        `${globalIndex}. <${fileUrl}|${test.name}> (${test.projectName}) ` +
-        `*now passing* (was broken ${wasBroken}x, flaky ${wasFlaky}x)`;
-
       blocks.push({
-        type: 'section',
-        text: { type: 'mrkdwn', text: line },
+        type: 'rich_text',
+        elements: [
+          {
+            type: 'rich_text_section',
+            elements: [
+              { type: 'text', text: `${globalIndex}. ` },
+              { type: 'link', url: fileUrl, text: test.name },
+              { type: 'text', text: ` (${test.projectName}) ` },
+              { type: 'text', text: 'now passing', style: { bold: true } },
+              { type: 'text', text: ` (was broken ${wasBroken}x, flaky ${wasFlaky}x)` },
+            ],
+          },
+        ],
       });
     });
   }

From fa21912b5c0ae90e183338a99e77f8d97fcce99d Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 13:07:10 +0100
Subject: [PATCH 10/15] implement separate per-section limits for
 broken/flaky/review tests

---
 .../lib/slack-test-health-blocks.mjs          | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index 5cfafe64..e6cf56a3 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -37,16 +37,13 @@ export function createSlackBlocks(summary, dateDisplay, options) {
     workflowCount,
   } = options;
 
-  const relevantItems = summary.filter(
-    item =>
-      item.brokenCount > 0 ||
-      item.flakyCount > 0 ||
-      (item.latestClassification === 'passed' && ((item.historicalBrokenCount ?? 0) > 0 || (item.historicalFlakyCount ?? 0) > 0)),
-  );
-  const topItems = relevantItems.slice(0, topN);
-  const broken = topItems.filter(item => item.brokenCount > 0);
-  const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
-  const review = topItems.filter(
+  const maxBroken = Math.min(topN > 5 ? 5 : Math.floor(topN / 2), topN);
+  const maxFlaky = Math.min(topN > 10 ? 3 : Math.floor(topN / 3), topN);
+  const maxReview = Math.min(topN > 10 ? 2 : 1, topN);
+
+  const brokenItems = summary.filter(item => item.brokenCount > 0);
+  const flakyItems = summary.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
+  const reviewItems = summary.filter(
     item =>
       item.latestClassification === 'passed' &&
       item.brokenCount === 0 &&
@@ -54,6 +51,11 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       ((item.historicalBrokenCount ?? 0) > 0 || (item.historicalFlakyCount ?? 0) > 0),
   );
 
+  const broken = brokenItems.slice(0, maxBroken);
+  const flaky = flakyItems.slice(0, maxFlaky);
+  const review = reviewItems.slice(0, maxReview);
+  const topItems = [...broken, ...flaky, ...review];
+
   const blocks = [
     {
       type: 'header',
@@ -71,7 +73,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
           text:
             `Period (UTC): ${dateDisplay} | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')} | ` +
             `Failed CI Runs: ${failedRunCount}/${workflowCount} from ${branch}` +
-            `\nFound: ${broken.length} broken, ${flaky.length} flaky, ${review.length} review`,
+            `\nTotal: ${brokenItems.length} broken, ${flakyItems.length} flaky, ${reviewItems.length} review | Showing top ${broken.length}, ${flaky.length}, ${review.length}`,
         },
       ],
     },

From 7e91612e71d7dbeabcbcf8ca9c249cef217d4a69 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 13:08:45 +0100
Subject: [PATCH 11/15] scale per-section limits proportionally to top-n
 parameter

---
 .../lib/slack-test-health-blocks.mjs                        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index e6cf56a3..1e145340 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -37,9 +37,9 @@ export function createSlackBlocks(summary, dateDisplay, options) {
     workflowCount,
   } = options;
 
-  const maxBroken = Math.min(topN > 5 ? 5 : Math.floor(topN / 2), topN);
-  const maxFlaky = Math.min(topN > 10 ? 3 : Math.floor(topN / 3), topN);
-  const maxReview = Math.min(topN > 10 ? 2 : 1, topN);
+  const maxBroken = Math.max(Math.ceil(topN * 0.5), 3);
+  const maxFlaky = Math.max(Math.ceil(topN * 0.3), 2);
+  const maxReview = Math.max(Math.ceil(topN * 0.2), 1);
 
   const brokenItems = summary.filter(item => item.brokenCount > 0);
   const flakyItems = summary.filter(item => item.brokenCount === 0 && item.flakyCount > 0);

From 0916d8cd191062ddf0349b5c484c4580c9688ba8 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 13:19:53 +0100
Subject: [PATCH 12/15] restrict review section to broken->passed tests only

---
 .../lib/slack-test-health-blocks.mjs                         | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index 1e145340..de37f40a 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -48,7 +48,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       item.latestClassification === 'passed' &&
       item.brokenCount === 0 &&
       item.flakyCount === 0 &&
-      ((item.historicalBrokenCount ?? 0) > 0 || (item.historicalFlakyCount ?? 0) > 0),
+      (item.historicalBrokenCount ?? 0) > 0,
   );
 
   const broken = brokenItems.slice(0, maxBroken);
@@ -222,7 +222,6 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       const globalIndex = broken.length + flaky.length + index + 1;
       const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
       const wasBroken = test.historicalBrokenCount ?? 0;
-      const wasFlaky = test.historicalFlakyCount ?? 0;
       blocks.push({
         type: 'rich_text',
         elements: [
@@ -233,7 +232,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
               { type: 'link', url: fileUrl, text: test.name },
               { type: 'text', text: ` (${test.projectName}) ` },
               { type: 'text', text: 'now passing', style: { bold: true } },
-              { type: 'text', text: ` (was broken ${wasBroken}x, flaky ${wasFlaky}x)` },
+              { type: 'text', text: ` (was broken ${wasBroken}x)` },
             ],
           },
         ],

From b7346fa7c8bdc730d414506cfc66a0b11cb9a3fe Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 13:20:43 +0100
Subject: [PATCH 13/15] update logging diagnostics to only reference
 broken->passed transitions

---
 .../create-playwright-test-health-report.mjs                | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
index d6741829..4f3cec6f 100644
--- a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
+++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
@@ -150,20 +150,20 @@ function logClassificationDiagnostics(summary) {
   const resolvedFromFailure = summary.filter(
     test =>
       test.latestClassification === 'passed' &&
-      ((test.historicalBrokenCount ?? 0) > 0 || (test.historicalFlakyCount ?? 0) > 0),
+      (test.historicalBrokenCount ?? 0) > 0,
   );
 
   console.log('\n🧾 Classification diagnostics');
   console.log(`  Unique tests observed: ${totalUniqueTests}`);
   console.log(`  Latest state -> broken: ${currentlyBroken.length}, flaky: ${currentlyFlaky.length}, passed: ${latestPassed.length}`);
-  console.log(`  Resolved since earlier runs (had broken/flaky history, latest passed): ${resolvedFromFailure.length}`);
+  console.log(`  Resolved since earlier runs (had broken history, latest passed): ${resolvedFromFailure.length}`);
 
   if (resolvedFromFailure.length > 0) {
     const preview = resolvedFromFailure
       .slice(0, 5)
       .map(test => `${test.name} (${test.projectName})`)
       .join('; ');
-    console.log(`  Sample resolved tests: ${preview}`);
+    console.log(`  Sample resolved (broken→passed): ${preview}`);
   }
 }
 

From fc0a23ff72bf01736bb4117bc10c60d1d7410943 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 13:58:16 +0100
Subject: [PATCH 14/15] enhance Playwright Test Health Report: improve input
 descriptions, refactor classification logic, and add infrastructure error
 handling

---
 .../playwright-test-health-report/action.yml  |   9 +-
 .../create-playwright-test-health-report.mjs  |  75 +++--
 .../lib/classify-report-buckets.mjs           | 100 ++++++
 .../lib/parse-playwright-json.mjs             |  35 +++
 .../lib/report-health.test.mjs                | 100 ++++++
 .../lib/slack-test-health-blocks.mjs          | 295 +++++++++---------
 .../lib/summarize-test-health.mjs             | 134 +++++---
 7 files changed, 535 insertions(+), 213 deletions(-)
 create mode 100644 .github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
 create mode 100644 .github/actions/playwright-test-health-report/lib/report-health.test.mjs

diff --git a/.github/actions/playwright-test-health-report/action.yml b/.github/actions/playwright-test-health-report/action.yml
index d1fbf9cf..e3d9cfaa 100644
--- a/.github/actions/playwright-test-health-report/action.yml
+++ b/.github/actions/playwright-test-health-report/action.yml
@@ -27,7 +27,10 @@ inputs:
     required: false
     default: main
   lookback-days:
-    description: Number of days to look back for workflow runs
+    description: >-
+      Number of days to look back for workflow runs. Use 1 for twice-daily
+      snapshots and 7 for a weekly summary. The same classification logic
+      applies regardless of window length.
     required: false
     default: '1'
   artifact-name-prefix:
@@ -44,9 +47,9 @@ inputs:
     required: false
     default: playwright-report
   top-n:
-    description: Maximum number of tests to include in Slack report
+    description: Maximum number of tests to include in the Slack report
     required: false
-    default: '10'
+    default: '15'
   report-title:
     description: Slack header title override
     required: false
diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
index 4f3cec6f..c8e69df8 100644
--- a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
+++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
@@ -4,6 +4,7 @@ import { Octokit } from '@octokit/rest';
 import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
 import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs';
 import { createSlackBlocks, sendSlackBatched } from './lib/slack-test-health-blocks.mjs';
+import { partitionSummary } from './lib/classify-report-buckets.mjs';
 import { summarizeTestHealth } from './lib/summarize-test-health.mjs';
 import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
 
@@ -12,15 +13,24 @@ if (!githubToken) {
   throw new Error('Missing GITHUB_TOKEN env var');
 }
 
+const parsePositiveInt = (value, fallback) => {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    return fallback;
+  }
+  const parsed = parseInt(trimmed, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+
 const env = {
   OWNER: process.env.OWNER || 'MetaMask',
   REPOSITORY: process.env.REPOSITORY,
   WORKFLOW_IDS: process.env.WORKFLOW_IDS,
   BRANCH: process.env.BRANCH || 'main',
-  LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
+  LOOKBACK_DAYS: parsePositiveInt(process.env.LOOKBACK_DAYS, 1),
   ARTIFACT_NAME_PREFIX: process.env.ARTIFACT_NAME_PREFIX || 'playwright-json-report',
   RESULTS_FILE_PATTERN: process.env.RESULTS_FILE_PATTERN || 'playwright-report',
-  TOP_N: parseInt(process.env.TOP_N ?? '10'),
+  TOP_N: parsePositiveInt(process.env.TOP_N, 15),
   REPORT_TITLE: process.env.REPORT_TITLE || 'Playwright Test Health Report',
   SLACK_WEBHOOK: process.env.SLACK_WEBHOOK || '',
   GITHUB_TOKEN: githubToken,
@@ -39,6 +49,14 @@ function getWorkflowIds() {
     .filter(Boolean);
 }
 
+function isTestFailureFinding(finding) {
+  return finding.classification === 'broken' || finding.classification === 'flaky' || finding.classification === 'infra';
+}
+
+function countTestFailureRuns(findings) {
+  return new Set(findings.filter(isTestFailureFinding).map(finding => finding.runId)).size;
+}
+
 async function getMergedWorkflowRuns(github, dateRange) {
   const workflowIds = getWorkflowIds();
   const runs = [];
@@ -107,6 +125,7 @@ async function collectFindings(github, runs) {
                 runId: run.id,
                 runUrl: run.html_url || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${run.id}`,
                 date: run.created_at,
+                artifactName: artifact.name,
               }),
             );
           } catch (error) {
@@ -135,35 +154,37 @@ async function sendSlackReport(summary, dateDisplay, metadata) {
     reportTitle: env.REPORT_TITLE,
     topN: env.TOP_N,
     workflowsScanned: metadata.workflowsScanned,
-    failedRunCount: metadata.failedRunCount,
     workflowCount: metadata.workflowCount,
+    testFailureRunCount: metadata.testFailureRunCount,
+    otherFailedRunCount: metadata.otherFailedRunCount,
+    lookbackDays: env.LOOKBACK_DAYS,
   });
   await sendSlackBatched(env.SLACK_WEBHOOK, blocks);
   console.log('✅ Report sent to Slack successfully');
 }
 
-function logClassificationDiagnostics(summary) {
-  const totalUniqueTests = summary.length;
-  const currentlyBroken = summary.filter(test => test.brokenCount > 0);
-  const currentlyFlaky = summary.filter(test => test.brokenCount === 0 && test.flakyCount > 0);
-  const latestPassed = summary.filter(test => test.latestClassification === 'passed');
-  const resolvedFromFailure = summary.filter(
-    test =>
-      test.latestClassification === 'passed' &&
-      (test.historicalBrokenCount ?? 0) > 0,
-  );
+function logClassificationDiagnostics(summary, metadata) {
+  const { brokenItems, flakyItems, watchItems, infraItems } = partitionSummary(summary);
 
   console.log('\n🧾 Classification diagnostics');
-  console.log(`  Unique tests observed: ${totalUniqueTests}`);
-  console.log(`  Latest state -> broken: ${currentlyBroken.length}, flaky: ${currentlyFlaky.length}, passed: ${latestPassed.length}`);
-  console.log(`  Resolved since earlier runs (had broken history, latest passed): ${resolvedFromFailure.length}`);
+  console.log(`  Lookback: ${env.LOOKBACK_DAYS} day(s)`);
+  console.log(`  Unique tests observed: ${summary.length}`);
+  console.log(
+    `  Buckets -> broken: ${brokenItems.length}, flaky: ${flakyItems.length}, watch: ${watchItems.length}, infra: ${infraItems.length}`,
+  );
+  console.log(`  CI runs: ${metadata.workflowCount} | Test-failure runs: ${metadata.testFailureRunCount}`);
+  console.log(`  Other CI failures: ${metadata.otherFailedRunCount}`);
 
-  if (resolvedFromFailure.length > 0) {
-    const preview = resolvedFromFailure
+  if (watchItems.length > 0) {
+    const preview = watchItems
       .slice(0, 5)
-      .map(test => `${test.name} (${test.projectName})`)
+      .map(test => {
+        const broken = test.historicalBrokenCount ?? 0;
+        const flaky = test.historicalFlakyCount ?? 0;
+        return `${test.name} (${test.projectName}, broken ${broken}, flaky ${flaky})`;
+      })
       .join('; ');
-    console.log(`  Sample resolved (broken→passed): ${preview}`);
+    console.log(`  Sample watch: ${preview}`);
   }
 }
 
@@ -173,6 +194,7 @@ async function main() {
   const workflowsScanned = getWorkflowIds();
 
   console.log('🧪 Playwright Test Health Report\n');
+  console.log(`Lookback: ${env.LOOKBACK_DAYS} day(s)`);
   console.log(`Time range: ${dateRange.from} to ${dateRange.to}`);
   console.log(`Workflows: ${workflowsScanned.join(', ')}\n`);
 
@@ -192,11 +214,20 @@ async function main() {
       return;
     }
 
+    const testFailureRunCount = countTestFailureRuns(findings);
+    const otherFailedRunCount = Math.max(0, failedRunCount - testFailureRunCount);
     const summary = summarizeTestHealth(findings);
-    logClassificationDiagnostics(summary);
+
+    logClassificationDiagnostics(summary, {
+      workflowCount: workflowRuns.length,
+      testFailureRunCount,
+      otherFailedRunCount,
+    });
+
     await sendSlackReport(summary, dateRange.display, {
       workflowCount: workflowRuns.length,
-      failedRunCount,
+      testFailureRunCount,
+      otherFailedRunCount,
       workflowsScanned,
     });
   } catch (error) {
diff --git a/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
new file mode 100644
index 00000000..357ba786
--- /dev/null
+++ b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
@@ -0,0 +1,100 @@
+function historicalBroken(test) {
+  return test.historicalBrokenCount ?? test.brokenCount ?? 0;
+}
+
+function historicalFlaky(test) {
+  return test.historicalFlakyCount ?? test.flakyCount ?? 0;
+}
+
+export function instabilityScore(test) {
+  return historicalBroken(test) + historicalFlaky(test);
+}
+
+export function partitionSummary(summary) {
+  const infraItems = summary
+    .filter(test => test.latestClassification === 'infra')
+    .sort((a, b) => (b.infraCount ?? 0) - (a.infraCount ?? 0));
+
+  const brokenItems = summary
+    .filter(test => test.latestClassification === 'broken')
+    .sort((a, b) => historicalBroken(b) - historicalBroken(a));
+
+  const flakyItems = summary
+    .filter(test => test.latestClassification === 'flaky')
+    .sort((a, b) => historicalFlaky(b) - historicalFlaky(a));
+
+  const watchItems = summary
+    .filter(
+      test =>
+        test.latestClassification === 'passed' &&
+        (historicalBroken(test) > 0 || historicalFlaky(test) > 0),
+    )
+    .sort((a, b) => {
+      const rateA = instabilityScore(a) / Math.max(a.totalRuns ?? 1, 1);
+      const rateB = instabilityScore(b) / Math.max(b.totalRuns ?? 1, 1);
+      if (rateB !== rateA) {
+        return rateB - rateA;
+      }
+      return instabilityScore(b) - instabilityScore(a);
+    });
+
+  return { brokenItems, flakyItems, watchItems, infraItems };
+}
+
+export function allocateBucketSlots(topN, counts) {
+  const { broken = 0, flaky = 0, watch = 0, infra = 0 } = counts;
+
+  let maxBroken = Math.min(broken, Math.max(Math.ceil(topN * 0.4), broken > 0 ? 2 : 0));
+  let maxFlaky = Math.min(flaky, Math.max(Math.ceil(topN * 0.25), flaky > 0 ? 2 : 0));
+  let maxInfra = Math.min(infra, Math.max(Math.ceil(topN * 0.1), infra > 0 ? 1 : 0));
+  let maxWatch = Math.min(watch, topN - maxBroken - maxFlaky - maxInfra);
+
+  let remaining = topN - (maxBroken + maxFlaky + maxWatch + maxInfra);
+
+  const buckets = [
+    { key: 'watch', available: watch - maxWatch, max: maxWatch },
+    { key: 'broken', available: broken - maxBroken, max: maxBroken },
+    { key: 'flaky', available: flaky - maxFlaky, max: maxFlaky },
+    { key: 'infra', available: infra - maxInfra, max: maxInfra },
+  ].sort((a, b) => b.available - a.available);
+
+  for (const bucket of buckets) {
+    if (remaining <= 0) {
+      break;
+    }
+    const extra = Math.min(remaining, bucket.available);
+    bucket.max += extra;
+    remaining -= extra;
+  }
+
+  const byKey = Object.fromEntries(buckets.map(bucket => [bucket.key, bucket.max]));
+
+  return {
+    maxBroken: byKey.broken,
+    maxFlaky: byKey.flaky,
+    maxWatch: byKey.watch,
+    maxInfra: byKey.infra,
+  };
+}
+
+export function formatRunRate(count, totalRuns) {
+  if (!totalRuns || totalRuns <= 0) {
+    return `${count}x`;
+  }
+  return `${count}/${totalRuns} runs`;
+}
+
+export function formatWatchHistory(test) {
+  const parts = [];
+  const broken = historicalBroken(test);
+  const flaky = historicalFlaky(test);
+
+  if (broken > 0) {
+    parts.push(`broken ${formatRunRate(broken, test.totalRuns)}`);
+  }
+  if (flaky > 0) {
+    parts.push(`flaky ${formatRunRate(flaky, test.totalRuns)}`);
+  }
+
+  return parts.join(', ');
+}
diff --git a/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
index 5c9ec5da..38d09c6e 100644
--- a/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
+++ b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs
@@ -39,6 +39,10 @@ function extractFirstFailureError(test) {
   return firstFailure?.error?.message ?? firstFailure?.errors?.[0]?.message ?? 'No error details';
 }
 
+function extractInfraError(error) {
+  return error?.message ?? error?.stack ?? 'Unknown setup error';
+}
+
 function walkSuites(suites, currentFile, findings, metadata) {
   for (const suite of suites ?? []) {
     const suiteFile = suite.file || currentFile;
@@ -67,6 +71,7 @@ function walkSuites(suites, currentFile, findings, metadata) {
           runId: metadata.runId,
           runUrl: metadata.runUrl,
           date: new Date(metadata.date),
+          artifactName: metadata.artifactName,
         });
       }
     }
@@ -75,8 +80,38 @@ function walkSuites(suites, currentFile, findings, metadata) {
   }
 }
 
+function parseInfraErrors(report, metadata, findings) {
+  const errors = Array.isArray(report?.errors) ? report.errors : [];
+  const suites = report?.suites ?? [];
+
+  if (errors.length === 0 || suites.length > 0) {
+    return;
+  }
+
+  for (const [index, error] of errors.entries()) {
+    const artifactLabel = metadata.artifactName ?? 'unknown-artifact';
+    const location = error?.location?.file ?? 'unknown-file';
+    const key = `infra::${artifactLabel}::${index}`;
+
+    findings.push({
+      key,
+      name: `Setup failure (${artifactLabel})`,
+      path: location,
+      projectName: 'infra',
+      classification: 'infra',
+      retries: 0,
+      error: extractInfraError(error),
+      runId: metadata.runId,
+      runUrl: metadata.runUrl,
+      date: new Date(metadata.date),
+      artifactName: metadata.artifactName,
+    });
+  }
+}
+
 export function parsePlaywrightJsonReport(report, metadata) {
   const findings = [];
   walkSuites(report?.suites ?? [], undefined, findings, metadata);
+  parseInfraErrors(report, metadata, findings);
   return findings;
 }
diff --git a/.github/actions/playwright-test-health-report/lib/report-health.test.mjs b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs
new file mode 100644
index 00000000..11d1c5f2
--- /dev/null
+++ b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs
@@ -0,0 +1,100 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  allocateBucketSlots,
+  formatWatchHistory,
+  partitionSummary,
+} from './classify-report-buckets.mjs';
+import { summarizeTestHealth } from './summarize-test-health.mjs';
+
+describe('summarizeTestHealth', () => {
+  it('keeps historical counts when latest run passed', () => {
+    const summary = summarizeTestHealth([
+      {
+        key: 'ios::file.spec.ts::deletes account',
+        name: 'deletes account',
+        path: 'file.spec.ts',
+        projectName: 'ios-smoke',
+        classification: 'broken',
+        retries: 1,
+        error: 'timeout',
+        runId: '1',
+        runUrl: 'https://example.com/1',
+        date: new Date('2026-06-16T10:00:00Z'),
+      },
+      {
+        key: 'ios::file.spec.ts::deletes account',
+        name: 'deletes account',
+        path: 'file.spec.ts',
+        projectName: 'ios-smoke',
+        classification: 'flaky',
+        retries: 1,
+        error: 'timeout',
+        runId: '2',
+        runUrl: 'https://example.com/2',
+        date: new Date('2026-06-16T12:00:00Z'),
+      },
+      {
+        key: 'ios::file.spec.ts::deletes account',
+        name: 'deletes account',
+        path: 'file.spec.ts',
+        projectName: 'ios-smoke',
+        classification: 'passed',
+        retries: 0,
+        error: '',
+        runId: '3',
+        runUrl: 'https://example.com/3',
+        date: new Date('2026-06-17T10:00:00Z'),
+      },
+    ]);
+
+    const test = summary[0];
+    assert.equal(test.latestClassification, 'passed');
+    assert.equal(test.historicalBrokenCount, 1);
+    assert.equal(test.historicalFlakyCount, 1);
+    assert.equal(test.totalRuns, 3);
+  });
+});
+
+describe('partitionSummary', () => {
+  it('puts passed tests with flaky history into watch', () => {
+    const { watchItems, flakyItems, brokenItems } = partitionSummary([
+      {
+        latestClassification: 'passed',
+        historicalBrokenCount: 0,
+        historicalFlakyCount: 4,
+        brokenCount: 0,
+        flakyCount: 4,
+        totalRuns: 10,
+        name: 'exports srp',
+      },
+    ]);
+
+    assert.equal(watchItems.length, 1);
+    assert.equal(flakyItems.length, 0);
+    assert.equal(brokenItems.length, 0);
+  });
+});
+
+describe('allocateBucketSlots', () => {
+  it('gives spare capacity to watch when nothing is currently broken', () => {
+    const slots = allocateBucketSlots(15, { broken: 0, flaky: 0, watch: 13, infra: 0 });
+
+    assert.equal(slots.maxBroken, 0);
+    assert.equal(slots.maxWatch, 13);
+    assert.equal(slots.maxBroken + slots.maxFlaky + slots.maxWatch + slots.maxInfra, 13);
+  });
+});
+
+describe('formatWatchHistory', () => {
+  it('includes both broken and flaky history', () => {
+    const text = formatWatchHistory({
+      historicalBrokenCount: 2,
+      historicalFlakyCount: 3,
+      totalRuns: 8,
+    });
+
+    assert.match(text, /broken 2\/8 runs/);
+    assert.match(text, /flaky 3\/8 runs/);
+  });
+});
diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index de37f40a..ee633bf2 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -1,4 +1,10 @@
 import { IncomingWebhook } from '@slack/webhook';
+import {
+  allocateBucketSlots,
+  formatRunRate,
+  formatWatchHistory,
+  partitionSummary,
+} from './classify-report-buckets.mjs';
 
 export function normalizeErrorForSlack(message, maxLength = 120) {
   if (!message) {
@@ -25,6 +31,75 @@ export function truncateError(message, maxLength = 120) {
   return normalizeErrorForSlack(message, maxLength);
 }
 
+function buildRunUrl(owner, repository, test, kind) {
+  const runUrlField = {
+    broken: 'lastBrokenRunUrl',
+    flaky: 'lastFlakyRunUrl',
+    infra: 'lastInfraRunUrl',
+  }[kind];
+  const runIdField = {
+    broken: 'lastBrokenRunId',
+    flaky: 'lastFlakyRunId',
+    infra: 'lastInfraRunId',
+  }[kind];
+
+  return (
+    test[runUrlField] ||
+    (test[runIdField]
+      ? `https://github.com/${owner}/${repository}/actions/runs/${test[runIdField]}`
+      : null)
+  );
+}
+
+function pushSectionHeader(blocks, emoji, title) {
+  blocks.push({
+    type: 'rich_text',
+    elements: [
+      {
+        type: 'rich_text_section',
+        elements: [
+          ...(emoji ? [{ type: 'emoji', name: emoji }] : []),
+          ...(emoji ? [{ type: 'text', text: ' ' }] : []),
+          { type: 'text', text: title, style: { bold: true } },
+        ],
+      },
+    ],
+  });
+}
+
+function pushTestLine(blocks, { index, owner, repository, branch, test, statusText, runKind }) {
+  const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
+  const runUrl = buildRunUrl(owner, repository, test, runKind);
+
+  blocks.push({
+    type: 'rich_text',
+    elements: [
+      {
+        type: 'rich_text_section',
+        elements: [
+          { type: 'text', text: `${index}. ` },
+          { type: 'link', url: fileUrl, text: test.name },
+          { type: 'text', text: ` (${test.projectName}) ` },
+          { type: 'text', text: statusText, style: { bold: true } },
+          ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []),
+        ],
+      },
+    ],
+  });
+}
+
+function pushErrorLine(blocks, message) {
+  blocks.push({
+    type: 'rich_text',
+    elements: [
+      {
+        type: 'rich_text_section',
+        elements: [{ type: 'text', text: truncateError(message), style: { italic: true } }],
+      },
+    ],
+  });
+}
+
 export function createSlackBlocks(summary, dateDisplay, options) {
   const {
     owner,
@@ -33,28 +108,25 @@ export function createSlackBlocks(summary, dateDisplay, options) {
     reportTitle,
     topN,
     workflowsScanned,
-    failedRunCount,
     workflowCount,
+    testFailureRunCount,
+    otherFailedRunCount,
+    lookbackDays = 1,
   } = options;
 
-  const maxBroken = Math.max(Math.ceil(topN * 0.5), 3);
-  const maxFlaky = Math.max(Math.ceil(topN * 0.3), 2);
-  const maxReview = Math.max(Math.ceil(topN * 0.2), 1);
-
-  const brokenItems = summary.filter(item => item.brokenCount > 0);
-  const flakyItems = summary.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
-  const reviewItems = summary.filter(
-    item =>
-      item.latestClassification === 'passed' &&
-      item.brokenCount === 0 &&
-      item.flakyCount === 0 &&
-      (item.historicalBrokenCount ?? 0) > 0,
-  );
+  const { brokenItems, flakyItems, watchItems, infraItems } = partitionSummary(summary);
+  const { maxBroken, maxFlaky, maxWatch, maxInfra } = allocateBucketSlots(topN, {
+    broken: brokenItems.length,
+    flaky: flakyItems.length,
+    watch: watchItems.length,
+    infra: infraItems.length,
+  });
 
   const broken = brokenItems.slice(0, maxBroken);
   const flaky = flakyItems.slice(0, maxFlaky);
-  const review = reviewItems.slice(0, maxReview);
-  const topItems = [...broken, ...flaky, ...review];
+  const watch = watchItems.slice(0, maxWatch);
+  const infra = infraItems.slice(0, maxInfra);
+  const topItems = [...broken, ...flaky, ...watch, ...infra];
 
   const blocks = [
     {
@@ -71,9 +143,10 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         {
           type: 'mrkdwn',
           text:
-            `Period (UTC): ${dateDisplay} | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')} | ` +
-            `Failed CI Runs: ${failedRunCount}/${workflowCount} from ${branch}` +
-            `\nTotal: ${brokenItems.length} broken, ${flakyItems.length} flaky, ${reviewItems.length} review | Showing top ${broken.length}, ${flaky.length}, ${review.length}`,
+            `Period (UTC): ${dateDisplay} | Lookback: ${lookbackDays} day(s) | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')}` +
+            `\nCI runs: ${workflowCount} on ${branch} | Test-failure runs: ${testFailureRunCount} | Other CI failures: ${otherFailedRunCount}` +
+            `\nTests: ${brokenItems.length} broken, ${flakyItems.length} flaky, ${watchItems.length} watch, ${infraItems.length} infra` +
+            ` | Showing ${broken.length}, ${flaky.length}, ${watch.length}, ${infra.length}`,
         },
       ],
     },
@@ -86,157 +159,91 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       elements: [
         {
           type: 'rich_text_section',
-          elements: [{ type: 'text', text: 'No broken/flaky/review tests found ✅' }],
+          elements: [{ type: 'text', text: 'No broken, flaky, watch, or infra issues found ✅' }],
         },
       ],
     });
     return blocks;
   }
 
-  if (broken.length > 0) {
-    blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [
-            { type: 'emoji', name: 'x' },
-            { type: 'text', text: ' ' },
-            { type: 'text', text: 'Broken', style: { bold: true } },
-          ],
-        },
-      ],
-    });
-
-    broken.forEach((test, index) => {
-      const globalIndex = index + 1;
-      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
-      const runUrl =
-        test.lastBrokenRunUrl ||
-        (test.lastBrokenRunId
-          ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastBrokenRunId}`
-          : null);
-      blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [
-              { type: 'text', text: `${globalIndex}. ` },
-              { type: 'link', url: fileUrl, text: test.name },
-              { type: 'text', text: ` (${test.projectName}) ` },
-              { type: 'text', text: `failed ${test.brokenCount}x`, style: { bold: true } },
-              ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []),
-            ],
-          },
-        ],
-      });
+  let itemIndex = 0;
 
-      blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [{ type: 'text', text: truncateError(test.lastBrokenError), style: { italic: true } }],
-          },
-        ],
+  if (broken.length > 0) {
+    pushSectionHeader(blocks, 'x', 'Broken (latest run failed)');
+    broken.forEach(test => {
+      itemIndex += 1;
+      pushTestLine(blocks, {
+        index: itemIndex,
+        owner,
+        repository,
+        branch,
+        test,
+        statusText: `failed ${formatRunRate(test.historicalBrokenCount ?? test.brokenCount, test.totalRuns)}`,
+        runKind: 'broken',
       });
+      pushErrorLine(blocks, test.lastBrokenError);
     });
   }
 
-  if (broken.length > 0 && flaky.length > 0) {
+  if (broken.length > 0 && (flaky.length > 0 || watch.length > 0 || infra.length > 0)) {
     blocks.push({ type: 'divider' });
   }
 
   if (flaky.length > 0) {
-    blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [
-            { type: 'emoji', name: 'large_yellow_circle' },
-            { type: 'text', text: ' ' },
-            { type: 'text', text: 'Flaky', style: { bold: true } },
-          ],
-        },
-      ],
+    pushSectionHeader(blocks, 'large_yellow_circle', 'Flaky (latest run flaky)');
+    flaky.forEach(test => {
+      itemIndex += 1;
+      pushTestLine(blocks, {
+        index: itemIndex,
+        owner,
+        repository,
+        branch,
+        test,
+        statusText: `flaky ${formatRunRate(test.historicalFlakyCount ?? test.flakyCount, test.totalRuns)}`,
+        runKind: 'flaky',
+      });
+      pushErrorLine(blocks, test.lastFlakyError);
     });
+  }
 
-    flaky.forEach((test, index) => {
-      const globalIndex = broken.length + index + 1;
-      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
-      const runUrl =
-        test.lastFlakyRunUrl ||
-        (test.lastFlakyRunId
-          ? `https://github.com/${owner}/${repository}/actions/runs/${test.lastFlakyRunId}`
-          : null);
-      blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [
-              { type: 'text', text: `${globalIndex}. ` },
-              { type: 'link', url: fileUrl, text: test.name },
-              { type: 'text', text: ` (${test.projectName}) ` },
-              { type: 'text', text: `flaky ${test.flakyCount}x`, style: { bold: true } },
-              ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []),
-            ],
-          },
-        ],
-      });
+  if (flaky.length > 0 && (watch.length > 0 || infra.length > 0)) {
+    blocks.push({ type: 'divider' });
+  }
 
-      blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [{ type: 'text', text: truncateError(test.lastFlakyError), style: { italic: true } }],
-          },
-        ],
+  if (watch.length > 0) {
+    pushSectionHeader(blocks, 'large_green_circle', 'Watch (unstable in window, passing now)');
+    watch.forEach(test => {
+      itemIndex += 1;
+      pushTestLine(blocks, {
+        index: itemIndex,
+        owner,
+        repository,
+        branch,
+        test,
+        statusText: `now passing (${formatWatchHistory(test)})`,
+        runKind: 'broken',
       });
     });
   }
 
-  if ((broken.length > 0 || flaky.length > 0) && review.length > 0) {
+  if (watch.length > 0 && infra.length > 0) {
     blocks.push({ type: 'divider' });
   }
 
-  if (review.length > 0) {
-    blocks.push({
-      type: 'rich_text',
-      elements: [
-        {
-          type: 'rich_text_section',
-          elements: [
-            { type: 'emoji', name: 'large_green_circle' },
-            { type: 'text', text: ' ' },
-            { type: 'text', text: 'Review (now passing)', style: { bold: true } },
-          ],
-        },
-      ],
-    });
-
-    review.forEach((test, index) => {
-      const globalIndex = broken.length + flaky.length + index + 1;
-      const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
-      const wasBroken = test.historicalBrokenCount ?? 0;
-      blocks.push({
-        type: 'rich_text',
-        elements: [
-          {
-            type: 'rich_text_section',
-            elements: [
-              { type: 'text', text: `${globalIndex}. ` },
-              { type: 'link', url: fileUrl, text: test.name },
-              { type: 'text', text: ` (${test.projectName}) ` },
-              { type: 'text', text: 'now passing', style: { bold: true } },
-              { type: 'text', text: ` (was broken ${wasBroken}x)` },
-            ],
-          },
-        ],
+  if (infra.length > 0) {
+    pushSectionHeader(blocks, 'warning', 'Infra (setup failed, no tests ran)');
+    infra.forEach(test => {
+      itemIndex += 1;
+      pushTestLine(blocks, {
+        index: itemIndex,
+        owner,
+        repository,
+        branch,
+        test,
+        statusText: `setup failed ${formatRunRate(test.historicalInfraCount ?? test.infraCount, test.totalRuns)}`,
+        runKind: 'infra',
       });
+      pushErrorLine(blocks, test.lastInfraError);
     });
   }
 
diff --git a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
index bd01ae75..0978eeb3 100644
--- a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
+++ b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs
@@ -1,68 +1,114 @@
+function createEntry(finding) {
+  return {
+    key: finding.key,
+    name: finding.name,
+    path: finding.path,
+    projectName: finding.projectName,
+    brokenCount: 0,
+    flakyCount: 0,
+    passedCount: 0,
+    infraCount: 0,
+    totalRuns: 0,
+    totalRetries: 0,
+    seenRunIds: new Set(),
+    lastSeen: finding.date,
+    latestClassification: finding.classification,
+    lastBrokenRunId: undefined,
+    lastBrokenRunUrl: undefined,
+    lastBrokenError: undefined,
+    lastFlakyRunId: undefined,
+    lastFlakyRunUrl: undefined,
+    lastFlakyError: undefined,
+    lastInfraRunId: undefined,
+    lastInfraRunUrl: undefined,
+    lastInfraError: undefined,
+  };
+}
+
+function applyClassification(entry, finding) {
+  if (!entry.seenRunIds.has(finding.runId)) {
+    entry.seenRunIds.add(finding.runId);
+    entry.totalRuns += 1;
+  }
+
+  entry.totalRetries += finding.retries;
+
+  if (finding.classification === 'broken') {
+    entry.brokenCount += 1;
+    entry.lastBrokenRunId = finding.runId;
+    entry.lastBrokenRunUrl = finding.runUrl;
+    entry.lastBrokenError = finding.error;
+  } else if (finding.classification === 'flaky') {
+    entry.flakyCount += 1;
+    entry.lastFlakyRunId = finding.runId;
+    entry.lastFlakyRunUrl = finding.runUrl;
+    entry.lastFlakyError = finding.error;
+  } else if (finding.classification === 'infra') {
+    entry.infraCount += 1;
+    entry.lastInfraRunId = finding.runId;
+    entry.lastInfraRunUrl = finding.runUrl;
+    entry.lastInfraError = finding.error;
+  } else if (finding.classification === 'passed') {
+    entry.passedCount += 1;
+  }
+
+  if (finding.date >= entry.lastSeen) {
+    entry.lastSeen = finding.date;
+    entry.latestClassification = finding.classification;
+  }
+}
+
 export function summarizeTestHealth(findings) {
   const summary = new Map();
 
   for (const finding of findings) {
     const existing = summary.get(finding.key);
-
     if (existing) {
-      if (finding.classification === 'broken') {
-        existing.brokenCount += 1;
-        existing.lastBrokenRunId = finding.runId;
-        existing.lastBrokenRunUrl = finding.runUrl;
-        existing.lastBrokenError = finding.error;
-      } else if (finding.classification === 'flaky') {
-        existing.flakyCount += 1;
-        existing.lastFlakyRunId = finding.runId;
-        existing.lastFlakyRunUrl = finding.runUrl;
-        existing.lastFlakyError = finding.error;
-      }
-
-      existing.totalRetries += finding.retries;
-      if (finding.date >= existing.lastSeen) {
-        existing.lastSeen = finding.date;
-        existing.latestClassification = finding.classification;
-      }
+      applyClassification(existing, finding);
       continue;
     }
 
-    summary.set(finding.key, {
-      key: finding.key,
-      name: finding.name,
-      path: finding.path,
-      projectName: finding.projectName,
-      brokenCount: finding.classification === 'broken' ? 1 : 0,
-      flakyCount: finding.classification === 'flaky' ? 1 : 0,
-      totalRetries: finding.retries,
-      lastSeen: finding.date,
-      latestClassification: finding.classification,
-      lastBrokenRunId: finding.classification === 'broken' ? finding.runId : undefined,
-      lastBrokenRunUrl: finding.classification === 'broken' ? finding.runUrl : undefined,
-      lastBrokenError: finding.classification === 'broken' ? finding.error : undefined,
-      lastFlakyRunId: finding.classification === 'flaky' ? finding.runId : undefined,
-      lastFlakyRunUrl: finding.classification === 'flaky' ? finding.runUrl : undefined,
-      lastFlakyError: finding.classification === 'flaky' ? finding.error : undefined,
-    });
+    const entry = createEntry(finding);
+    applyClassification(entry, finding);
+    summary.set(finding.key, entry);
   }
 
   return Array.from(summary.values())
     .map(item => {
-      const latestIsBroken = item.latestClassification === 'broken';
-      const latestIsFlaky = item.latestClassification === 'flaky';
+      const { seenRunIds, ...rest } = item;
       return {
-        ...item,
+        ...rest,
         historicalBrokenCount: item.brokenCount,
         historicalFlakyCount: item.flakyCount,
-        brokenCount: latestIsBroken ? item.brokenCount : 0,
-        flakyCount: latestIsFlaky ? item.flakyCount : 0,
+        historicalInfraCount: item.infraCount,
       };
     })
     .sort((a, b) => {
-      if (a.brokenCount !== b.brokenCount) {
-        return b.brokenCount - a.brokenCount;
+      const latestRank = (classification, item) => {
+        if (classification === 'broken' || classification === 'infra') {
+          return 3;
+        }
+        if (classification === 'flaky') {
+          return 2;
+        }
+        if ((item.brokenCount ?? 0) > 0 || (item.flakyCount ?? 0) > 0) {
+          return 1;
+        }
+        return 0;
+      };
+
+      const rankA = latestRank(a.latestClassification, a);
+      const rankB = latestRank(b.latestClassification, b);
+      if (rankA !== rankB) {
+        return rankB - rankA;
       }
-      if (a.flakyCount !== b.flakyCount) {
-        return b.flakyCount - a.flakyCount;
+
+      const instabilityA = a.brokenCount + a.flakyCount;
+      const instabilityB = b.brokenCount + b.flakyCount;
+      if (instabilityA !== instabilityB) {
+        return instabilityB - instabilityA;
       }
+
       return b.totalRetries - a.totalRetries;
     });
 }

From 8a34c2353f0bacfcb362f958e6679933947a5a83 Mon Sep 17 00:00:00 2001
From: cmd-ob <ola.bale@consensys.net>
Date: Wed, 17 Jun 2026 14:08:30 +0100
Subject: [PATCH 15/15] refactor Playwright Test Health Report: update
 terminology from 'broken' to 'failing' and reorganize Slack block structure
 for clarity

---
 .../playwright-test-health-report/action.yml  |   2 +-
 .../create-playwright-test-health-report.mjs  |   4 +-
 .../lib/classify-report-buckets.mjs           |   2 +-
 .../lib/report-health.test.mjs                |   2 +-
 .../lib/slack-test-health-blocks.mjs          | 129 +++++++++---------
 5 files changed, 66 insertions(+), 73 deletions(-)

diff --git a/.github/actions/playwright-test-health-report/action.yml b/.github/actions/playwright-test-health-report/action.yml
index e3d9cfaa..10306330 100644
--- a/.github/actions/playwright-test-health-report/action.yml
+++ b/.github/actions/playwright-test-health-report/action.yml
@@ -1,7 +1,7 @@
 name: Playwright Test Health Report
 description: >-
   Aggregates Playwright JSON test reports from GitHub Actions artifacts,
-  classifies flaky and broken tests, and posts a summary to Slack.
+  classifies flaky and failing tests, and posts a summary to Slack.
 
 inputs:
   repository:
diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
index c8e69df8..b3932ba2 100644
--- a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
+++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs
@@ -170,7 +170,7 @@ function logClassificationDiagnostics(summary, metadata) {
   console.log(`  Lookback: ${env.LOOKBACK_DAYS} day(s)`);
   console.log(`  Unique tests observed: ${summary.length}`);
   console.log(
-    `  Buckets -> broken: ${brokenItems.length}, flaky: ${flakyItems.length}, watch: ${watchItems.length}, infra: ${infraItems.length}`,
+    `  Buckets -> failing: ${brokenItems.length}, infra: ${infraItems.length}, flaky: ${flakyItems.length}, watch: ${watchItems.length}`,
   );
   console.log(`  CI runs: ${metadata.workflowCount} | Test-failure runs: ${metadata.testFailureRunCount}`);
   console.log(`  Other CI failures: ${metadata.otherFailedRunCount}`);
@@ -181,7 +181,7 @@ function logClassificationDiagnostics(summary, metadata) {
       .map(test => {
         const broken = test.historicalBrokenCount ?? 0;
         const flaky = test.historicalFlakyCount ?? 0;
-        return `${test.name} (${test.projectName}, broken ${broken}, flaky ${flaky})`;
+        return `${test.name} (${test.projectName}, failed ${broken}, flaky ${flaky})`;
       })
       .join('; ');
     console.log(`  Sample watch: ${preview}`);
diff --git a/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
index 357ba786..c3c9c1c5 100644
--- a/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
+++ b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs
@@ -90,7 +90,7 @@ export function formatWatchHistory(test) {
   const flaky = historicalFlaky(test);
 
   if (broken > 0) {
-    parts.push(`broken ${formatRunRate(broken, test.totalRuns)}`);
+    parts.push(`failed ${formatRunRate(broken, test.totalRuns)}`);
   }
   if (flaky > 0) {
     parts.push(`flaky ${formatRunRate(flaky, test.totalRuns)}`);
diff --git a/.github/actions/playwright-test-health-report/lib/report-health.test.mjs b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs
index 11d1c5f2..a382a568 100644
--- a/.github/actions/playwright-test-health-report/lib/report-health.test.mjs
+++ b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs
@@ -94,7 +94,7 @@ describe('formatWatchHistory', () => {
       totalRuns: 8,
     });
 
-    assert.match(text, /broken 2\/8 runs/);
+    assert.match(text, /failed 2\/8 runs/);
     assert.match(text, /flaky 3\/8 runs/);
   });
 });
diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
index ee633bf2..de3dbca0 100644
--- a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
+++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs
@@ -123,10 +123,10 @@ export function createSlackBlocks(summary, dateDisplay, options) {
   });
 
   const broken = brokenItems.slice(0, maxBroken);
+  const infra = infraItems.slice(0, maxInfra);
   const flaky = flakyItems.slice(0, maxFlaky);
   const watch = watchItems.slice(0, maxWatch);
-  const infra = infraItems.slice(0, maxInfra);
-  const topItems = [...broken, ...flaky, ...watch, ...infra];
+  const topItems = [...broken, ...infra, ...flaky, ...watch];
 
   const blocks = [
     {
@@ -145,8 +145,8 @@ export function createSlackBlocks(summary, dateDisplay, options) {
           text:
             `Period (UTC): ${dateDisplay} | Lookback: ${lookbackDays} day(s) | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')}` +
             `\nCI runs: ${workflowCount} on ${branch} | Test-failure runs: ${testFailureRunCount} | Other CI failures: ${otherFailedRunCount}` +
-            `\nTests: ${brokenItems.length} broken, ${flakyItems.length} flaky, ${watchItems.length} watch, ${infraItems.length} infra` +
-            ` | Showing ${broken.length}, ${flaky.length}, ${watch.length}, ${infra.length}`,
+            `\nTests: ${brokenItems.length} failing, ${infraItems.length} infra, ${flakyItems.length} flaky, ${watchItems.length} watch` +
+            ` | Showing ${broken.length}, ${infra.length}, ${flaky.length}, ${watch.length}`,
         },
       ],
     },
@@ -159,7 +159,7 @@ export function createSlackBlocks(summary, dateDisplay, options) {
       elements: [
         {
           type: 'rich_text_section',
-          elements: [{ type: 'text', text: 'No broken, flaky, watch, or infra issues found ✅' }],
+          elements: [{ type: 'text', text: 'No failing, infra, flaky, or watch issues found ✅' }],
         },
       ],
     });
@@ -168,51 +168,58 @@ export function createSlackBlocks(summary, dateDisplay, options) {
 
   let itemIndex = 0;
 
-  if (broken.length > 0) {
-    pushSectionHeader(blocks, 'x', 'Broken (latest run failed)');
-    broken.forEach(test => {
-      itemIndex += 1;
-      pushTestLine(blocks, {
-        index: itemIndex,
-        owner,
-        repository,
-        branch,
-        test,
-        statusText: `failed ${formatRunRate(test.historicalBrokenCount ?? test.brokenCount, test.totalRuns)}`,
-        runKind: 'broken',
-      });
-      pushErrorLine(blocks, test.lastBrokenError);
-    });
-  }
+  const sections = [
+    {
+      items: broken,
+      emoji: 'x',
+      title: 'Failing (latest run failed)',
+      runKind: 'broken',
+      statusText: test =>
+        `failed ${formatRunRate(test.historicalBrokenCount ?? test.brokenCount, test.totalRuns)}`,
+      error: test => test.lastBrokenError,
+    },
+    {
+      items: infra,
+      emoji: 'warning',
+      title: 'Infra (setup failed, no tests ran)',
+      runKind: 'infra',
+      statusText: test =>
+        `setup failed ${formatRunRate(test.historicalInfraCount ?? test.infraCount, test.totalRuns)}`,
+      error: test => test.lastInfraError,
+    },
+    {
+      items: flaky,
+      emoji: 'large_yellow_circle',
+      title: 'Flaky (latest run flaky)',
+      runKind: 'flaky',
+      statusText: test =>
+        `flaky ${formatRunRate(test.historicalFlakyCount ?? test.flakyCount, test.totalRuns)}`,
+      error: test => test.lastFlakyError,
+    },
+    {
+      items: watch,
+      emoji: 'large_green_circle',
+      title: 'Watch (unstable in window, passing now)',
+      runKind: 'broken',
+      statusText: test => `now passing (${formatWatchHistory(test)})`,
+      error: () => null,
+    },
+  ];
 
-  if (broken.length > 0 && (flaky.length > 0 || watch.length > 0 || infra.length > 0)) {
-    blocks.push({ type: 'divider' });
-  }
+  let previousSectionHadItems = false;
 
-  if (flaky.length > 0) {
-    pushSectionHeader(blocks, 'large_yellow_circle', 'Flaky (latest run flaky)');
-    flaky.forEach(test => {
-      itemIndex += 1;
-      pushTestLine(blocks, {
-        index: itemIndex,
-        owner,
-        repository,
-        branch,
-        test,
-        statusText: `flaky ${formatRunRate(test.historicalFlakyCount ?? test.flakyCount, test.totalRuns)}`,
-        runKind: 'flaky',
-      });
-      pushErrorLine(blocks, test.lastFlakyError);
-    });
-  }
+  for (const section of sections) {
+    if (section.items.length === 0) {
+      continue;
+    }
 
-  if (flaky.length > 0 && (watch.length > 0 || infra.length > 0)) {
-    blocks.push({ type: 'divider' });
-  }
+    if (previousSectionHadItems) {
+      blocks.push({ type: 'divider' });
+    }
+
+    pushSectionHeader(blocks, section.emoji, section.title);
 
-  if (watch.length > 0) {
-    pushSectionHeader(blocks, 'large_green_circle', 'Watch (unstable in window, passing now)');
-    watch.forEach(test => {
+    for (const test of section.items) {
       itemIndex += 1;
       pushTestLine(blocks, {
         index: itemIndex,
@@ -220,31 +227,17 @@ export function createSlackBlocks(summary, dateDisplay, options) {
         repository,
         branch,
         test,
-        statusText: `now passing (${formatWatchHistory(test)})`,
-        runKind: 'broken',
+        statusText: section.statusText(test),
+        runKind: section.runKind,
       });
-    });
-  }
 
-  if (watch.length > 0 && infra.length > 0) {
-    blocks.push({ type: 'divider' });
-  }
+      const error = section.error(test);
+      if (error) {
+        pushErrorLine(blocks, error);
+      }
+    }
 
-  if (infra.length > 0) {
-    pushSectionHeader(blocks, 'warning', 'Infra (setup failed, no tests ran)');
-    infra.forEach(test => {
-      itemIndex += 1;
-      pushTestLine(blocks, {
-        index: itemIndex,
-        owner,
-        repository,
-        branch,
-        test,
-        statusText: `setup failed ${formatRunRate(test.historicalInfraCount ?? test.infraCount, test.totalRuns)}`,
-        runKind: 'infra',
-      });
-      pushErrorLine(blocks, test.lastInfraError);
-    });
+    previousSectionHadItems = true;
   }
 
   return blocks;