enhance Playwright Test Health Report: improve input descriptions, refactor classification logic, and add infrastructure error handling

Author: cmd-ob

.github/actions/playwright-test-health-report/action.yml

@@ -27,7 +27,10 @@ inputs:
     required: false
     default: main
   lookback-days:
-    description: Number of days to look back for workflow runs
+    description: >-
+      Number of days to look back for workflow runs. Use 1 for twice-daily
+      snapshots and 7 for a weekly summary. The same classification logic
+      applies regardless of window length.
     required: false
     default: '1'
   artifact-name-prefix:
@@ -44,9 +47,9 @@ inputs:
     required: false
     default: playwright-report
   top-n:
-    description: Maximum number of tests to include in Slack report
+    description: Maximum number of tests to include in the Slack report
     required: false
-    default: '10'
+    default: '15'
   report-title:
     description: Slack header title override
     required: false

.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs

@@ -4,6 +4,7 @@ import { Octokit } from '@octokit/rest';
 import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
 import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs';
 import { createSlackBlocks, sendSlackBatched } from './lib/slack-test-health-blocks.mjs';
+import { partitionSummary } from './lib/classify-report-buckets.mjs';
 import { summarizeTestHealth } from './lib/summarize-test-health.mjs';
 import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
 
@@ -12,15 +13,24 @@ if (!githubToken) {
   throw new Error('Missing GITHUB_TOKEN env var');
 }
 
+const parsePositiveInt = (value, fallback) => {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    return fallback;
+  }
+  const parsed = parseInt(trimmed, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+
 const env = {
   OWNER: process.env.OWNER || 'MetaMask',
   REPOSITORY: process.env.REPOSITORY,
   WORKFLOW_IDS: process.env.WORKFLOW_IDS,
   BRANCH: process.env.BRANCH || 'main',
-  LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
+  LOOKBACK_DAYS: parsePositiveInt(process.env.LOOKBACK_DAYS, 1),
   ARTIFACT_NAME_PREFIX: process.env.ARTIFACT_NAME_PREFIX || 'playwright-json-report',
   RESULTS_FILE_PATTERN: process.env.RESULTS_FILE_PATTERN || 'playwright-report',
-  TOP_N: parseInt(process.env.TOP_N ?? '10'),
+  TOP_N: parsePositiveInt(process.env.TOP_N, 15),
   REPORT_TITLE: process.env.REPORT_TITLE || 'Playwright Test Health Report',
   SLACK_WEBHOOK: process.env.SLACK_WEBHOOK || '',
   GITHUB_TOKEN: githubToken,
@@ -39,6 +49,14 @@ function getWorkflowIds() {
     .filter(Boolean);
 }
 
+function isTestFailureFinding(finding) {
+  return finding.classification === 'broken' || finding.classification === 'flaky' || finding.classification === 'infra';
+}
+
+function countTestFailureRuns(findings) {
+  return new Set(findings.filter(isTestFailureFinding).map(finding => finding.runId)).size;
+}
+
 async function getMergedWorkflowRuns(github, dateRange) {
   const workflowIds = getWorkflowIds();
   const runs = [];
@@ -107,6 +125,7 @@ async function collectFindings(github, runs) {
                 runId: run.id,
                 runUrl: run.html_url || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${run.id}`,
                 date: run.created_at,
+                artifactName: artifact.name,
               }),
             );
           } catch (error) {
@@ -135,35 +154,37 @@ async function sendSlackReport(summary, dateDisplay, metadata) {
     reportTitle: env.REPORT_TITLE,
     topN: env.TOP_N,
     workflowsScanned: metadata.workflowsScanned,
-    failedRunCount: metadata.failedRunCount,
     workflowCount: metadata.workflowCount,
+    testFailureRunCount: metadata.testFailureRunCount,
+    otherFailedRunCount: metadata.otherFailedRunCount,
+    lookbackDays: env.LOOKBACK_DAYS,
   });
   await sendSlackBatched(env.SLACK_WEBHOOK, blocks);
   console.log('✅ Report sent to Slack successfully');
 }
 
-function logClassificationDiagnostics(summary) {
-  const totalUniqueTests = summary.length;
-  const currentlyBroken = summary.filter(test => test.brokenCount > 0);
-  const currentlyFlaky = summary.filter(test => test.brokenCount === 0 && test.flakyCount > 0);
-  const latestPassed = summary.filter(test => test.latestClassification === 'passed');
-  const resolvedFromFailure = summary.filter(
-    test =>
-      test.latestClassification === 'passed' &&
-      (test.historicalBrokenCount ?? 0) > 0,
-  );
+function logClassificationDiagnostics(summary, metadata) {
+  const { brokenItems, flakyItems, watchItems, infraItems } = partitionSummary(summary);
 
   console.log('\n🧾 Classification diagnostics');
-  console.log(`  Unique tests observed: ${totalUniqueTests}`);
-  console.log(`  Latest state -> broken: ${currentlyBroken.length}, flaky: ${currentlyFlaky.length}, passed: ${latestPassed.length}`);
-  console.log(`  Resolved since earlier runs (had broken history, latest passed): ${resolvedFromFailure.length}`);
+  console.log(`  Lookback: ${env.LOOKBACK_DAYS} day(s)`);
+  console.log(`  Unique tests observed: ${summary.length}`);
+  console.log(
+    `  Buckets -> broken: ${brokenItems.length}, flaky: ${flakyItems.length}, watch: ${watchItems.length}, infra: ${infraItems.length}`,
+  );
+  console.log(`  CI runs: ${metadata.workflowCount} | Test-failure runs: ${metadata.testFailureRunCount}`);
+  console.log(`  Other CI failures: ${metadata.otherFailedRunCount}`);
 
-  if (resolvedFromFailure.length > 0) {
-    const preview = resolvedFromFailure
+  if (watchItems.length > 0) {
+    const preview = watchItems
       .slice(0, 5)
-      .map(test => `${test.name} (${test.projectName})`)
+      .map(test => {
+        const broken = test.historicalBrokenCount ?? 0;
+        const flaky = test.historicalFlakyCount ?? 0;
+        return `${test.name} (${test.projectName}, broken ${broken}, flaky ${flaky})`;
+      })
       .join('; ');
-    console.log(`  Sample resolved (broken→passed): ${preview}`);
+    console.log(`  Sample watch: ${preview}`);
   }
 }
 
@@ -173,6 +194,7 @@ async function main() {
   const workflowsScanned = getWorkflowIds();
 
   console.log('🧪 Playwright Test Health Report\n');
+  console.log(`Lookback: ${env.LOOKBACK_DAYS} day(s)`);
   console.log(`Time range: ${dateRange.from} to ${dateRange.to}`);
   console.log(`Workflows: ${workflowsScanned.join(', ')}\n`);
 
@@ -192,11 +214,20 @@ async function main() {
       return;
     }
 
+    const testFailureRunCount = countTestFailureRuns(findings);
+    const otherFailedRunCount = Math.max(0, failedRunCount - testFailureRunCount);
     const summary = summarizeTestHealth(findings);
-    logClassificationDiagnostics(summary);
+
+    logClassificationDiagnostics(summary, {
+      workflowCount: workflowRuns.length,
+      testFailureRunCount,
+      otherFailedRunCount,
+    });
+
     await sendSlackReport(summary, dateRange.display, {
       workflowCount: workflowRuns.length,
-      failedRunCount,
+      testFailureRunCount,
+      otherFailedRunCount,
       workflowsScanned,
     });
   } catch (error) {

.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs

@@ -0,0 +1,100 @@
+function historicalBroken(test) {
+  return test.historicalBrokenCount ?? test.brokenCount ?? 0;
+}
+
+function historicalFlaky(test) {
+  return test.historicalFlakyCount ?? test.flakyCount ?? 0;
+}
+
+export function instabilityScore(test) {
+  return historicalBroken(test) + historicalFlaky(test);
+}
+
+export function partitionSummary(summary) {
+  const infraItems = summary
+    .filter(test => test.latestClassification === 'infra')
+    .sort((a, b) => (b.infraCount ?? 0) - (a.infraCount ?? 0));
+
+  const brokenItems = summary
+    .filter(test => test.latestClassification === 'broken')
+    .sort((a, b) => historicalBroken(b) - historicalBroken(a));
+
+  const flakyItems = summary
+    .filter(test => test.latestClassification === 'flaky')
+    .sort((a, b) => historicalFlaky(b) - historicalFlaky(a));
+
+  const watchItems = summary
+    .filter(
+      test =>
+        test.latestClassification === 'passed' &&
+        (historicalBroken(test) > 0 || historicalFlaky(test) > 0),
+    )
+    .sort((a, b) => {
+      const rateA = instabilityScore(a) / Math.max(a.totalRuns ?? 1, 1);
+      const rateB = instabilityScore(b) / Math.max(b.totalRuns ?? 1, 1);
+      if (rateB !== rateA) {
+        return rateB - rateA;
+      }
+      return instabilityScore(b) - instabilityScore(a);
+    });
+
+  return { brokenItems, flakyItems, watchItems, infraItems };
+}
+
+export function allocateBucketSlots(topN, counts) {
+  const { broken = 0, flaky = 0, watch = 0, infra = 0 } = counts;
+
+  let maxBroken = Math.min(broken, Math.max(Math.ceil(topN * 0.4), broken > 0 ? 2 : 0));
+  let maxFlaky = Math.min(flaky, Math.max(Math.ceil(topN * 0.25), flaky > 0 ? 2 : 0));
+  let maxInfra = Math.min(infra, Math.max(Math.ceil(topN * 0.1), infra > 0 ? 1 : 0));
+  let maxWatch = Math.min(watch, topN - maxBroken - maxFlaky - maxInfra);
+
+  let remaining = topN - (maxBroken + maxFlaky + maxWatch + maxInfra);
+
+  const buckets = [
+    { key: 'watch', available: watch - maxWatch, max: maxWatch },
+    { key: 'broken', available: broken - maxBroken, max: maxBroken },
+    { key: 'flaky', available: flaky - maxFlaky, max: maxFlaky },
+    { key: 'infra', available: infra - maxInfra, max: maxInfra },
+  ].sort((a, b) => b.available - a.available);
+
+  for (const bucket of buckets) {
+    if (remaining <= 0) {
+      break;
+    }
+    const extra = Math.min(remaining, bucket.available);
+    bucket.max += extra;
+    remaining -= extra;
+  }
+
+  const byKey = Object.fromEntries(buckets.map(bucket => [bucket.key, bucket.max]));
+
+  return {
+    maxBroken: byKey.broken,
+    maxFlaky: byKey.flaky,
+    maxWatch: byKey.watch,
+    maxInfra: byKey.infra,
+  };
+}
+
+export function formatRunRate(count, totalRuns) {
+  if (!totalRuns || totalRuns <= 0) {
+    return `${count}x`;
+  }
+  return `${count}/${totalRuns} runs`;
+}
+
+export function formatWatchHistory(test) {
+  const parts = [];
+  const broken = historicalBroken(test);
+  const flaky = historicalFlaky(test);
+
+  if (broken > 0) {
+    parts.push(`broken ${formatRunRate(broken, test.totalRuns)}`);
+  }
+  if (flaky > 0) {
+    parts.push(`flaky ${formatRunRate(flaky, test.totalRuns)}`);
+  }
+
+  return parts.join(', ');
+}

.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs

@@ -39,6 +39,10 @@ function extractFirstFailureError(test) {
   return firstFailure?.error?.message ?? firstFailure?.errors?.[0]?.message ?? 'No error details';
 }
 
+function extractInfraError(error) {
+  return error?.message ?? error?.stack ?? 'Unknown setup error';
+}
+
 function walkSuites(suites, currentFile, findings, metadata) {
   for (const suite of suites ?? []) {
     const suiteFile = suite.file || currentFile;
@@ -67,6 +71,7 @@ function walkSuites(suites, currentFile, findings, metadata) {
           runId: metadata.runId,
           runUrl: metadata.runUrl,
           date: new Date(metadata.date),
+          artifactName: metadata.artifactName,
         });
       }
     }
@@ -75,8 +80,38 @@ function walkSuites(suites, currentFile, findings, metadata) {
   }
 }
 
+function parseInfraErrors(report, metadata, findings) {
+  const errors = Array.isArray(report?.errors) ? report.errors : [];
+  const suites = report?.suites ?? [];
+
+  if (errors.length === 0 || suites.length > 0) {
+    return;
+  }
+
+  for (const [index, error] of errors.entries()) {
+    const artifactLabel = metadata.artifactName ?? 'unknown-artifact';
+    const location = error?.location?.file ?? 'unknown-file';
+    const key = `infra::${artifactLabel}::${index}`;
+
+    findings.push({
+      key,
+      name: `Setup failure (${artifactLabel})`,
+      path: location,
+      projectName: 'infra',
+      classification: 'infra',
+      retries: 0,
+      error: extractInfraError(error),
+      runId: metadata.runId,
+      runUrl: metadata.runUrl,
+      date: new Date(metadata.date),
+      artifactName: metadata.artifactName,
+    });
+  }
+}
+
 export function parsePlaywrightJsonReport(report, metadata) {
   const findings = [];
   walkSuites(report?.suites ?? [], undefined, findings, metadata);
+  parseInfraErrors(report, metadata, findings);
   return findings;
 }