diff --git a/.github/actions/playwright-test-health-report/action.yml b/.github/actions/playwright-test-health-report/action.yml new file mode 100644 index 00000000..10306330 --- /dev/null +++ b/.github/actions/playwright-test-health-report/action.yml @@ -0,0 +1,94 @@ +name: Playwright Test Health Report +description: >- + Aggregates Playwright JSON test reports from GitHub Actions artifacts, + classifies flaky and failing tests, and posts a summary to Slack. + +inputs: + repository: + description: Repository name (e.g. metamask-mobile, metamask-extension) + required: true + workflow-ids: + description: >- + Comma-separated workflow files to analyze (e.g. ci.yml or main.yml). + Runs from all listed workflows are merged into one report. + required: true + github-token: + description: GitHub token with repo and actions:read + required: true + slack-webhook: + description: Slack incoming webhook URL + required: true + owner: + description: GitHub org/user. Defaults to MetaMask. + required: false + default: MetaMask + branch: + description: Branch to filter workflow runs + required: false + default: main + lookback-days: + description: >- + Number of days to look back for workflow runs. Use 1 for twice-daily + snapshots and 7 for a weekly summary. The same classification logic + applies regardless of window length. + required: false + default: '1' + artifact-name-prefix: + description: >- + Only download artifacts whose name starts with this prefix. + Example: playwright-json-report + required: false + default: playwright-json-report + results-file-pattern: + description: >- + File name prefix inside artifact zip to parse as JSON. + Example: playwright-report (matches playwright-report.json, + playwright-report-1.json, etc.) + required: false + default: playwright-report + top-n: + description: Maximum number of tests to include in the Slack report + required: false + default: '15' + report-title: + description: Slack header title override + required: false + default: Playwright Test Health Report + github-tools-repository: + required: false + default: ${{ github.action_repository }} + github-tools-ref: + required: false + default: ${{ github.action_ref }} + +runs: + using: composite + steps: + - uses: actions/checkout@v6 + with: + repository: ${{ inputs.github-tools-repository }} + ref: ${{ inputs.github-tools-ref }} + path: ./github-tools + - uses: actions/setup-node@v6 + with: + node-version-file: ./github-tools/.nvmrc + cache: yarn + cache-dependency-path: ./github-tools/yarn.lock + - shell: bash + working-directory: ./github-tools + run: corepack enable && yarn --immutable + - shell: bash + working-directory: ./github-tools + env: + OWNER: ${{ inputs.owner }} + REPOSITORY: ${{ inputs.repository }} + WORKFLOW_IDS: ${{ inputs.workflow-ids }} + BRANCH: ${{ inputs.branch }} + LOOKBACK_DAYS: ${{ inputs.lookback-days }} + ARTIFACT_NAME_PREFIX: ${{ inputs.artifact-name-prefix }} + RESULTS_FILE_PATTERN: ${{ inputs.results-file-pattern }} + TOP_N: ${{ inputs.top-n }} + REPORT_TITLE: ${{ inputs.report-title }} + GITHUB_TOKEN: ${{ inputs.github-token }} + SLACK_WEBHOOK: ${{ inputs.slack-webhook }} + run: node .github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs diff --git a/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs new file mode 100644 index 00000000..b3932ba2 --- /dev/null +++ b/.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs @@ -0,0 +1,248 @@ +#!/usr/bin/env node + +import { Octokit } from '@octokit/rest'; +import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs'; +import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs'; +import { createSlackBlocks, sendSlackBatched } from './lib/slack-test-health-blocks.mjs'; +import { partitionSummary } from './lib/classify-report-buckets.mjs'; +import { summarizeTestHealth } from './lib/summarize-test-health.mjs'; +import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs'; + +const githubToken = process.env.GITHUB_TOKEN; +if (!githubToken) { + throw new Error('Missing GITHUB_TOKEN env var'); +} + +const parsePositiveInt = (value, fallback) => { + const trimmed = value?.trim(); + if (!trimmed) { + return fallback; + } + const parsed = parseInt(trimmed, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const env = { + OWNER: process.env.OWNER || 'MetaMask', + REPOSITORY: process.env.REPOSITORY, + WORKFLOW_IDS: process.env.WORKFLOW_IDS, + BRANCH: process.env.BRANCH || 'main', + LOOKBACK_DAYS: parsePositiveInt(process.env.LOOKBACK_DAYS, 1), + ARTIFACT_NAME_PREFIX: process.env.ARTIFACT_NAME_PREFIX || 'playwright-json-report', + RESULTS_FILE_PATTERN: process.env.RESULTS_FILE_PATTERN || 'playwright-report', + TOP_N: parsePositiveInt(process.env.TOP_N, 15), + REPORT_TITLE: process.env.REPORT_TITLE || 'Playwright Test Health Report', + SLACK_WEBHOOK: process.env.SLACK_WEBHOOK || '', + GITHUB_TOKEN: githubToken, +}; + +if (!env.REPOSITORY) { + throw new Error('Missing REPOSITORY env var'); +} +if (!env.WORKFLOW_IDS) { + throw new Error('Missing WORKFLOW_IDS env var'); +} + +function getWorkflowIds() { + return env.WORKFLOW_IDS.split(',') + .map(value => value.trim()) + .filter(Boolean); +} + +function isTestFailureFinding(finding) { + return finding.classification === 'broken' || finding.classification === 'flaky' || finding.classification === 'infra'; +} + +function countTestFailureRuns(findings) { + return new Set(findings.filter(isTestFailureFinding).map(finding => finding.runId)).size; +} + +async function getMergedWorkflowRuns(github, dateRange) { + const workflowIds = getWorkflowIds(); + const runs = []; + + for (const workflowId of workflowIds) { + const workflowRuns = await getWorkflowRuns(github, { + owner: env.OWNER, + repo: env.REPOSITORY, + workflowId, + branch: env.BRANCH, + from: dateRange.from, + to: dateRange.to, + }); + runs.push(...workflowRuns); + } + + const dedupedRuns = Array.from(new Map(runs.map(run => [run.id, run])).values()); + dedupedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + return dedupedRuns; +} + +async function collectFindings(github, runs) { + const findings = []; + let matchingArtifacts = 0; + + for (const [index, run] of runs.entries()) { + console.log(`📦 Processing run ${index + 1}/${runs.length}: ${run.id}`); + + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { + owner: env.OWNER, + repo: env.REPOSITORY, + run_id: run.id, + }, + ); + + const matching = artifacts.filter(artifact => artifact.name.startsWith(env.ARTIFACT_NAME_PREFIX)); + matchingArtifacts += matching.length; + + if (matching.length === 0) { + console.log(` ⚠️ No matching artifacts found for run ${run.id}`); + continue; + } + + for (const artifact of matching) { + try { + const zip = await downloadArtifactZip(github, { + owner: env.OWNER, + repo: env.REPOSITORY, + artifactId: artifact.id, + }); + const jsonFiles = findFilesInZip(zip, env.RESULTS_FILE_PATTERN); + + if (jsonFiles.length === 0) { + console.log(` ⚠️ No ${env.RESULTS_FILE_PATTERN} file found in ${artifact.name}`); + continue; + } + + for (const file of jsonFiles) { + try { + const content = await file.buffer(); + const report = JSON.parse(content.toString()); + findings.push( + ...parsePlaywrightJsonReport(report, { + runId: run.id, + runUrl: run.html_url || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${run.id}`, + date: run.created_at, + artifactName: artifact.name, + }), + ); + } catch (error) { + console.log(` ❌ Invalid JSON in ${artifact.name}/${file.path}: ${error.message}`); + } + } + } catch (error) { + console.log(` ❌ Failed to process artifact ${artifact.name}: ${error.message}`); + } + } + } + + return { findings, matchingArtifacts }; +} + +async function sendSlackReport(summary, dateDisplay, metadata) { + if (!env.SLACK_WEBHOOK || !env.SLACK_WEBHOOK.startsWith('https://')) { + console.log('Skipping Slack notification'); + return; + } + + const blocks = createSlackBlocks(summary, dateDisplay, { + owner: env.OWNER, + repository: env.REPOSITORY, + branch: env.BRANCH, + reportTitle: env.REPORT_TITLE, + topN: env.TOP_N, + workflowsScanned: metadata.workflowsScanned, + workflowCount: metadata.workflowCount, + testFailureRunCount: metadata.testFailureRunCount, + otherFailedRunCount: metadata.otherFailedRunCount, + lookbackDays: env.LOOKBACK_DAYS, + }); + await sendSlackBatched(env.SLACK_WEBHOOK, blocks); + console.log('✅ Report sent to Slack successfully'); +} + +function logClassificationDiagnostics(summary, metadata) { + const { brokenItems, flakyItems, watchItems, infraItems } = partitionSummary(summary); + + console.log('\n🧾 Classification diagnostics'); + console.log(` Lookback: ${env.LOOKBACK_DAYS} day(s)`); + console.log(` Unique tests observed: ${summary.length}`); + console.log( + ` Buckets -> failing: ${brokenItems.length}, infra: ${infraItems.length}, flaky: ${flakyItems.length}, watch: ${watchItems.length}`, + ); + console.log(` CI runs: ${metadata.workflowCount} | Test-failure runs: ${metadata.testFailureRunCount}`); + console.log(` Other CI failures: ${metadata.otherFailedRunCount}`); + + if (watchItems.length > 0) { + const preview = watchItems + .slice(0, 5) + .map(test => { + const broken = test.historicalBrokenCount ?? 0; + const flaky = test.historicalFlakyCount ?? 0; + return `${test.name} (${test.projectName}, failed ${broken}, flaky ${flaky})`; + }) + .join('; '); + console.log(` Sample watch: ${preview}`); + } +} + +async function main() { + const github = new Octokit({ auth: env.GITHUB_TOKEN }); + const dateRange = getDateRange(env.LOOKBACK_DAYS); + const workflowsScanned = getWorkflowIds(); + + console.log('🧪 Playwright Test Health Report\n'); + console.log(`Lookback: ${env.LOOKBACK_DAYS} day(s)`); + console.log(`Time range: ${dateRange.from} to ${dateRange.to}`); + console.log(`Workflows: ${workflowsScanned.join(', ')}\n`); + + try { + const workflowRuns = await getMergedWorkflowRuns(github, dateRange); + + if (workflowRuns.length === 0) { + console.log('⚠️ No workflow runs found in lookback window.'); + return; + } + + const failedRunCount = workflowRuns.filter(run => run.conclusion === 'failure').length; + const { findings, matchingArtifacts } = await collectFindings(github, workflowRuns); + + if (matchingArtifacts === 0) { + console.log('⚠️ No matching artifacts found.'); + return; + } + + const testFailureRunCount = countTestFailureRuns(findings); + const otherFailedRunCount = Math.max(0, failedRunCount - testFailureRunCount); + const summary = summarizeTestHealth(findings); + + logClassificationDiagnostics(summary, { + workflowCount: workflowRuns.length, + testFailureRunCount, + otherFailedRunCount, + }); + + await sendSlackReport(summary, dateRange.display, { + workflowCount: workflowRuns.length, + testFailureRunCount, + otherFailedRunCount, + workflowsScanned, + }); + } catch (error) { + console.error('❌ Error:', error.message); + if (error.status === 401) { + console.log('\n💡 GitHub token is unauthorized. Ensure it has repo and actions:read permissions.'); + } + if (error.status === 404) { + console.log('\n💡 One or more workflows were not found. Check WORKFLOW_IDS values.'); + } + process.exit(1); + } +} + +main().catch(error => { + console.error('\n❌ Unexpected error:', error); + process.exit(1); +}); diff --git a/.github/actions/playwright-test-health-report/lib/artifact-download.mjs b/.github/actions/playwright-test-health-report/lib/artifact-download.mjs new file mode 100644 index 00000000..9e11987a --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/artifact-download.mjs @@ -0,0 +1,21 @@ +import path from 'path'; +import unzipper from 'unzipper'; + +export async function downloadArtifactZip(github, { owner, repo, artifactId }) { + const response = await github.rest.actions.downloadArtifact({ + owner, + repo, + artifact_id: artifactId, + archive_format: 'zip', + }); + + const buffer = Buffer.from(response.data); + return unzipper.Open.buffer(buffer); +} + +export function findFilesInZip(zip, filePattern) { + return zip.files.filter(file => { + const fileName = path.basename(file.path); + return file.path.startsWith(filePattern) || fileName.startsWith(filePattern); + }); +} diff --git a/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs new file mode 100644 index 00000000..c3c9c1c5 --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/classify-report-buckets.mjs @@ -0,0 +1,100 @@ +function historicalBroken(test) { + return test.historicalBrokenCount ?? test.brokenCount ?? 0; +} + +function historicalFlaky(test) { + return test.historicalFlakyCount ?? test.flakyCount ?? 0; +} + +export function instabilityScore(test) { + return historicalBroken(test) + historicalFlaky(test); +} + +export function partitionSummary(summary) { + const infraItems = summary + .filter(test => test.latestClassification === 'infra') + .sort((a, b) => (b.infraCount ?? 0) - (a.infraCount ?? 0)); + + const brokenItems = summary + .filter(test => test.latestClassification === 'broken') + .sort((a, b) => historicalBroken(b) - historicalBroken(a)); + + const flakyItems = summary + .filter(test => test.latestClassification === 'flaky') + .sort((a, b) => historicalFlaky(b) - historicalFlaky(a)); + + const watchItems = summary + .filter( + test => + test.latestClassification === 'passed' && + (historicalBroken(test) > 0 || historicalFlaky(test) > 0), + ) + .sort((a, b) => { + const rateA = instabilityScore(a) / Math.max(a.totalRuns ?? 1, 1); + const rateB = instabilityScore(b) / Math.max(b.totalRuns ?? 1, 1); + if (rateB !== rateA) { + return rateB - rateA; + } + return instabilityScore(b) - instabilityScore(a); + }); + + return { brokenItems, flakyItems, watchItems, infraItems }; +} + +export function allocateBucketSlots(topN, counts) { + const { broken = 0, flaky = 0, watch = 0, infra = 0 } = counts; + + let maxBroken = Math.min(broken, Math.max(Math.ceil(topN * 0.4), broken > 0 ? 2 : 0)); + let maxFlaky = Math.min(flaky, Math.max(Math.ceil(topN * 0.25), flaky > 0 ? 2 : 0)); + let maxInfra = Math.min(infra, Math.max(Math.ceil(topN * 0.1), infra > 0 ? 1 : 0)); + let maxWatch = Math.min(watch, topN - maxBroken - maxFlaky - maxInfra); + + let remaining = topN - (maxBroken + maxFlaky + maxWatch + maxInfra); + + const buckets = [ + { key: 'watch', available: watch - maxWatch, max: maxWatch }, + { key: 'broken', available: broken - maxBroken, max: maxBroken }, + { key: 'flaky', available: flaky - maxFlaky, max: maxFlaky }, + { key: 'infra', available: infra - maxInfra, max: maxInfra }, + ].sort((a, b) => b.available - a.available); + + for (const bucket of buckets) { + if (remaining <= 0) { + break; + } + const extra = Math.min(remaining, bucket.available); + bucket.max += extra; + remaining -= extra; + } + + const byKey = Object.fromEntries(buckets.map(bucket => [bucket.key, bucket.max])); + + return { + maxBroken: byKey.broken, + maxFlaky: byKey.flaky, + maxWatch: byKey.watch, + maxInfra: byKey.infra, + }; +} + +export function formatRunRate(count, totalRuns) { + if (!totalRuns || totalRuns <= 0) { + return `${count}x`; + } + return `${count}/${totalRuns} runs`; +} + +export function formatWatchHistory(test) { + const parts = []; + const broken = historicalBroken(test); + const flaky = historicalFlaky(test); + + if (broken > 0) { + parts.push(`failed ${formatRunRate(broken, test.totalRuns)}`); + } + if (flaky > 0) { + parts.push(`flaky ${formatRunRate(flaky, test.totalRuns)}`); + } + + return parts.join(', '); +} diff --git a/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs new file mode 100644 index 00000000..38d09c6e --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/parse-playwright-json.mjs @@ -0,0 +1,117 @@ +function isFailureResult(result) { + return result.status === 'failed' || result.status === 'timedOut'; +} + +function classifyTest(test) { + if (test.status === 'expected') { + return 'passed'; + } + if (test.status === 'unexpected') { + return 'broken'; + } + if (test.status === 'flaky') { + return 'flaky'; + } + if (test.status === 'skipped') { + return null; + } + + const results = Array.isArray(test.results) ? test.results : []; + if (results.length <= 1) { + return null; + } + + const lastResult = results[results.length - 1]; + const anyFailed = results.some(isFailureResult); + if (!anyFailed) { + return null; + } + + return lastResult?.status === 'passed' ? 'flaky' : 'broken'; +} + +function extractFirstFailureError(test) { + const results = Array.isArray(test.results) ? test.results : []; + const firstFailure = results.find(isFailureResult); + if (!firstFailure) { + return 'No error details'; + } + return firstFailure?.error?.message ?? firstFailure?.errors?.[0]?.message ?? 'No error details'; +} + +function extractInfraError(error) { + return error?.message ?? error?.stack ?? 'Unknown setup error'; +} + +function walkSuites(suites, currentFile, findings, metadata) { + for (const suite of suites ?? []) { + const suiteFile = suite.file || currentFile; + + for (const spec of suite.specs ?? []) { + for (const test of spec.tests ?? []) { + const classification = classifyTest(test); + if (!classification) { + continue; + } + + const projectName = test.projectName || 'default'; + const specFile = spec.file || suiteFile || 'unknown-file'; + const testTitle = test.title || spec.title || 'unknown-test'; + const key = `${projectName}::${specFile}::${testTitle}`; + const results = Array.isArray(test.results) ? test.results : []; + + findings.push({ + key, + name: testTitle, + path: specFile, + projectName, + classification, + retries: Math.max(0, results.length - 1), + error: extractFirstFailureError(test), + runId: metadata.runId, + runUrl: metadata.runUrl, + date: new Date(metadata.date), + artifactName: metadata.artifactName, + }); + } + } + + walkSuites(suite.suites, suiteFile, findings, metadata); + } +} + +function parseInfraErrors(report, metadata, findings) { + const errors = Array.isArray(report?.errors) ? report.errors : []; + const suites = report?.suites ?? []; + + if (errors.length === 0 || suites.length > 0) { + return; + } + + for (const [index, error] of errors.entries()) { + const artifactLabel = metadata.artifactName ?? 'unknown-artifact'; + const location = error?.location?.file ?? 'unknown-file'; + const key = `infra::${artifactLabel}::${index}`; + + findings.push({ + key, + name: `Setup failure (${artifactLabel})`, + path: location, + projectName: 'infra', + classification: 'infra', + retries: 0, + error: extractInfraError(error), + runId: metadata.runId, + runUrl: metadata.runUrl, + date: new Date(metadata.date), + artifactName: metadata.artifactName, + }); + } +} + +export function parsePlaywrightJsonReport(report, metadata) { + const findings = []; + walkSuites(report?.suites ?? [], undefined, findings, metadata); + parseInfraErrors(report, metadata, findings); + return findings; +} diff --git a/.github/actions/playwright-test-health-report/lib/report-health.test.mjs b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs new file mode 100644 index 00000000..a382a568 --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/report-health.test.mjs @@ -0,0 +1,100 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + allocateBucketSlots, + formatWatchHistory, + partitionSummary, +} from './classify-report-buckets.mjs'; +import { summarizeTestHealth } from './summarize-test-health.mjs'; + +describe('summarizeTestHealth', () => { + it('keeps historical counts when latest run passed', () => { + const summary = summarizeTestHealth([ + { + key: 'ios::file.spec.ts::deletes account', + name: 'deletes account', + path: 'file.spec.ts', + projectName: 'ios-smoke', + classification: 'broken', + retries: 1, + error: 'timeout', + runId: '1', + runUrl: 'https://example.com/1', + date: new Date('2026-06-16T10:00:00Z'), + }, + { + key: 'ios::file.spec.ts::deletes account', + name: 'deletes account', + path: 'file.spec.ts', + projectName: 'ios-smoke', + classification: 'flaky', + retries: 1, + error: 'timeout', + runId: '2', + runUrl: 'https://example.com/2', + date: new Date('2026-06-16T12:00:00Z'), + }, + { + key: 'ios::file.spec.ts::deletes account', + name: 'deletes account', + path: 'file.spec.ts', + projectName: 'ios-smoke', + classification: 'passed', + retries: 0, + error: '', + runId: '3', + runUrl: 'https://example.com/3', + date: new Date('2026-06-17T10:00:00Z'), + }, + ]); + + const test = summary[0]; + assert.equal(test.latestClassification, 'passed'); + assert.equal(test.historicalBrokenCount, 1); + assert.equal(test.historicalFlakyCount, 1); + assert.equal(test.totalRuns, 3); + }); +}); + +describe('partitionSummary', () => { + it('puts passed tests with flaky history into watch', () => { + const { watchItems, flakyItems, brokenItems } = partitionSummary([ + { + latestClassification: 'passed', + historicalBrokenCount: 0, + historicalFlakyCount: 4, + brokenCount: 0, + flakyCount: 4, + totalRuns: 10, + name: 'exports srp', + }, + ]); + + assert.equal(watchItems.length, 1); + assert.equal(flakyItems.length, 0); + assert.equal(brokenItems.length, 0); + }); +}); + +describe('allocateBucketSlots', () => { + it('gives spare capacity to watch when nothing is currently broken', () => { + const slots = allocateBucketSlots(15, { broken: 0, flaky: 0, watch: 13, infra: 0 }); + + assert.equal(slots.maxBroken, 0); + assert.equal(slots.maxWatch, 13); + assert.equal(slots.maxBroken + slots.maxFlaky + slots.maxWatch + slots.maxInfra, 13); + }); +}); + +describe('formatWatchHistory', () => { + it('includes both broken and flaky history', () => { + const text = formatWatchHistory({ + historicalBrokenCount: 2, + historicalFlakyCount: 3, + totalRuns: 8, + }); + + assert.match(text, /failed 2\/8 runs/); + assert.match(text, /flaky 3\/8 runs/); + }); +}); diff --git a/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs new file mode 100644 index 00000000..de3dbca0 --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs @@ -0,0 +1,254 @@ +import { IncomingWebhook } from '@slack/webhook'; +import { + allocateBucketSlots, + formatRunRate, + formatWatchHistory, + partitionSummary, +} from './classify-report-buckets.mjs'; + +export function normalizeErrorForSlack(message, maxLength = 120) { + if (!message) { + return 'No error details'; + } + + const withoutEmojiShortcodes = String(message).replace(/:[a-z0-9_+\-]+:/gi, ' '); + const firstMeaningfulLine = withoutEmojiShortcodes + .split(/\r?\n/) + .map(line => line.trim()) + .find(Boolean); + const normalized = (firstMeaningfulLine || withoutEmojiShortcodes) + .replace(/\s+/g, ' ') + .trim(); + + if (!normalized) { + return 'No error details'; + } + + return normalized.length > maxLength ? `${normalized.substring(0, maxLength - 3)}...` : normalized; +} + +export function truncateError(message, maxLength = 120) { + return normalizeErrorForSlack(message, maxLength); +} + +function buildRunUrl(owner, repository, test, kind) { + const runUrlField = { + broken: 'lastBrokenRunUrl', + flaky: 'lastFlakyRunUrl', + infra: 'lastInfraRunUrl', + }[kind]; + const runIdField = { + broken: 'lastBrokenRunId', + flaky: 'lastFlakyRunId', + infra: 'lastInfraRunId', + }[kind]; + + return ( + test[runUrlField] || + (test[runIdField] + ? `https://github.com/${owner}/${repository}/actions/runs/${test[runIdField]}` + : null) + ); +} + +function pushSectionHeader(blocks, emoji, title) { + blocks.push({ + type: 'rich_text', + elements: [ + { + type: 'rich_text_section', + elements: [ + ...(emoji ? [{ type: 'emoji', name: emoji }] : []), + ...(emoji ? [{ type: 'text', text: ' ' }] : []), + { type: 'text', text: title, style: { bold: true } }, + ], + }, + ], + }); +} + +function pushTestLine(blocks, { index, owner, repository, branch, test, statusText, runKind }) { + const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`; + const runUrl = buildRunUrl(owner, repository, test, runKind); + + blocks.push({ + type: 'rich_text', + elements: [ + { + type: 'rich_text_section', + elements: [ + { type: 'text', text: `${index}. ` }, + { type: 'link', url: fileUrl, text: test.name }, + { type: 'text', text: ` (${test.projectName}) ` }, + { type: 'text', text: statusText, style: { bold: true } }, + ...(runUrl ? [{ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' }] : []), + ], + }, + ], + }); +} + +function pushErrorLine(blocks, message) { + blocks.push({ + type: 'rich_text', + elements: [ + { + type: 'rich_text_section', + elements: [{ type: 'text', text: truncateError(message), style: { italic: true } }], + }, + ], + }); +} + +export function createSlackBlocks(summary, dateDisplay, options) { + const { + owner, + repository, + branch, + reportTitle, + topN, + workflowsScanned, + workflowCount, + testFailureRunCount, + otherFailedRunCount, + lookbackDays = 1, + } = options; + + const { brokenItems, flakyItems, watchItems, infraItems } = partitionSummary(summary); + const { maxBroken, maxFlaky, maxWatch, maxInfra } = allocateBucketSlots(topN, { + broken: brokenItems.length, + flaky: flakyItems.length, + watch: watchItems.length, + infra: infraItems.length, + }); + + const broken = brokenItems.slice(0, maxBroken); + const infra = infraItems.slice(0, maxInfra); + const flaky = flakyItems.slice(0, maxFlaky); + const watch = watchItems.slice(0, maxWatch); + const topItems = [...broken, ...infra, ...flaky, ...watch]; + + const blocks = [ + { + type: 'header', + text: { + type: 'plain_text', + text: `${reportTitle} - Top ${topN}`, + emoji: true, + }, + }, + { + type: 'context', + elements: [ + { + type: 'mrkdwn', + text: + `Period (UTC): ${dateDisplay} | Lookback: ${lookbackDays} day(s) | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')}` + + `\nCI runs: ${workflowCount} on ${branch} | Test-failure runs: ${testFailureRunCount} | Other CI failures: ${otherFailedRunCount}` + + `\nTests: ${brokenItems.length} failing, ${infraItems.length} infra, ${flakyItems.length} flaky, ${watchItems.length} watch` + + ` | Showing ${broken.length}, ${infra.length}, ${flaky.length}, ${watch.length}`, + }, + ], + }, + { type: 'divider' }, + ]; + + if (topItems.length === 0) { + blocks.push({ + type: 'rich_text', + elements: [ + { + type: 'rich_text_section', + elements: [{ type: 'text', text: 'No failing, infra, flaky, or watch issues found ✅' }], + }, + ], + }); + return blocks; + } + + let itemIndex = 0; + + const sections = [ + { + items: broken, + emoji: 'x', + title: 'Failing (latest run failed)', + runKind: 'broken', + statusText: test => + `failed ${formatRunRate(test.historicalBrokenCount ?? test.brokenCount, test.totalRuns)}`, + error: test => test.lastBrokenError, + }, + { + items: infra, + emoji: 'warning', + title: 'Infra (setup failed, no tests ran)', + runKind: 'infra', + statusText: test => + `setup failed ${formatRunRate(test.historicalInfraCount ?? test.infraCount, test.totalRuns)}`, + error: test => test.lastInfraError, + }, + { + items: flaky, + emoji: 'large_yellow_circle', + title: 'Flaky (latest run flaky)', + runKind: 'flaky', + statusText: test => + `flaky ${formatRunRate(test.historicalFlakyCount ?? test.flakyCount, test.totalRuns)}`, + error: test => test.lastFlakyError, + }, + { + items: watch, + emoji: 'large_green_circle', + title: 'Watch (unstable in window, passing now)', + runKind: 'broken', + statusText: test => `now passing (${formatWatchHistory(test)})`, + error: () => null, + }, + ]; + + let previousSectionHadItems = false; + + for (const section of sections) { + if (section.items.length === 0) { + continue; + } + + if (previousSectionHadItems) { + blocks.push({ type: 'divider' }); + } + + pushSectionHeader(blocks, section.emoji, section.title); + + for (const test of section.items) { + itemIndex += 1; + pushTestLine(blocks, { + index: itemIndex, + owner, + repository, + branch, + test, + statusText: section.statusText(test), + runKind: section.runKind, + }); + + const error = section.error(test); + if (error) { + pushErrorLine(blocks, error); + } + } + + previousSectionHadItems = true; + } + + return blocks; +} + +export async function sendSlackBatched(webhookUrl, blocks) { + const webhook = new IncomingWebhook(webhookUrl); + const batchSize = 50; + + for (let i = 0; i < blocks.length; i += batchSize) { + const batch = blocks.slice(i, i + batchSize); + await webhook.send({ blocks: batch }); + } +} diff --git a/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs new file mode 100644 index 00000000..0978eeb3 --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs @@ -0,0 +1,114 @@ +function createEntry(finding) { + return { + key: finding.key, + name: finding.name, + path: finding.path, + projectName: finding.projectName, + brokenCount: 0, + flakyCount: 0, + passedCount: 0, + infraCount: 0, + totalRuns: 0, + totalRetries: 0, + seenRunIds: new Set(), + lastSeen: finding.date, + latestClassification: finding.classification, + lastBrokenRunId: undefined, + lastBrokenRunUrl: undefined, + lastBrokenError: undefined, + lastFlakyRunId: undefined, + lastFlakyRunUrl: undefined, + lastFlakyError: undefined, + lastInfraRunId: undefined, + lastInfraRunUrl: undefined, + lastInfraError: undefined, + }; +} + +function applyClassification(entry, finding) { + if (!entry.seenRunIds.has(finding.runId)) { + entry.seenRunIds.add(finding.runId); + entry.totalRuns += 1; + } + + entry.totalRetries += finding.retries; + + if (finding.classification === 'broken') { + entry.brokenCount += 1; + entry.lastBrokenRunId = finding.runId; + entry.lastBrokenRunUrl = finding.runUrl; + entry.lastBrokenError = finding.error; + } else if (finding.classification === 'flaky') { + entry.flakyCount += 1; + entry.lastFlakyRunId = finding.runId; + entry.lastFlakyRunUrl = finding.runUrl; + entry.lastFlakyError = finding.error; + } else if (finding.classification === 'infra') { + entry.infraCount += 1; + entry.lastInfraRunId = finding.runId; + entry.lastInfraRunUrl = finding.runUrl; + entry.lastInfraError = finding.error; + } else if (finding.classification === 'passed') { + entry.passedCount += 1; + } + + if (finding.date >= entry.lastSeen) { + entry.lastSeen = finding.date; + entry.latestClassification = finding.classification; + } +} + +export function summarizeTestHealth(findings) { + const summary = new Map(); + + for (const finding of findings) { + const existing = summary.get(finding.key); + if (existing) { + applyClassification(existing, finding); + continue; + } + + const entry = createEntry(finding); + applyClassification(entry, finding); + summary.set(finding.key, entry); + } + + return Array.from(summary.values()) + .map(item => { + const { seenRunIds, ...rest } = item; + return { + ...rest, + historicalBrokenCount: item.brokenCount, + historicalFlakyCount: item.flakyCount, + historicalInfraCount: item.infraCount, + }; + }) + .sort((a, b) => { + const latestRank = (classification, item) => { + if (classification === 'broken' || classification === 'infra') { + return 3; + } + if (classification === 'flaky') { + return 2; + } + if ((item.brokenCount ?? 0) > 0 || (item.flakyCount ?? 0) > 0) { + return 1; + } + return 0; + }; + + const rankA = latestRank(a.latestClassification, a); + const rankB = latestRank(b.latestClassification, b); + if (rankA !== rankB) { + return rankB - rankA; + } + + const instabilityA = a.brokenCount + a.flakyCount; + const instabilityB = b.brokenCount + b.flakyCount; + if (instabilityA !== instabilityB) { + return instabilityB - instabilityA; + } + + return b.totalRetries - a.totalRetries; + }); +} diff --git a/.github/actions/playwright-test-health-report/lib/workflow-runs.mjs b/.github/actions/playwright-test-health-report/lib/workflow-runs.mjs new file mode 100644 index 00000000..2536f792 --- /dev/null +++ b/.github/actions/playwright-test-health-report/lib/workflow-runs.mjs @@ -0,0 +1,52 @@ +export function getDateRange(lookbackDays = 1) { + const today = new Date(); + const daysAgo = new Date(today.getTime() - (lookbackDays * 24 * 60 * 60 * 1000)); + + const fromDisplay = daysAgo.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + }); + + const toDisplay = today.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + }); + + return { + from: daysAgo.toISOString(), + to: today.toISOString(), + display: `${fromDisplay} - ${toDisplay}`, + }; +} + +export async function getWorkflowRuns(github, { owner, repo, workflowId, branch, from, to }) { + try { + const runs = await github.paginate( + github.rest.actions.listWorkflowRuns, + { + owner, + repo, + workflow_id: workflowId, + branch, + created: `${from}..${to}`, + per_page: 100, + }, + ); + + const completedRuns = runs.filter( + run => run.status === 'completed' && (run.event === 'push' || run.event === 'schedule'), + ); + completedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + + return completedRuns; + } catch (error) { + if (error.status === 404) { + throw new Error(`Workflow '${workflowId}' not found in ${owner}/${repo}`); + } + throw error; + } +}