Skip to content

Commit 4bb5ba7

Browse files
cmd-obcmd-ob
committed
refactor playwright test health Slack blocks and broken classification
1 parent aaf10ac commit 4bb5ba7

3 files changed

Lines changed: 206 additions & 97 deletions

File tree

‎.github/actions/playwright-test-health-report/create-playwright-test-health-report.mjs‎

Lines changed: 11 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import { Octokit } from '@octokit/rest';
44
import { downloadArtifactZip, findFilesInZip } from './lib/artifact-download.mjs';
55
import { parsePlaywrightJsonReport } from './lib/parse-playwright-json.mjs';
6-
import { sendSlackBatched, truncateError } from './lib/slack-test-health-blocks.mjs';
6+
import { createSlackBlocks, sendSlackBatched } from './lib/slack-test-health-blocks.mjs';
77
import { summarizeTestHealth } from './lib/summarize-test-health.mjs';
88
import { getDateRange, getWorkflowRuns } from './lib/workflow-runs.mjs';
99

@@ -122,103 +122,22 @@ async function collectFindings(github, runs) {
122122
return { findings, matchingArtifacts };
123123
}
124124

125-
function createSlackBlocks(summary, dateDisplay, metadata) {
126-
const topItems = summary.slice(0, env.TOP_N);
127-
const broken = topItems.filter(item => item.brokenCount > 0);
128-
const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
129-
130-
const blocks = [
131-
{
132-
type: 'header',
133-
text: {
134-
type: 'plain_text',
135-
text: `${env.REPORT_TITLE} — Top ${env.TOP_N}`,
136-
emoji: true,
137-
},
138-
},
139-
{
140-
type: 'context',
141-
elements: [
142-
{
143-
type: 'mrkdwn',
144-
text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Workflows: ${metadata.workflowsScanned.join(', ')} | Failed CI Runs: ${metadata.failedRunCount}/${metadata.workflowCount} from ${env.BRANCH}`,
145-
},
146-
],
147-
},
148-
{ type: 'divider' },
149-
];
150-
151-
if (topItems.length === 0) {
152-
blocks.push({
153-
type: 'section',
154-
text: {
155-
type: 'mrkdwn',
156-
text: 'No flaky or broken tests found ✅',
157-
},
158-
});
159-
return blocks;
160-
}
161-
162-
if (broken.length > 0) {
163-
blocks.push({
164-
type: 'section',
165-
text: { type: 'mrkdwn', text: '*❌ Broken (failed all retries)*' },
166-
});
167-
168-
broken.forEach((test, index) => {
169-
const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
170-
const runUrl = test.lastBrokenRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastBrokenRunId}`;
171-
blocks.push({
172-
type: 'section',
173-
text: {
174-
type: 'mrkdwn',
175-
text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — broken: *${test.brokenCount}*, flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
176-
},
177-
});
178-
blocks.push({
179-
type: 'section',
180-
text: { type: 'mrkdwn', text: `_${truncateError(test.lastBrokenError)}_` },
181-
});
182-
});
183-
}
184-
185-
if (broken.length > 0 && flaky.length > 0) {
186-
blocks.push({ type: 'divider' });
187-
}
188-
189-
if (flaky.length > 0) {
190-
blocks.push({
191-
type: 'section',
192-
text: { type: 'mrkdwn', text: '*🟡 Flaky (passed on retry)*' },
193-
});
194-
195-
flaky.forEach((test, index) => {
196-
const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
197-
const runUrl = test.lastFlakyRunUrl || `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastFlakyRunId}`;
198-
blocks.push({
199-
type: 'section',
200-
text: {
201-
type: 'mrkdwn',
202-
text: `${index + 1}. <${fileUrl}|${test.name}> *(${test.projectName})* — flaky: *${test.flakyCount}*, retries: *${test.totalRetries}*${runUrl ? ` | <${runUrl}|run log>` : ''}`,
203-
},
204-
});
205-
blocks.push({
206-
type: 'section',
207-
text: { type: 'mrkdwn', text: `_${truncateError(test.lastFlakyError)}_` },
208-
});
209-
});
210-
}
211-
212-
return blocks;
213-
}
214-
215125
async function sendSlackReport(summary, dateDisplay, metadata) {
216126
if (!env.SLACK_WEBHOOK || !env.SLACK_WEBHOOK.startsWith('https://')) {
217127
console.log('Skipping Slack notification');
218128
return;
219129
}
220130

221-
const blocks = createSlackBlocks(summary, dateDisplay, metadata);
131+
const blocks = createSlackBlocks(summary, dateDisplay, {
132+
owner: env.OWNER,
133+
repository: env.REPOSITORY,
134+
branch: env.BRANCH,
135+
reportTitle: env.REPORT_TITLE,
136+
topN: env.TOP_N,
137+
workflowsScanned: metadata.workflowsScanned,
138+
failedRunCount: metadata.failedRunCount,
139+
workflowCount: metadata.workflowCount,
140+
});
222141
await sendSlackBatched(env.SLACK_WEBHOOK, blocks);
223142
console.log('✅ Report sent to Slack successfully');
224143
}

‎.github/actions/playwright-test-health-report/lib/slack-test-health-blocks.mjs‎

Lines changed: 182 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,190 @@
11
import { IncomingWebhook } from '@slack/webhook';
22

3-
export function truncateError(message, maxLength = 150) {
3+
export function normalizeErrorForSlack(message, maxLength = 120) {
44
if (!message) {
55
return 'No error details';
66
}
7-
return message.length > maxLength ? `${message.substring(0, maxLength)}...` : message;
7+
8+
const withoutEmojiShortcodes = String(message).replace(/:[a-z0-9_+\-]+:/gi, ' ');
9+
const firstMeaningfulLine = withoutEmojiShortcodes
10+
.split(/\r?\n/)
11+
.map(line => line.trim())
12+
.find(Boolean);
13+
const normalized = (firstMeaningfulLine || withoutEmojiShortcodes)
14+
.replace(/\s+/g, ' ')
15+
.trim();
16+
17+
if (!normalized) {
18+
return 'No error details';
19+
}
20+
21+
return normalized.length > maxLength ? `${normalized.substring(0, maxLength - 3)}...` : normalized;
22+
}
23+
24+
export function truncateError(message, maxLength = 120) {
25+
return normalizeErrorForSlack(message, maxLength);
26+
}
27+
28+
export function createSlackBlocks(summary, dateDisplay, options) {
29+
const {
30+
owner,
31+
repository,
32+
branch,
33+
reportTitle,
34+
topN,
35+
workflowsScanned,
36+
failedRunCount,
37+
workflowCount,
38+
} = options;
39+
40+
const topItems = summary.slice(0, topN);
41+
const broken = topItems.filter(item => item.brokenCount > 0);
42+
const flaky = topItems.filter(item => item.brokenCount === 0 && item.flakyCount > 0);
43+
44+
const blocks = [
45+
{
46+
type: 'header',
47+
text: {
48+
type: 'plain_text',
49+
text: `${reportTitle} - Top ${topN}`,
50+
emoji: true,
51+
},
52+
},
53+
{
54+
type: 'context',
55+
elements: [
56+
{
57+
type: 'mrkdwn',
58+
text:
59+
`Period (UTC): ${dateDisplay} | Repo: ${repository} | Workflows: ${workflowsScanned.join(', ')} | ` +
60+
`Failed CI Runs: ${failedRunCount}/${workflowCount} from ${branch}` +
61+
`\nFound: ${broken.length} broken, ${flaky.length} flaky`,
62+
},
63+
],
64+
},
65+
{ type: 'divider' },
66+
];
67+
68+
if (topItems.length === 0) {
69+
blocks.push({
70+
type: 'rich_text',
71+
elements: [
72+
{
73+
type: 'rich_text_section',
74+
elements: [{ type: 'text', text: 'No flaky or broken tests found ✅' }],
75+
},
76+
],
77+
});
78+
return blocks;
79+
}
80+
81+
if (broken.length > 0) {
82+
blocks.push({
83+
type: 'rich_text',
84+
elements: [
85+
{
86+
type: 'rich_text_section',
87+
elements: [
88+
{ type: 'emoji', name: 'x' },
89+
{ type: 'text', text: ' ' },
90+
{ type: 'text', text: 'Broken', style: { bold: true } },
91+
],
92+
},
93+
],
94+
});
95+
96+
broken.forEach((test, index) => {
97+
const globalIndex = index + 1;
98+
const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
99+
const runUrl =
100+
test.lastBrokenRunUrl ||
101+
(test.lastBrokenRunId
102+
? `https://github.com/${owner}/${repository}/actions/runs/${test.lastBrokenRunId}`
103+
: null);
104+
const elements = [
105+
{ type: 'text', text: ` ${globalIndex}. ` },
106+
{ type: 'link', url: fileUrl, text: test.name },
107+
{ type: 'text', text: ` (${test.projectName})` },
108+
{ type: 'text', text: ` failed ${test.brokenCount}x`, style: { bold: true } },
109+
];
110+
111+
if (runUrl) {
112+
elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
113+
}
114+
115+
blocks.push({
116+
type: 'rich_text',
117+
elements: [{ type: 'rich_text_section', elements }],
118+
});
119+
120+
blocks.push({
121+
type: 'rich_text',
122+
elements: [
123+
{
124+
type: 'rich_text_section',
125+
elements: [{ type: 'text', text: ` ${truncateError(test.lastBrokenError)}`, style: { italic: true } }],
126+
},
127+
],
128+
});
129+
});
130+
}
131+
132+
if (broken.length > 0 && flaky.length > 0) {
133+
blocks.push({ type: 'divider' });
134+
}
135+
136+
if (flaky.length > 0) {
137+
blocks.push({
138+
type: 'rich_text',
139+
elements: [
140+
{
141+
type: 'rich_text_section',
142+
elements: [
143+
{ type: 'emoji', name: 'large_yellow_circle' },
144+
{ type: 'text', text: ' ' },
145+
{ type: 'text', text: 'Flaky', style: { bold: true } },
146+
],
147+
},
148+
],
149+
});
150+
151+
flaky.forEach((test, index) => {
152+
const globalIndex = broken.length + index + 1;
153+
const fileUrl = `https://github.com/${owner}/${repository}/blob/${branch}/${test.path}`;
154+
const runUrl =
155+
test.lastFlakyRunUrl ||
156+
(test.lastFlakyRunId
157+
? `https://github.com/${owner}/${repository}/actions/runs/${test.lastFlakyRunId}`
158+
: null);
159+
const elements = [
160+
{ type: 'text', text: ` ${globalIndex}. ` },
161+
{ type: 'link', url: fileUrl, text: test.name },
162+
{ type: 'text', text: ` (${test.projectName})` },
163+
{ type: 'text', text: ` flaky ${test.flakyCount}x`, style: { bold: true } },
164+
];
165+
166+
if (runUrl) {
167+
elements.push({ type: 'text', text: ' - ' }, { type: 'link', url: runUrl, text: 'run log' });
168+
}
169+
170+
blocks.push({
171+
type: 'rich_text',
172+
elements: [{ type: 'rich_text_section', elements }],
173+
});
174+
175+
blocks.push({
176+
type: 'rich_text',
177+
elements: [
178+
{
179+
type: 'rich_text_section',
180+
elements: [{ type: 'text', text: ` ${truncateError(test.lastFlakyError)}`, style: { italic: true } }],
181+
},
182+
],
183+
});
184+
});
185+
}
186+
187+
return blocks;
8188
}
9189

10190
export async function sendSlackBatched(webhookUrl, blocks) {

‎.github/actions/playwright-test-health-report/lib/summarize-test-health.mjs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ export function summarizeTestHealth(findings) {
1818
}
1919

2020
existing.totalRetries += finding.retries;
21-
if (finding.date > existing.lastSeen) {
21+
if (finding.date >= existing.lastSeen) {
2222
existing.lastSeen = finding.date;
23+
existing.latestClassification = finding.classification;
2324
}
2425
continue;
2526
}
@@ -33,6 +34,7 @@ export function summarizeTestHealth(findings) {
3334
flakyCount: finding.classification === 'flaky' ? 1 : 0,
3435
totalRetries: finding.retries,
3536
lastSeen: finding.date,
37+
latestClassification: finding.classification,
3638
lastBrokenRunId: finding.classification === 'broken' ? finding.runId : undefined,
3739
lastBrokenRunUrl: finding.classification === 'broken' ? finding.runUrl : undefined,
3840
lastBrokenError: finding.classification === 'broken' ? finding.error : undefined,
@@ -42,13 +44,21 @@ export function summarizeTestHealth(findings) {
4244
});
4345
}
4446

45-
return Array.from(summary.values()).sort((a, b) => {
47+
return Array.from(summary.values())
48+
.map(item => {
49+
const latestIsBroken = item.latestClassification === 'broken';
50+
return {
51+
...item,
52+
brokenCount: latestIsBroken ? item.brokenCount : 0,
53+
};
54+
})
55+
.sort((a, b) => {
4656
if (a.brokenCount !== b.brokenCount) {
4757
return b.brokenCount - a.brokenCount;
4858
}
4959
if (a.flakyCount !== b.flakyCount) {
5060
return b.flakyCount - a.flakyCount;
5161
}
5262
return b.totalRetries - a.totalRetries;
53-
});
63+
});
5464
}

0 commit comments

Comments
 (0)