diff --git a/.github/FLAKY_CI_FAILURE_TEMPLATE.md b/.github/FLAKY_CI_FAILURE_TEMPLATE.md index 2a2ad5109561..c105f6928d27 100644 --- a/.github/FLAKY_CI_FAILURE_TEMPLATE.md +++ b/.github/FLAKY_CI_FAILURE_TEMPLATE.md @@ -1,5 +1,5 @@ --- -title: '[Flaky CI]: {{ env.JOB_NAME }}' +title: '[Flaky CI]: {{ env.JOB_NAME }} - {{ env.TEST_NAME }}' labels: Tests --- @@ -13,7 +13,7 @@ Other / Unknown ### Name of Test -_Not available - check the run link for details_ +{{ env.TEST_NAME }} ### Link to Test Run diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63c44de0d68f..f4ef407e1126 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1155,82 +1155,25 @@ jobs: runs-on: ubuntu-24.04 permissions: issues: write + checks: read steps: - name: Check out current commit if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') uses: actions/checkout@v6 with: - sparse-checkout: .github + sparse-checkout: | + .github + scripts - name: Create issues for failed jobs if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') uses: actions/github-script@v7 with: script: | - const fs = require('fs'); - - // Fetch actual job details from the API to get descriptive names - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100 - }); - - const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); - - if (failedJobs.length === 0) { - console.log('No failed jobs found'); - return; - } - - // Read and parse template - const template = fs.readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8'); - const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); - - // Get existing open issues with Tests label - const existing = await github.paginate(github.rest.issues.listForRepo, { - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - labels: 'Tests', - per_page: 100 - }); - - for (const job of failedJobs) { - const jobName = job.name; - const jobUrl = job.html_url; - - // Replace template variables - const vars = { - 'JOB_NAME': jobName, - 'RUN_LINK': jobUrl - }; - - let title = frontmatter.match(/title:\s*'(.*)'/)[1]; - let issueBody = bodyTemplate; - for (const [key, value] of Object.entries(vars)) { - const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g'); - title = title.replace(pattern, value); - issueBody = issueBody.replace(pattern, value); - } - - const existingIssue = existing.find(i => i.title === title); - - if (existingIssue) { - console.log(`Issue already exists for ${jobName}: #${existingIssue.number}`); - continue; - } - - const newIssue = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: title, - body: issueBody.trim(), - labels: ['Tests'] - }); - console.log(`Created issue #${newIssue.data.number} for ${jobName}`); - } + const { default: run } = await import( + `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` + ); + await run({ github, context, core }); - name: Check for failures if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') diff --git a/dev-packages/browser-integration-tests/playwright.config.ts b/dev-packages/browser-integration-tests/playwright.config.ts index 821c0291ccfb..681de57d4e59 100644 --- a/dev-packages/browser-integration-tests/playwright.config.ts +++ b/dev-packages/browser-integration-tests/playwright.config.ts @@ -30,7 +30,7 @@ const config: PlaywrightTestConfig = { }, ], - reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + reporter: process.env.CI ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', globalSetup: require.resolve('./playwright.setup.ts'), globalTeardown: require.resolve('./playwright.teardown.ts'), diff --git a/dev-packages/test-utils/src/playwright-config.ts b/dev-packages/test-utils/src/playwright-config.ts index fb15fc325232..823f47380c05 100644 --- a/dev-packages/test-utils/src/playwright-config.ts +++ b/dev-packages/test-utils/src/playwright-config.ts @@ -37,7 +37,7 @@ export function getPlaywrightConfig( /* In dev mode some apps are flaky, so we allow retry there... */ retries: testEnv === 'development' ? 3 : 0, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + reporter: process.env.CI ? [['line'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ diff --git a/packages/nuxt/vite.config.ts b/packages/nuxt/vite.config.ts index 75dc3957244a..9dce31278f4f 100644 --- a/packages/nuxt/vite.config.ts +++ b/packages/nuxt/vite.config.ts @@ -3,6 +3,7 @@ import baseConfig from '../../vite/vite.config'; export default { ...baseConfig, test: { + ...baseConfig.test, environment: 'jsdom', setupFiles: ['./test/vitest.setup.ts'], typecheck: { diff --git a/scripts/report-ci-failures.mjs b/scripts/report-ci-failures.mjs new file mode 100644 index 000000000000..5a05e1144bec --- /dev/null +++ b/scripts/report-ci-failures.mjs @@ -0,0 +1,109 @@ +/** + * CI Failure Reporter script. + * + * Creates GitHub issues for tests that fail on the develop branch. + * For each failed job in the workflow run, it fetches check run annotations + * to identify individual failing tests, then creates one issue per failing + * test using the FLAKY_CI_FAILURE_TEMPLATE.md template. Existing open issues + * with matching titles are skipped to avoid duplicates. + * + * Intended to be called from a GitHub Actions workflow via actions/github-script: + * + * const { default: run } = await import( + * `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` + * ); + * await run({ github, context, core }); + */ + +import { readFileSync } from 'node:fs'; + +export default async function run({ github, context, core }) { + const { owner, repo } = context.repo; + + // Fetch actual job details from the API to get descriptive names + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: context.runId, + per_page: 100, + }); + + const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); + + if (failedJobs.length === 0) { + core.info('No failed jobs found'); + return; + } + + // Read and parse template + const template = readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8'); + const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + + // Get existing open issues with Tests label + const existing = await github.paginate(github.rest.issues.listForRepo, { + owner, + repo, + state: 'open', + labels: 'Tests', + per_page: 100, + }); + + for (const job of failedJobs) { + const jobName = job.name; + const jobUrl = job.html_url; + + // Fetch annotations from the check run to extract failed test names + let testNames = []; + try { + const annotations = await github.paginate(github.rest.checks.listAnnotations, { + owner, + repo, + check_run_id: job.id, + per_page: 100, + }); + + const testAnnotations = annotations.filter(a => a.annotation_level === 'failure' && a.path !== '.github'); + testNames = [...new Set(testAnnotations.map(a => a.title || a.path))]; + } catch (e) { + core.info(`Could not fetch annotations for ${jobName}: ${e.message}`); + } + + // If no test names found, fall back to one issue per job + if (testNames.length === 0) { + testNames = ['Unknown test']; + } + + // Create one issue per failing test for proper deduplication + for (const testName of testNames) { + const vars = { + JOB_NAME: jobName, + RUN_LINK: jobUrl, + TEST_NAME: testName, + }; + + let title = frontmatter.match(/title:\s*'(.*)'/)[1]; + let issueBody = bodyTemplate; + for (const [key, value] of Object.entries(vars)) { + const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g'); + title = title.replace(pattern, value); + issueBody = issueBody.replace(pattern, value); + } + + const existingIssue = existing.find(i => i.title === title); + + if (existingIssue) { + core.info(`Issue already exists for "${testName}" in ${jobName}: #${existingIssue.number}`); + continue; + } + + const newIssue = await github.rest.issues.create({ + owner, + repo, + title, + body: issueBody.trim(), + labels: ['Tests'], + }); + core.info(`Created issue #${newIssue.data.number} for "${testName}" in ${jobName}`); + } + } +} diff --git a/vite/vite.config.ts b/vite/vite.config.ts index 62f89a570f52..c603d95fe856 100644 --- a/vite/vite.config.ts +++ b/vite/vite.config.ts @@ -19,7 +19,9 @@ export default defineConfig({ 'vite.config.*', ], }, - reporters: process.env.CI ? ['default', ['junit', { classnameTemplate: '{filepath}' }]] : ['default'], + reporters: process.env.CI + ? ['default', 'github-actions', ['junit', { classnameTemplate: '{filepath}' }]] + : ['default'], outputFile: { junit: 'vitest.junit.xml', },