diff --git a/.github/workflows/e2e-validation.yml b/.github/workflows/e2e-validation.yml index 64b644eca1..8d4961e240 100644 --- a/.github/workflows/e2e-validation.yml +++ b/.github/workflows/e2e-validation.yml @@ -218,6 +218,8 @@ jobs: runs-on: ubuntu-latest needs: [define-test-matrix, launch-ec2-node] if: always() && (needs.define-test-matrix.result == 'success') && (inputs.environment != 'ec2-node' || needs.launch-ec2-node.result == 'success') + outputs: + has-flaky-failures: ${{ steps.set-outputs.outputs.has-flaky-failures }} strategy: fail-fast: true matrix: @@ -334,6 +336,14 @@ jobs: echo "Cypress Test task succeeded!" break fi + + # Check if this is a flaky failure (exit code 53) + if [ "$exit_code" = "53" ]; then + echo "Flaky failure detected (exit code 53) - marking as flaky failure" + echo "flaky_failure=true" >> "$GITHUB_ENV" + exit 53 + fi + n=$((n+1)) echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..." done @@ -351,6 +361,16 @@ jobs: path: app/web/cypress/videos/**/*.mp4 retention-days: 5 + - name: Set job outputs + id: set-outputs + if: always() + run: | + if [ "${{ env.flaky_failure }}" = "true" ]; then + echo "has-flaky-failures=true" >> "$GITHUB_OUTPUT" + else + echo "has-flaky-failures=false" >> "$GITHUB_OUTPUT" + fi + - name: Check Test Results if: failure() run: exit 1 @@ -434,7 +454,10 @@ jobs: break fi done - if [ "$has_artifacts" = true ] && [ "${{ github.ref_name }}" = "main" ]; then + + # Only send FireHydrant alert for non-flaky failures on main branch + if [ "$has_artifacts" = true ] && [ "${{ github.ref_name }}" = "main" ] && [ "${{ needs.cypress-tests.outputs.has-flaky-failures }}" != "true" ]; then + echo "Sending FireHydrant alert for real test failures" curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ --header "Content-Type: application/json" \ --data "{ @@ -450,6 +473,8 @@ jobs: \"service:github\" ] }" + elif [ "${{ needs.cypress-tests.outputs.has-flaky-failures }}" = "true" ]; then + echo "Skipping FireHydrant alert - flaky failure detected (exit code 53)" fi - name: Send Slack notification with deployment error @@ -466,12 +491,22 @@ jobs: if: ${{ inputs.environment != 'ec2-node' || (inputs.environment == 'ec2-node' && needs.launch-ec2-node.result != 'failure') }} run: | failed_tests="${{ steps.failed-tests.outputs.failed_tests }}" - if [ -n "$failed_tests" ]; then + is_flaky="${{ needs.cypress-tests.outputs.has-flaky-failures }}" + + if [ "$is_flaky" = "true" ]; then + # Flaky failure - send different message + curl -X POST \ + --header 'Content-type: application/json' \ + --data "{\"text\": \":si: Cypress E2E Test Flaked for ${{ inputs.environment }} (exit code 53): \n\`\`\`Flaky test failure detected - not paging just notifying\`\`\`\"}" \ + ${{ secrets.SLACK_WEBHOOK_URL }} + elif [ -n "$failed_tests" ]; then + # Regular test failure curl -X POST \ --header 'Content-type: application/json' \ --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: \n\`\`\`Failed tests: $failed_tests\`\`\`\"}" \ ${{ secrets.SLACK_WEBHOOK_URL }} else + # Generic failure curl -X POST \ --header 'Content-type: application/json' \ --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: \"}" \ diff --git a/app/web/cypress.config.ts b/app/web/cypress.config.ts index 32e143b9ce..5cf0833bcb 100644 --- a/app/web/cypress.config.ts +++ b/app/web/cypress.config.ts @@ -2,6 +2,8 @@ import path from 'path' import { defineConfig } from 'cypress' import vitePreprocessor from 'cypress-vite' +const FLAKY_EXIT_CODE = 53 + export default defineConfig({ e2e: { injectDocumentDomain: true, @@ -11,13 +13,45 @@ export default defineConfig({ vitePreprocessor( path.resolve('./vite.cypress.ts'), ) - ), - on('task', { - log(message) { - console.log(message) - return null - } - }) + ); + + on('task', { + log(message) { + console.log(message) + return null + }, + flakyFailure() { + console.log('Flaky failure detected - will fail test and exit with code', FLAKY_EXIT_CODE) + // Set a flag that can be checked in after:run hook + process.env.FLAKY_FAILURE_DETECTED = 'true' + // Throw an error to fail the test + throw new Error('Simulated flaky failure') + } + }); + + on('after:run', (results: any) => { + // Check if flaky failure was explicitly triggered + if (process.env.FLAKY_FAILURE_DETECTED === 'true') { + console.log('Flaky failure was detected during test run - exiting with code', FLAKY_EXIT_CODE) + // Use setImmediate to allow Cypress to finish cleanup before exiting + setImmediate(() => process.exit(FLAKY_EXIT_CODE)) + return + } + + // Check for Auth0-related failures in test results + const hasAuth0Failures = results.runs?.some((run: any) => + run.tests?.some((test: any) => + test.displayError?.includes('Auth0') || + test.title?.includes('auth0') || + test.err?.message?.includes('Auth0') + ) + ); + + if (hasAuth0Failures) { + console.log('Detected Auth0-related test failures - exiting with code', FLAKY_EXIT_CODE) + setImmediate(() => process.exit(FLAKY_EXIT_CODE)) + } + }); }, // Hotfix, needs amended diff --git a/app/web/cypress/e2e/workspace-management/workspace-dashboard.cy.ts b/app/web/cypress/e2e/workspace-management/workspace-dashboard.cy.ts index b52b55d15b..3252439eef 100644 --- a/app/web/cypress/e2e/workspace-management/workspace-dashboard.cy.ts +++ b/app/web/cypress/e2e/workspace-management/workspace-dashboard.cy.ts @@ -15,9 +15,20 @@ Cypress._.times(SI_CYPRESS_MULTIPLIER, () => { cy.visit("/"); }); + it("simulate_flaky_failure", () => { + // Temporary test to simulate flaky failure for workflow testing + cy.task('flakyFailure'); + }); + it("dashboard_redirect", () => { - cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); + try { + cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); + } catch (_err) { + // flaky failures should not ping us + cy.task('flakyFailure'); + return; + } // Go to the Synthetic User's Dashboard cy.visit(AUTH_PORTAL_URL + '/dashboard') diff --git a/app/web/cypress/support/commands.ts b/app/web/cypress/support/commands.ts index 51cd147153..69eed2c5bc 100644 --- a/app/web/cypress/support/commands.ts +++ b/app/web/cypress/support/commands.ts @@ -107,7 +107,13 @@ Cypress.Commands.add('basicLogin', () => { const SI_WORKSPACE_ID = Cypress.env('VITE_SI_WORKSPACE_ID') || import.meta.env.VITE_SI_WORKSPACE_ID; const UUID = Cypress.env('VITE_UUID') || import.meta.env.VITE_UUID || "local"; - cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); + try { + cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); + } catch (_err) { + // flaky failures should not ping us + cy.task('flakyFailure'); + return; + } cy.visit({ url:AUTH_API_URL + '/workspaces/' + SI_WORKSPACE_ID + '/go', failOnStatusCode: false diff --git a/app/web/cypress/ux/tile-view/get-tiles.cy.ts b/app/web/cypress/ux/tile-view/get-tiles.cy.ts index 9962e4b6a7..f0889c27f4 100644 --- a/app/web/cypress/ux/tile-view/get-tiles.cy.ts +++ b/app/web/cypress/ux/tile-view/get-tiles.cy.ts @@ -12,11 +12,15 @@ const AUTH_API_URL = Cypress.env('VITE_AUTH_API_URL') || import.meta.env.VITE_AU const AUTH_PORTAL_URL = Cypress.env('VITE_AUTH_PORTAL_URL') || import.meta.env.VITE_AUTH_PORTAL_URL; describe("web", () => { - beforeEach(function () { - cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); - }); it("get_ptlw_tiles", () => { + try { + cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD); + } catch (_err) { + // flaky failures should not ping us + cy.task('flakyFailure'); + return; + } // Go to the Synthetic User's Dashboard cy.visit(AUTH_PORTAL_URL + '/dashboard') cy.sendPosthogEvent(Cypress.currentTest.titlePath.join("/"), "test_uuid", UUID);