e2e-web #358
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # E2E Web - Stagehand browser tests on Vercel preview deploys | |
| # Trigger: deployment_status (Vercel webhook) | |
| # Parallel: runs alongside e2e-api on preview deploy | |
| name: e2e-web | |
| on: | |
| deployment_status: | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| e2e: | |
| if: > | |
| github.event.deployment_status.state == 'success' && | |
| github.event.deployment_status.environment_url != '' && | |
| (startsWith(github.event.deployment_status.environment_url, 'https://') || | |
| startsWith(github.event.deployment_status.environment_url, 'http://')) && | |
| (github.event.deployment.environment == 'Preview' || github.event.deployment.environment == 'preview') | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| timeout-minutes: 35 | |
| steps: | |
| - name: Check for doc-only changes | |
| id: doc_check | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const sha = | |
| context.payload.deployment?.sha ?? | |
| context.payload.deployment_status?.deployment?.sha; | |
| if (!sha) { | |
| core.warning("No deployment SHA found; running tests."); | |
| core.setOutput("run_tests", "true"); | |
| return; | |
| } | |
| let prs = []; | |
| try { | |
| const response = | |
| await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner, | |
| repo, | |
| commit_sha: sha, | |
| }); | |
| prs = response.data ?? []; | |
| } catch (error) { | |
| core.warning(`Failed to resolve PR for ${sha}: ${error.message}`); | |
| core.setOutput("run_tests", "true"); | |
| return; | |
| } | |
| if (!prs.length) { | |
| core.notice(`No PR found for ${sha}; running tests.`); | |
| core.setOutput("run_tests", "true"); | |
| return; | |
| } | |
| const pr = prs[0]; | |
| const prNumber = pr.number; | |
| const isBumpPr = | |
| pr.title?.startsWith("chore: bump version") || | |
| pr.head?.ref?.startsWith("chore/bump-version"); | |
| if (isBumpPr) { | |
| core.notice("Bump PR detected; skipping E2E web tests."); | |
| core.setOutput("run_tests", "false"); | |
| return; | |
| } | |
| const files = await github.paginate( | |
| github.rest.pulls.listFiles, | |
| { owner, repo, pull_number: prNumber, per_page: 100 } | |
| ); | |
| const docOnly = files.every( | |
| (file) => | |
| file.filename === "README.md" || | |
| file.filename === "AGENTS.md" || | |
| file.filename.startsWith("docs/") || | |
| file.filename.startsWith(".codex/") || | |
| file.filename.startsWith("packages/opencode-excalidraw/") || | |
| file.filename.startsWith(".github/workflows/opencode-excalidraw-") | |
| ); | |
| core.setOutput("run_tests", docOnly ? "false" : "true"); | |
| if (docOnly) { | |
| core.notice("Docs-only change; skipping E2E web tests."); | |
| } | |
| - name: Checkout | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Setup Bun | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install E2E dependencies | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| run: bun install --frozen-lockfile | |
| working-directory: tests/e2e | |
| - name: Run Stagehand smoke scenarios | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| env: | |
| STAGEHAND_TARGET_URL: ${{ github.event.deployment_status.environment_url }} | |
| STAGEHAND_ENV: BROWSERBASE | |
| STAGEHAND_HEADLESS: "true" | |
| BROWSERBASE_SESSION_TIMEOUT_SECONDS: "900" | |
| BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} | |
| BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| MODEL_NAME: ${{ secrets.MODEL_NAME }} | |
| VISION_MODEL_NAME: ${{ secrets.VISION_MODEL_NAME }} | |
| STAGEHAND_VISUAL_STRICT: "false" | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| run: | | |
| cd tests/e2e | |
| # Smoke/sanity coverage: fast navigation/auth gating checks. | |
| bun run visual-sanity | |
| bun run auth-gates | |
| - name: Validate authenticated E2E secrets | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| env: | |
| SKETCHI_E2E_EMAIL: ${{ secrets.SKETCHI_E2E_EMAIL }} | |
| SKETCHI_E2E_PASSWORD: ${{ secrets.SKETCHI_E2E_PASSWORD }} | |
| run: | | |
| if [ -z "$SKETCHI_E2E_EMAIL" ]; then | |
| echo "Missing required secret: SKETCHI_E2E_EMAIL" | |
| exit 1 | |
| fi | |
| if [ -z "$SKETCHI_E2E_PASSWORD" ]; then | |
| echo "Missing required secret: SKETCHI_E2E_PASSWORD" | |
| exit 1 | |
| fi | |
| echo "::add-mask::$SKETCHI_E2E_EMAIL" | |
| echo "::add-mask::$SKETCHI_E2E_PASSWORD" | |
| - name: Run Stagehand authenticated continuity scenarios | |
| if: steps.doc_check.outputs.run_tests == 'true' | |
| env: | |
| STAGEHAND_TARGET_URL: ${{ github.event.deployment_status.environment_url }} | |
| STAGEHAND_ENV: BROWSERBASE | |
| STAGEHAND_HEADLESS: "true" | |
| BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} | |
| BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| MODEL_NAME: ${{ secrets.MODEL_NAME }} | |
| VISION_MODEL_NAME: ${{ secrets.VISION_MODEL_NAME }} | |
| STAGEHAND_VISUAL_STRICT: "false" | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| SKETCHI_E2E_EMAIL: ${{ secrets.SKETCHI_E2E_EMAIL }} | |
| SKETCHI_E2E_PASSWORD: ${{ secrets.SKETCHI_E2E_PASSWORD }} | |
| run: | | |
| cd tests/e2e | |
| # Assertion coverage: strict CLI<->web continuity + conflict recovery behavior. | |
| bun run opencode-web-continuity | |
| bun run diagram-studio-happy-path | |
| bun run diagram-studio-occ-conflict | |
| - name: Summarize Browserbase replay links | |
| if: always() && steps.doc_check.outputs.run_tests == 'true' | |
| run: | | |
| node <<'NODE' | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const summaryPath = process.env.GITHUB_STEP_SUMMARY; | |
| const replayPath = path.join( | |
| process.cwd(), | |
| "tests", | |
| "e2e", | |
| "artifacts", | |
| "browserbase-sessions.jsonl" | |
| ); | |
| if (!summaryPath) { | |
| process.exit(0); | |
| } | |
| const lines = []; | |
| lines.push("## Browserbase Replays"); | |
| if (!fs.existsSync(replayPath)) { | |
| lines.push("No Browserbase session metadata found."); | |
| fs.appendFileSync(summaryPath, `${lines.join("\n")}\n`); | |
| process.exit(0); | |
| } | |
| const raw = fs | |
| .readFileSync(replayPath, "utf8") | |
| .split("\n") | |
| .map((line) => line.trim()) | |
| .filter(Boolean); | |
| const seen = new Set(); | |
| const sessions = []; | |
| for (const line of raw) { | |
| try { | |
| const entry = JSON.parse(line); | |
| const key = | |
| entry.sessionId || | |
| entry.sessionUrl || | |
| `${entry.scenario || "unknown"}:${entry.capturedAt || ""}`; | |
| if (seen.has(key)) continue; | |
| seen.add(key); | |
| sessions.push(entry); | |
| } catch { | |
| // ignore malformed lines | |
| } | |
| } | |
| if (!sessions.length) { | |
| lines.push("No Browserbase sessions recorded."); | |
| fs.appendFileSync(summaryPath, `${lines.join("\n")}\n`); | |
| process.exit(0); | |
| } | |
| for (const session of sessions) { | |
| const idText = session.sessionId | |
| ? `\`${session.sessionId}\`` | |
| : "`unknown`"; | |
| const scenarioText = session.scenario | |
| ? ` (scenario: \`${session.scenario}\`)` | |
| : ""; | |
| const links = []; | |
| if (session.sessionUrl) { | |
| links.push(`[replay](${session.sessionUrl})`); | |
| } | |
| if (session.debugUrl) { | |
| links.push(`[debug](${session.debugUrl})`); | |
| } | |
| lines.push(`- Session ${idText}${scenarioText}: ${links.join(" | ") || "link unavailable"}`); | |
| } | |
| fs.appendFileSync(summaryPath, `${lines.join("\n")}\n`); | |
| NODE | |
| - name: Comment Browserbase replay links on PR | |
| if: always() && steps.doc_check.outputs.run_tests == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const marker = "<!-- e2e-web-browserbase-replays -->"; | |
| const replayPath = path.join( | |
| process.cwd(), | |
| "tests", | |
| "e2e", | |
| "artifacts", | |
| "browserbase-sessions.jsonl" | |
| ); | |
| if (!fs.existsSync(replayPath)) { | |
| core.info("No Browserbase replay file found; skipping PR comment."); | |
| return; | |
| } | |
| const sha = | |
| context.payload.deployment?.sha ?? | |
| context.payload.deployment_status?.deployment?.sha; | |
| if (!sha) { | |
| core.info("No deployment SHA available; skipping PR comment."); | |
| return; | |
| } | |
| let prs = []; | |
| try { | |
| const response = | |
| await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: sha, | |
| }); | |
| prs = response.data ?? []; | |
| } catch (error) { | |
| core.warning(`Failed to resolve PR from deployment SHA: ${error.message}`); | |
| return; | |
| } | |
| if (!prs.length) { | |
| core.info(`No PR associated with ${sha}; skipping replay comment.`); | |
| return; | |
| } | |
| const records = fs | |
| .readFileSync(replayPath, "utf8") | |
| .split("\n") | |
| .map((line) => line.trim()) | |
| .filter(Boolean) | |
| .flatMap((line) => { | |
| try { | |
| return [JSON.parse(line)]; | |
| } catch { | |
| return []; | |
| } | |
| }); | |
| if (!records.length) { | |
| core.info("Replay file exists but has no valid records."); | |
| return; | |
| } | |
| const unique = []; | |
| const seen = new Set(); | |
| for (const record of records) { | |
| const key = | |
| record.sessionId || | |
| record.sessionUrl || | |
| `${record.scenario || "unknown"}:${record.capturedAt || ""}`; | |
| if (seen.has(key)) continue; | |
| seen.add(key); | |
| unique.push(record); | |
| } | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const lines = [ | |
| marker, | |
| "## Browserbase Replays", | |
| "", | |
| ...unique.map((record) => { | |
| const idText = record.sessionId ? `\`${record.sessionId}\`` : "`unknown`"; | |
| const scenarioText = record.scenario ? ` (scenario: \`${record.scenario}\`)` : ""; | |
| const links = []; | |
| if (record.sessionUrl) links.push(`[replay](${record.sessionUrl})`); | |
| if (record.debugUrl) links.push(`[debug](${record.debugUrl})`); | |
| return `- Session ${idText}${scenarioText}: ${links.join(" | ") || "link unavailable"}`; | |
| }), | |
| "", | |
| `Workflow run: [${context.runId}](${runUrl})`, | |
| ]; | |
| const body = lines.join("\n"); | |
| const prNumber = prs[0].number; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find((comment) => | |
| comment.user?.type === "Bot" && typeof comment.body === "string" && comment.body.includes(marker) | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| core.info(`Updated replay comment on PR #${prNumber}.`); | |
| return; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body, | |
| }); | |
| core.info(`Created replay comment on PR #${prNumber}.`); | |
| - name: Upload E2E artifacts | |
| if: always() && steps.doc_check.outputs.run_tests == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-artifacts-${{ github.run_id }} | |
| path: tests/e2e/artifacts | |
| if-no-files-found: warn | |
| retention-days: 7 |