preview-validation #152
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: preview-validation | |
| on: | |
| deployment_status: | |
| permissions: | |
| contents: read | |
| deployments: read | |
| pull-requests: write | |
| jobs: | |
| seed-preview: | |
| name: Preview validation (seed + runtime QA) | |
| if: > | |
| github.event.deployment_status.state == 'success' && | |
| ( | |
| startsWith(github.event.deployment.environment || '', 'Preview') || | |
| startsWith(github.event.deployment_status.environment || '', 'Preview') | |
| ) | |
| concurrency: | |
| group: preview-seed-${{ github.event.deployment.id }} | |
| cancel-in-progress: false | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| env: | |
| PREVIEW_URL: ${{ github.event.deployment_status.environment_url || github.event.deployment_status.target_url || github.event.deployment.environment_url || '' }} | |
| DEPLOYMENT_ID: ${{ github.event.deployment.id }} | |
| DEPLOYMENT_REF: ${{ github.event.deployment.ref }} | |
| DEPLOYMENT_SHA: ${{ github.event.deployment.sha }} | |
| CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY_PREVIEW }} | |
| BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} | |
| BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| steps: | |
| - name: Resolve pull request context | |
| id: pr | |
| uses: actions/github-script@v8 | |
| env: | |
| DEPLOY_REF: ${{ env.DEPLOYMENT_REF }} | |
| DEPLOY_SHA: ${{ env.DEPLOYMENT_SHA }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const rawRef = process.env.DEPLOY_REF ?? ""; | |
| const rawSha = process.env.DEPLOY_SHA ?? ""; | |
| const repo = context.repo; | |
| const isCommitSha = (value) => /^[0-9a-f]{40}$/i.test(value); | |
| const setPrOutputs = (pr, reason) => { | |
| core.info(`Resolved PR #${pr.number} (${reason}).`); | |
| const isReleaseVersionBumpPr = | |
| /^chore\/bump-version-/.test(pr.head?.ref ?? "") && | |
| (pr.user?.login === "github-actions[bot]" || pr.user?.login === "app/github-actions"); | |
| if (isReleaseVersionBumpPr) { | |
| core.info(`PR #${pr.number} is an automated release bump; skipping preview seed.`); | |
| core.setOutput("skip", "true"); | |
| return; | |
| } | |
| core.setOutput("skip", "false"); | |
| core.setOutput("number", String(pr.number)); | |
| core.setOutput("title", pr.title ?? ""); | |
| core.setOutput("headSha", pr.head?.sha ?? ""); | |
| }; | |
| const resolveByCommitSha = async (commitSha) => { | |
| if (!commitSha) { | |
| return null; | |
| } | |
| core.info(`Looking up PR associated with commit ${commitSha}.`); | |
| const { data: associated } = | |
| await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| ...repo, | |
| commit_sha: commitSha, | |
| per_page: 20, | |
| }); | |
| const openPr = associated.find((pr) => pr.state === "open"); | |
| if (openPr) { | |
| return openPr; | |
| } | |
| return null; | |
| }; | |
| const match = rawRef.match(/^refs\/pull\/(\d+)\/merge$/); | |
| if (match) { | |
| const pull_number = Number.parseInt(match[1], 10); | |
| const { data: pr } = await github.rest.pulls.get({ | |
| ...repo, | |
| pull_number, | |
| }); | |
| if (pr.state !== "open") { | |
| core.info(`PR #${pull_number} is not open; skipping preview seed.`); | |
| core.setOutput("skip", "true"); | |
| return; | |
| } | |
| setPrOutputs(pr, `deployment ref ${rawRef}`); | |
| return; | |
| } | |
| const normalized = | |
| rawRef.startsWith("refs/heads/") ? | |
| rawRef.replace("refs/heads/", "") : | |
| rawRef; | |
| if (normalized && !isCommitSha(normalized)) { | |
| const { data: prs } = await github.rest.pulls.list({ | |
| ...repo, | |
| head: `${repo.owner}:${normalized}`, | |
| state: "open", | |
| per_page: 1, | |
| }); | |
| if (prs.length > 0) { | |
| setPrOutputs(prs[0], `branch ${normalized}`); | |
| return; | |
| } | |
| core.info(`No open PR found by branch ${normalized}.`); | |
| } | |
| const commitCandidate = | |
| isCommitSha(normalized) ? normalized : (isCommitSha(rawSha) ? rawSha : ""); | |
| const commitPr = await resolveByCommitSha(commitCandidate); | |
| if (commitPr) { | |
| setPrOutputs(commitPr, `commit ${commitCandidate}`); | |
| return; | |
| } | |
| core.info( | |
| `No open PR found for deployment ref '${rawRef}' and sha '${rawSha}'; skipping preview seed.`, | |
| ); | |
| core.setOutput("skip", "true"); | |
| - name: Skip when deployment has no matching PR | |
| if: steps.pr.outputs.skip == 'true' | |
| run: | | |
| echo "Preview deployment ${DEPLOYMENT_ID} is not tied to an open PR; skipping seed run." | |
| - name: Validate preview URL | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| if [ -z "${PREVIEW_URL:-}" ]; then | |
| echo "Deployment status event did not include an environment URL." | |
| exit 1 | |
| fi | |
| case "${PREVIEW_URL}" in | |
| http://*|https://*) ;; | |
| *) | |
| echo "Deployment URL '${PREVIEW_URL}' is invalid; expected http(s) URL." | |
| exit 1 | |
| ;; | |
| esac | |
| echo "Seeding preview deployment at ${PREVIEW_URL}" | |
| - name: Check Convex credentials | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| if [ -z "${CONVEX_DEPLOY_KEY:-}" ]; then | |
| echo "CONVEX_DEPLOY_KEY_PREVIEW secret is not configured." | |
| exit 1 | |
| fi | |
| - name: Check Browserbase credentials | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| if [ -z "${BROWSERBASE_API_KEY:-}" ]; then | |
| echo "BROWSERBASE_API_KEY secret is not configured." | |
| exit 1 | |
| fi | |
| if [ -z "${BROWSERBASE_PROJECT_ID:-}" ]; then | |
| echo "BROWSERBASE_PROJECT_ID secret is not configured." | |
| exit 1 | |
| fi | |
| - name: Check Vercel automation bypass credential | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| if [ -z "${VERCEL_AUTOMATION_BYPASS_SECRET:-}" ]; then | |
| echo "VERCEL_AUTOMATION_BYPASS_SECRET secret is not configured." | |
| exit 1 | |
| fi | |
| - name: Checkout | |
| if: steps.pr.outputs.skip != 'true' | |
| uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| if: steps.pr.outputs.skip != 'true' | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: 1.3.11 | |
| - name: Install dependencies | |
| if: steps.pr.outputs.skip != 'true' | |
| run: bun install --frozen-lockfile | |
| - name: Install agent-browser CLI | |
| if: steps.pr.outputs.skip != 'true' | |
| run: npm install -g agent-browser@latest | |
| - name: Install Playwright Chromium | |
| if: steps.pr.outputs.skip != 'true' | |
| run: npx playwright install --with-deps chromium | |
| - name: Seed preview catalog | |
| id: seed | |
| if: steps.pr.outputs.skip != 'true' | |
| env: | |
| VERCEL_URL: ${{ env.PREVIEW_URL }} | |
| run: | | |
| set -euo pipefail | |
| LOG_FILE="$RUNNER_TEMP/preview-seed.log" | |
| echo "log=$LOG_FILE" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "Target preview: ${VERCEL_URL}" | |
| bun run scripts/seed-vercel-deployment.ts --vercel-url "${VERCEL_URL}" | |
| } | tee "$LOG_FILE" | |
| - name: Run browser QA on preview deployment | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| USE_BROWSERBASE=1 \ | |
| bash ./scripts/browser/run-local-browser-qa.sh \ | |
| --external-url "${PREVIEW_URL}" \ | |
| --out-dir artifacts/browser-qa-preview | |
| - name: Read replay metadata | |
| id: browser_qa_meta | |
| if: steps.pr.outputs.skip != 'true' | |
| run: | | |
| REPLAY_URL=$(jq -r '.browserbaseReplayUrl // empty' artifacts/browser-qa-preview/results.json) | |
| echo "replay_url=${REPLAY_URL}" >> "$GITHUB_OUTPUT" | |
| - name: Upload browser QA artifacts | |
| id: upload_browser_qa | |
| if: steps.pr.outputs.skip != 'true' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: browser-qa-preview-${{ github.event.deployment.id }} | |
| path: artifacts/browser-qa-preview/** | |
| if-no-files-found: error | |
| - name: Comment browser QA links on PR | |
| if: steps.pr.outputs.skip != 'true' | |
| uses: actions/github-script@v8 | |
| env: | |
| ARTIFACT_URL: ${{ steps.upload_browser_qa.outputs.artifact-url }} | |
| REPLAY_URL: ${{ steps.browser_qa_meta.outputs.replay_url }} | |
| PR_NUMBER: ${{ steps.pr.outputs.number }} | |
| with: | |
| script: | | |
| const marker = '<!-- browser-qa-report -->'; | |
| const issue_number = Number.parseInt(process.env.PR_NUMBER ?? "", 10); | |
| if (!Number.isFinite(issue_number) || issue_number <= 0) { | |
| core.info("No PR number found; skipping browser QA comment."); | |
| return; | |
| } | |
| const artifactUrl = process.env.ARTIFACT_URL ?? ''; | |
| const replayUrl = process.env.REPLAY_URL ?? ''; | |
| const bodyLines = [ | |
| marker, | |
| '## Browser QA (agent-browser)', | |
| artifactUrl | |
| ? `- Screenshots artifact: [browser-qa](${artifactUrl})` | |
| : '- Screenshots artifact: unavailable', | |
| replayUrl | |
| ? `- Browserbase replay: [session replay](${replayUrl})` | |
| : '- Browserbase replay: unavailable', | |
| ]; | |
| const body = bodyLines.join('\n'); | |
| const { owner, repo } = context.repo; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find((comment) => | |
| typeof comment.body === 'string' && comment.body.includes(marker) | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| return; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body, | |
| }); | |
| - name: Publish summary | |
| if: steps.pr.outputs.skip != 'true' && always() | |
| env: | |
| SUMMARY_STATUS: ${{ job.status }} | |
| PR_NUMBER: ${{ steps.pr.outputs.number }} | |
| PR_TITLE: ${{ steps.pr.outputs.title }} | |
| LOG_PATH: ${{ steps.seed.outputs.log }} | |
| PREVIEW_URL: ${{ env.PREVIEW_URL }} | |
| run: | | |
| { | |
| echo "## Preview seed status" | |
| echo "" | |
| echo "- Deployment ID: ${DEPLOYMENT_ID}" | |
| echo "- Preview URL: ${PREVIEW_URL}" | |
| if [ -n "${PR_NUMBER:-}" ]; then | |
| echo "- PR: #${PR_NUMBER} ${PR_TITLE}" | |
| fi | |
| echo "- Result: ${SUMMARY_STATUS}" | |
| echo "- SHA: ${DEPLOYMENT_SHA}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if [ -f "artifacts/browser-qa-preview/results.json" ]; then | |
| REPLAY_URL=$(jq -r '.browserbaseReplayUrl // empty' artifacts/browser-qa-preview/results.json) | |
| { | |
| echo "" | |
| echo "## Browser QA" | |
| if [ -n "$REPLAY_URL" ]; then | |
| echo "- Browserbase replay: ${REPLAY_URL}" | |
| else | |
| echo "- Browserbase replay: unavailable" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| if [ -n "${LOG_PATH:-}" ] && [ -f "${LOG_PATH}" ]; then | |
| { | |
| echo "" | |
| echo "<details>" | |
| echo "<summary>Seed output (tail)</summary>" | |
| echo "" | |
| tail -n 50 "${LOG_PATH}" | |
| echo "" | |
| echo "</details>" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi |