From 2ae63aadc09b7218da865aa95514d47c2b66f961 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 11 Jun 2026 04:49:22 -0700 Subject: [PATCH 01/68] chore: initial public snapshot v0.1.0 --- .github/workflows/ci.yml | 97 + .github/workflows/release.yaml | 25 + .github/workflows/test-coverage.yml | 81 + .gitignore | 22 + .npmrc | 1 + .prettierignore | 8 + .prettierrc | 7 + CHANGELOG.md | 110 + CONTRIBUTING.md | 90 + DOCUMENTATION.md | 485 + LICENSE | 201 + README.md | 190 + assets/testsprite-logo-dark.svg | 16 + assets/testsprite-logo-light.svg | 16 + docs/cli-v1-agent-install/skill-template.md | 303 + eslint.config.mjs | 27 + package-lock.json | 4284 +++++++++ package.json | 65 + scripts/postbuild.mjs | 18 + .../agent-skill/testsprite-verify.codex.md | 119 + .../agent-skill/testsprite-verify.skill.md | 423 + src/commands/agent.test.ts | 1957 ++++ src/commands/agent.ts | 770 ++ src/commands/auth.test.ts | 933 ++ src/commands/auth.ts | 337 + src/commands/init.test.ts | 828 ++ src/commands/init.ts | 493 + src/commands/project.test.ts | 751 ++ src/commands/project.ts | 608 ++ src/commands/test.artifact.spec.ts | 874 ++ src/commands/test.create-batch-run.spec.ts | 2790 ++++++ src/commands/test.quickwins.spec.ts | 939 ++ src/commands/test.rerun.spec.ts | 4477 +++++++++ src/commands/test.result.history.spec.ts | 1293 +++ src/commands/test.run.spec.ts | 3528 +++++++ src/commands/test.test.ts | 7562 +++++++++++++++ src/commands/test.ts | 8537 +++++++++++++++++ src/commands/test.wait.spec.ts | 1244 +++ src/commands/usage.test.ts | 313 + src/commands/usage.ts | 237 + src/index.ts | 160 + src/lib/agent-targets.test.ts | 382 + src/lib/agent-targets.ts | 129 + src/lib/bundle.test.ts | 594 ++ src/lib/bundle.ts | 802 ++ src/lib/client-factory.test.ts | 286 + src/lib/client-factory.ts | 183 + src/lib/config.test.ts | 90 + src/lib/config.ts | 46 + src/lib/credentials.test.ts | 170 + src/lib/credentials.ts | 146 + src/lib/dry-run/fetch.test.ts | 52 + src/lib/dry-run/fetch.ts | 70 + src/lib/dry-run/samples.test.ts | 630 ++ src/lib/dry-run/samples.ts | 777 ++ src/lib/errors.test.ts | 488 + src/lib/errors.ts | 444 + src/lib/facade.test.ts | 149 + src/lib/facade.ts | 113 + src/lib/failing-fe-resolver.spec.ts | 183 + src/lib/failing-fe-resolver.ts | 155 + src/lib/http.runs.spec.ts | 526 + src/lib/http.test.ts | 511 + src/lib/http.ts | 809 ++ src/lib/output.test.ts | 106 + src/lib/output.ts | 109 + src/lib/pagination.test.ts | 116 + src/lib/pagination.ts | 131 + src/lib/poll.spec.ts | 796 ++ src/lib/poll.ts | 308 + src/lib/prompt.test.ts | 79 + src/lib/prompt.ts | 116 + src/lib/rate-throttle.spec.ts | 133 + src/lib/rate-throttle.ts | 77 + src/lib/render-error.test.ts | 90 + src/lib/render-error.ts | 46 + src/lib/runs.types.ts | 391 + src/lib/target-url.spec.ts | 270 + src/lib/target-url.ts | 152 + src/lib/ticker.spec.ts | 224 + src/lib/ticker.ts | 81 + src/lib/validate.spec.ts | 411 + src/lib/validate.ts | 225 + src/version.test.ts | 12 + src/version.ts | 1 + test/__snapshots__/help.snapshot.test.ts.snap | 600 ++ test/cli.subprocess.test.ts | 1102 +++ test/contract/p4-schema.test.ts | 348 + test/contract/p5-schema.test.ts | 233 + test/e2e/agent-install.e2e.test.ts | 650 ++ test/e2e/init.e2e.test.ts | 266 + test/help.snapshot.test.ts | 61 + test/helpers/stdoutPurity.test.ts | 84 + test/helpers/stdoutPurity.ts | 68 + test/mock-backend/fixtures.ts | 375 + test/mock-backend/handlers.smoke.test.ts | 364 + test/mock-backend/handlers.ts | 458 + test/mock-backend/index.ts | 16 + test/mock-backend/server.ts | 53 + tsconfig.build.json | 9 + tsconfig.json | 21 + vitest.config.ts | 20 + vitest.e2e.config.ts | 5 + 103 files changed, 61561 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test-coverage.yml create mode 100644 .gitignore create mode 100644 .npmrc create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 DOCUMENTATION.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 assets/testsprite-logo-dark.svg create mode 100644 assets/testsprite-logo-light.svg create mode 100644 docs/cli-v1-agent-install/skill-template.md create mode 100644 eslint.config.mjs create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 scripts/postbuild.mjs create mode 100644 src/assets/agent-skill/testsprite-verify.codex.md create mode 100644 src/assets/agent-skill/testsprite-verify.skill.md create mode 100644 src/commands/agent.test.ts create mode 100644 src/commands/agent.ts create mode 100644 src/commands/auth.test.ts create mode 100644 src/commands/auth.ts create mode 100644 src/commands/init.test.ts create mode 100644 src/commands/init.ts create mode 100644 src/commands/project.test.ts create mode 100644 src/commands/project.ts create mode 100644 src/commands/test.artifact.spec.ts create mode 100644 src/commands/test.create-batch-run.spec.ts create mode 100644 src/commands/test.quickwins.spec.ts create mode 100644 src/commands/test.rerun.spec.ts create mode 100644 src/commands/test.result.history.spec.ts create mode 100644 src/commands/test.run.spec.ts create mode 100644 src/commands/test.test.ts create mode 100644 src/commands/test.ts create mode 100644 src/commands/test.wait.spec.ts create mode 100644 src/commands/usage.test.ts create mode 100644 src/commands/usage.ts create mode 100644 src/index.ts create mode 100644 src/lib/agent-targets.test.ts create mode 100644 src/lib/agent-targets.ts create mode 100644 src/lib/bundle.test.ts create mode 100644 src/lib/bundle.ts create mode 100644 src/lib/client-factory.test.ts create mode 100644 src/lib/client-factory.ts create mode 100644 src/lib/config.test.ts create mode 100644 src/lib/config.ts create mode 100644 src/lib/credentials.test.ts create mode 100644 src/lib/credentials.ts create mode 100644 src/lib/dry-run/fetch.test.ts create mode 100644 src/lib/dry-run/fetch.ts create mode 100644 src/lib/dry-run/samples.test.ts create mode 100644 src/lib/dry-run/samples.ts create mode 100644 src/lib/errors.test.ts create mode 100644 src/lib/errors.ts create mode 100644 src/lib/facade.test.ts create mode 100644 src/lib/facade.ts create mode 100644 src/lib/failing-fe-resolver.spec.ts create mode 100644 src/lib/failing-fe-resolver.ts create mode 100644 src/lib/http.runs.spec.ts create mode 100644 src/lib/http.test.ts create mode 100644 src/lib/http.ts create mode 100644 src/lib/output.test.ts create mode 100644 src/lib/output.ts create mode 100644 src/lib/pagination.test.ts create mode 100644 src/lib/pagination.ts create mode 100644 src/lib/poll.spec.ts create mode 100644 src/lib/poll.ts create mode 100644 src/lib/prompt.test.ts create mode 100644 src/lib/prompt.ts create mode 100644 src/lib/rate-throttle.spec.ts create mode 100644 src/lib/rate-throttle.ts create mode 100644 src/lib/render-error.test.ts create mode 100644 src/lib/render-error.ts create mode 100644 src/lib/runs.types.ts create mode 100644 src/lib/target-url.spec.ts create mode 100644 src/lib/target-url.ts create mode 100644 src/lib/ticker.spec.ts create mode 100644 src/lib/ticker.ts create mode 100644 src/lib/validate.spec.ts create mode 100644 src/lib/validate.ts create mode 100644 src/version.test.ts create mode 100644 src/version.ts create mode 100644 test/__snapshots__/help.snapshot.test.ts.snap create mode 100644 test/cli.subprocess.test.ts create mode 100644 test/contract/p4-schema.test.ts create mode 100644 test/contract/p5-schema.test.ts create mode 100644 test/e2e/agent-install.e2e.test.ts create mode 100644 test/e2e/init.e2e.test.ts create mode 100644 test/help.snapshot.test.ts create mode 100644 test/helpers/stdoutPurity.test.ts create mode 100644 test/helpers/stdoutPurity.ts create mode 100644 test/mock-backend/fixtures.ts create mode 100644 test/mock-backend/handlers.smoke.test.ts create mode 100644 test/mock-backend/handlers.ts create mode 100644 test/mock-backend/index.ts create mode 100644 test/mock-backend/server.ts create mode 100644 tsconfig.build.json create mode 100644 tsconfig.json create mode 100644 vitest.config.ts create mode 100644 vitest.e2e.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..660df82 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,97 @@ +name: CI + +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: [main, dev, stg] + +jobs: + lint: + name: Lint & Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + + - name: ESLint + run: npm run lint + + - name: Prettier + run: npm run format:check + + typecheck: + name: Typecheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm run typecheck + + test: + name: Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm test + env: + CI: true + + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm run build + + - name: Smoke test built CLI + run: | + node dist/index.js --version + node dist/index.js --help + + e2e: + name: Local E2E Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm run test:e2e + env: + CI: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..d9c616f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,25 @@ +name: release +on: + push: + tags: ['v*.*.*'] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # npm provenance + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + registry-url: 'https://registry.npmjs.org' + - run: npm ci + - run: npm run lint + - run: npm run typecheck + - run: npm run format:check + - run: npm run test:coverage + - run: npm run build + - run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 0000000..8409a99 --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,81 @@ +name: Test Coverage + +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: [main, dev, stg] + +permissions: + contents: read + pull-requests: write + +jobs: + coverage: + name: Coverage (>= 80%) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + + - name: Run unit tests with coverage + run: npm run test:coverage + env: + CI: true + + - name: Enforce 80% line coverage minimum + run: | + if [ -f coverage/coverage-summary.json ]; then + LINES_PCT=$(jq -r '.total.lines.pct' coverage/coverage-summary.json) + echo "Line coverage: ${LINES_PCT}%" + if (( $(echo "$LINES_PCT < 80" | bc -l) )); then + echo "::error::Line coverage ${LINES_PCT}% is below the 80% minimum threshold" + exit 1 + fi + else + echo "::error::Coverage report not generated" + exit 1 + fi + + - name: Generate Coverage Summary + id: coverage-summary + if: always() + run: | + if [ -f coverage/coverage-summary.json ]; then + TOTAL_LINES=$(jq -r '.total.lines.pct' coverage/coverage-summary.json) + TOTAL_STATEMENTS=$(jq -r '.total.statements.pct' coverage/coverage-summary.json) + TOTAL_FUNCTIONS=$(jq -r '.total.functions.pct' coverage/coverage-summary.json) + TOTAL_BRANCHES=$(jq -r '.total.branches.pct' coverage/coverage-summary.json) + + { + echo "## Test Coverage Report" + echo "" + echo "| Metric | Coverage |" + echo "|--------|----------|" + echo "| Lines | ${TOTAL_LINES}% |" + echo "| Statements | ${TOTAL_STATEMENTS}% |" + echo "| Functions | ${TOTAL_FUNCTIONS}% |" + echo "| Branches | ${TOTAL_BRANCHES}% |" + } | tee -a "$GITHUB_STEP_SUMMARY" > coverage-report.md + + echo "coverage_generated=true" >> "$GITHUB_OUTPUT" + else + echo "## Coverage Report Not Available" >> "$GITHUB_STEP_SUMMARY" + echo "Coverage data was not generated. Check the test run for errors." >> "$GITHUB_STEP_SUMMARY" + echo "coverage_generated=false" >> "$GITHUB_OUTPUT" + fi + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' + with: + recreate: true + path: coverage-report.md + continue-on-error: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d313bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +node_modules/ +dist/ +coverage/ +*.log +*.tgz +.DS_Store +.env +.env.* +!.env.example +.idea/ +.vscode/ + +# Live-dev e2e harness — operator-supplied fixtures and credentials must never be committed +test/dev-e2e/fixtures.local.json +test/dev-e2e/.env.local + +# CLI artifact downloads (default output dir for test artifact get) +.testsprite/ + +# Per-session Claude Code worktrees (parallel-session scratch space). Other +# `.claude/` content (e.g., `skills/`) stays tracked. +.claude/worktrees/ diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..b6f27f1 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..b00f087 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,8 @@ +dist/ +coverage/ +node_modules/ +*.tgz +package-lock.json +.claude/worktrees/ + + diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..50f03be --- /dev/null +++ b/.prettierrc @@ -0,0 +1,7 @@ +{ + "singleQuote": true, + "trailingComma": "all", + "printWidth": 100, + "arrowParens": "avoid", + "semi": true +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..36dc78e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,110 @@ +# Changelog + +All notable changes to `@testsprite/testsprite-cli` are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-06-10 + +### Added + +- **`testsprite init`** — one-shot onboarding command that chains `auth configure` → `auth whoami` → `agent install` in a single interactive invocation. Accepts `--from-env`, `--yes`, and `--agent ` for non-interactive and CI use. + +- **`agent install` / `agent list`** — write a ready-made TestSprite verification-loop skill file into your project so your coding agent knows the commands, the exit codes, and the failure-bundle layout. Pure-local command: no network, no credentials. Supported targets: `claude` (GA), `codex`, `cursor`, `cline`, `antigravity` (experimental). The `codex` target uses managed-section mode that writes a sentinel-delimited block inside `AGENTS.md` without clobbering surrounding content. `--force` backs up existing own-file targets before overwriting. + +- **`auth configure` / `auth whoami` / `auth logout`** — API-key management. `--from-env` reads `TESTSPRITE_API_KEY` for non-interactive setup. Credentials stored at `~/.testsprite/credentials` (INI, mode `0600`). + +- **`project list` / `project get`** — cursor-paginated project listing and single-project lookup. + +- **`test list` / `test get`** — cursor-paginated test listing under a project (with `--status`, `--type`, `--created-from` filters) and single-test lookup. + +- **`test create`** — create a frontend or backend test. Backend tests supply a code file directly (`--code-file`); frontend tests use `--code-file` or generate from a plan-steps document (`--plan-from`). The `--run --wait` flags chain create → trigger → poll in one invocation. Dependency metadata flags for backend tests: `--produces ` (repeatable), `--needs ` (repeatable), `--category `. + +- **`test create-batch`** — bulk-create frontend tests from a JSONL plan file (`--plans`) or a directory of plan files (`--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. + +- **`test update ` / `test delete ` / `test delete-batch`** — metadata update (name, description) and permanent hard-delete of one or many tests. `--confirm` is required for destructive operations. `test delete-batch` supports `--all --project ` and `--status ` for bulk targeted deletes. + +- **`test code get ` / `test code put `** — read the generated test source and replace it with etag-guarded optimistic concurrency (`--expected-version`, or `--force` to skip the guard). + +- **`test plan put `** — replace a frontend test's plan-steps with a refined plan. Optional `--expected-step-count` drift guard. + +- **`project create` / `project update`** — manage projects from the CLI. Both commands pre-flight `--target-url` against local addresses for fast feedback. + +- **`test steps `** — list a test's run steps with screenshot and DOM-snapshot pointers. `--run-id ` filters to the steps of one specific run. Without the flag, returns the cumulative step log across all runs with an advisory when steps span multiple runs. + +- **`test result `** — latest result: status, started/finished timestamps, video URL, step summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). `--include-analysis` adds an inline root-cause hypothesis, recommended fix target, and failure kind. + +- **`test result --history`** — list a test's prior runs (newest-first). Filters: `--source cli|portal|mcp|schedule|github_action`, `--since 24h|7d|ISO`, `--page-size`, `--cursor`. Each row carries `runId`, `status`, `source`, `isRerun`, timestamps, `codeVersion`, and `failureKind`. A note is shown in place of a blank table for tests created before run-history tracking began. + +- **`test failure get `** — the agent entry point. Returns one self-consistent failure bundle: the failing step and its immediate neighbors with screenshots and DOM snapshots, the test source, the video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. Every artifact in the bundle shares a single `snapshotId`; the CLI refuses to stitch data from different runs or code versions. `--out ` writes the bundle atomically to disk. `--failed-only` keeps only the failing step and its neighbors. + +- **`test failure summary `** — one-screen triage card (status, failure kind, root-cause hypothesis, recommended fix target) without downloading media. + +- **`test run `** — trigger a fresh run. Without `--wait`, prints `{ runId, status: "queued", … }` and exits 0. With `--wait`, polls until terminal; exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on timeout with a `nextAction` pointing at `test wait `. Accepts `--target-url`, `--timeout`, `--idempotency-key`. + +- **`test run --all --project `** — wave-ordered fresh batch run for all (or filtered) backend tests in a project. Routes to a batch endpoint; response enumerates `accepted[]`, `conflicts[]`, `deferred[]`, `skippedFrontend[]`, and `skippedIntegration[]` so a machine consumer reading `accepted` alone can't silently undercount. `--wait` polls all dispatched run IDs concurrently. + +- **`test rerun [test-id…]`** — cheap replay of one or more tests. Frontend reruns replay the saved script verbatim (AI heal-on-drift is **on by default**, opt out with `--no-auto-heal`). Backend reruns expand the producer/teardown dependency closure; use `--skip-dependencies` for just the named test. `--all --project ` reruns every test in a project. Returns `accepted[]` plus `deferred[]` for any tests shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a retry hint. + +- **`test wait `** — block until a run reaches a terminal status. Resumes polling after a timed-out `test run --wait`, or when an agent already has a `runId`. Uses server-driven long-poll where supported; exponential backoff with `Retry-After` otherwise. + +- **`test artifact get `** — download the failure bundle for a specific run, addressed by `runId` instead of `testId`. Enforces `meta.runId === ` as an integrity check; exits 5 on mismatch. Default output directory: `./.testsprite/runs//`. + +- **`--dry-run` (global)** — every command runs end-to-end without touching the network, credentials, or the local filesystem; emits canned data matching the API contract. + +- **Global flags**: `--profile`, `--output json|text`, `--endpoint-url`, `--request-timeout `, `--verbose`, `--debug`. + +- **Pagination flags on every list**: `--page-size`, `--starting-token`, `--max-items`. + +- **`--debug` HTTP tracing** to stderr (method, URL, request-id, latency, retry decisions). The API key is never included. + +- **Dashboard URL in outputs** — commands that know both `projectId` and `testId` include a `dashboardUrl` deep-link to the TestSprite web portal in JSON output. Text mode: create paths print a `Dashboard:` line on stderr; run-completion output (`test run --wait`, `test wait`, `test rerun --wait`, `test run --all`) ends the run card with a `dashboard` line on stdout. The portal domain is resolved from the configured API endpoint per environment. + +- **TTY-gated progress ticker** — single-line in-place `\x1b[2K\r` updates during polling on TTY; completely silent on non-TTY (CI) and when `--output json` is set. + +- **AWS-CLI-style exit-code taxonomy** — see [Exit codes](#exit-codes) in README. + +### Changed + +- `blocked` is a distinct top-level status alongside `failed` (was collapsed into `failed` in earlier previews). Triage routes: `blocked` → infra (stale seed, login failure, unreachable target), not bug. + +- `test failure get` / `test steps` now synthesize a terminal `assertion` step row when no individual step is in error but the test failed at the assertion or overall-outcome layer. Previously the bundle shipped `steps: []` for these tests. Synthetic rows have no screenshot or DOM snapshot. + +- `outcomeContributesToFailure` boolean on every step row (`null` when unclassified). The text renderer prefixes contributing rows with `*` so a 50-step list highlights which rows the failure landed on. + +- `failureKind` enum widened: adds `assertion_blocked`, `routing_404`, and `network_timeout` (previously these collapsed into `unknown`). The CLI accepts unrecognized values from the wire as `unknown` so new enum values are non-breaking. + +- `recommendedFixTarget` returns `null` (not an `unknown` wrapper) when the analysis pipeline produced no fill. Applied uniformly to `/result?includeAnalysis`, `/failure`, and `/failure summary`. + +- `Test.details` debug block ships a structured `processingStatus` / `testStatus` pair alongside the previous `rawStatus` string (deprecated but preserved for the transition window). + +- `test create --run/--wait/--timeout/--target-url` fully wired: chains `POST /tests` → `POST /tests/{testId}/runs` → `GET /runs/{runId}` in one invocation. `--target-url` pre-flighted against local addresses on the client (exit 5) before the request is sent. + +- Auto-minted idempotency keys and request IDs are suppressed by default; exposed under `--verbose` for retry and support use. + +- Per-request wall-clock timeout (`--request-timeout`, default 120 s) applied to every outgoing fetch. Under `--wait`, the per-request timeout is auto-raised to cover `--timeout` + 5 s so a large batch under load is never cut at the default. + +- `test run --wait` auto-resumes on 409 `run_in_flight` by polling the existing run instead of exiting 6. An advisory is printed to stderr. Other conflict reasons and body-mismatch conflicts still propagate as exit 6. + +- Backend test `test run --wait` / `test rerun --wait` include a fallback path that reads `GET /tests/{id}/result` when the run row is not yet finalized server-side, so the verdict is reachable without waiting for a timeout. + +- `test rerun` batch `--wait` summary enumerates `deferred` and `conflicts` counts alongside `total` (dispatched) so machine readers can't silently undercount. + +### Fixed + +- `parseEnvelopeBody` now recognizes NestJS raw 404 shape, surfacing the original `Cannot POST /api/cli/v1/…` message so the user sees which endpoint isn't deployed on the current backend rather than a generic "Server error." message. + +- `test run --wait` CONFLICT auto-resume is gated on `details.reason === 'run_in_flight'` only. When `--target-url` is supplied and the in-flight run's URL differs, the CLI fetches the existing run's URL and reports a descriptive conflict (exit 6) with `nextAction: testsprite test wait `. + +- `test steps` now surfaces the synthetic terminal `assertion` step row for assertion-only failures (previously this row was wired only for `test failure get` and `test failure summary`). + +- `test create-batch --plan-from-dir`: the `MAX_BATCH_SPECS` (50) cap is enforced on valid specs after non-plan JSON files are skipped, not on the raw directory entry count. The duplicate-name advisory lookup uses a bounded 5-second deadline so a stalled listing endpoint delays `test create` by at most 5 s. + +- `localValidationError` and `ApiError.getDetail()` are shared library helpers; redundant inline cast patterns removed from call sites. + +- `engine-strict=true` in `.npmrc` so `npm install` hard-fails on Node < 20 instead of warning and proceeding. + +- Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). + +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..55f167d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# Contributing to testsprite-cli + +Thanks for contributing! Drive-by fixes and small PRs are welcome — you don't need permission to start. + +## Questions & support + +This project is young — if anything is confusing or broken, please tell us. We're responsive on all of these: + +- 💬 [Discord](https://discord.gg/W4JDrZfdB) — fastest way to reach us +- 🐛 [GitHub issues](https://github.com/TestSprite/testsprite-cli/issues) — bugs and feature requests +- 📧 [contact@testsprite.com](mailto:contact@testsprite.com) — email works too + +## Prerequisites + +- Node 20 or newer (development happens on Node 22). + +## Build from source + +```bash +npm install +npm run build # tsc → dist/ +npm link # optional: makes `testsprite` resolve to your local checkout +``` + +The compiled binary lands at `dist/index.js`. During development you can run it +directly with `node dist/index.js `. + +## Testing + +```bash +npm test # Vitest unit suite (mock-based; no network or credentials) +npm run test:coverage # Vitest with v8 coverage (>= 80% gate) +``` + +Unit tests are mock-based and run with no external dependencies — this is the +suite contributors run locally and in CI. A separate live end-to-end suite +(`npm run test:dev`) runs against an internal TestSprite backend and requires +maintainer credentials; it is not needed to contribute. + +## Lint, format, and type-check + +```bash +npm run lint # ESLint +npm run lint:fix # ESLint with --fix +npm run format # Prettier (write) +npm run format:check # Prettier (check only) +npm run typecheck # tsc --noEmit +``` + +## CI gates (required for merge) + +- ESLint + Prettier clean +- TypeScript type-check clean +- All unit tests passing +- Coverage ≥ 80% on lines / statements / functions / branches +- Build + smoke test of the CLI binary + +## Branches, pull requests, and releases + +- `main` is the only long-lived branch in this repo — it tracks the latest + released code. +- To contribute: **fork the repo, create a branch in your fork, and open a + pull request against `main`.** You don't need any access to this repo to + do that. +- Branch names use a type prefix: `feat/…`, `fix/…`, `docs/…`, `refactor/…`, `test/…`, `chore/…`. +- Commit messages follow [Conventional Commits](https://www.conventionalcommits.org) — e.g. `feat(cli): …`, `fix(http): …`. +- This repo mirrors TestSprite's internal integration branch. After your PR + is merged, maintainers fold the change into the internal source of truth, + and it ships with the next release sync — so the file you touched may be + updated again by a later sync commit. +- Releases are cut by pushing a semver git tag (`vX.Y.Z`), which publishes the + package and creates a GitHub Release. + +## Project layout + +- `src/commands/` — one module per command group (`auth`, `project`, `test`, `agent`). +- `src/lib/` — shared building blocks: `http` (client + retry/backoff), `poll` + (run polling), `output` (JSON/text rendering), `config` / `credentials` + (profile + API-key handling), `errors` (typed envelopes + exit-code mapping), + `target-url` (URL pre-flight guard), and `dry-run/` (offline sample responses). +- `src/index.ts` — CLI entry point; maps API errors to exit codes. +- `test/` — Vitest unit and snapshot suites; `test/mock-backend/` provides the + MSW-based fake API used by the unit suite. + +See [`DOCUMENTATION.md`](./DOCUMENTATION.md) for the full command reference. + +## License + +By contributing, you agree that your contributions will be licensed under the +[Apache License 2.0](./LICENSE). diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..d14bf88 --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,485 @@ +# `testsprite` CLI — Documentation + +The full reference for the TestSprite CLI: install verification, manual setup, every command with examples, configuration, scripting, and exit codes. + +> Looking for the quick tour? Start with the [README](./README.md). +> This reference will progressively move to [docs.testsprite.com](https://www.testsprite.com/docs); this file is the source of truth until then. + +## Contents + +- [Install & verify](#install--verify) +- [Manual setup](#manual-setup) +- [The complete agent loop](#the-complete-agent-loop) +- [Agent onboarding (`agent install`)](#agent-onboarding-agent-install) +- [Command reference](#command-reference) + - [Read commands](#read-commands) + - [Write commands](#write-commands) + - [Run commands](#run-commands) +- [Configuration](#configuration) +- [Output & scripting](#output--scripting) +- [Exit codes](#exit-codes) +- [Design principles](#design-principles) + +--- + +## Install & verify + +```bash +npm install -g @testsprite/testsprite-cli +testsprite --version +``` + +Or run it without installing: + +```bash +npx @testsprite/testsprite-cli --version +``` + +Requires **Node.js ≥ 20**. + +Confirm the binary works **without** configuring an API key: + +```bash +testsprite --version +testsprite project list --dry-run --output json +``` + +`--dry-run` is a global flag that skips the network, credentials, and the local filesystem and emits a canned sample matching the API contract. It's the right way to confirm an install or learn the surface before configuring auth — the response _shapes_ match the wire contract, but the data is fake. + +## Manual setup + +The recommended path is `testsprite init` (see the [README quickstart](./README.md#quickstart)). If you prefer to configure each step separately: + +### 1. Authenticate + +The CLI uses API keys. Create one from your [TestSprite dashboard](https://www.testsprite.com), then configure it: + +```bash +# Interactive — prompts for your API key (input is masked); endpoint defaults to prod +testsprite auth configure + +# Non-interactive — reads TESTSPRITE_API_KEY from the environment (CI / scripts) +TESTSPRITE_API_KEY=sk-... testsprite auth configure --from-env + +# Verify +testsprite auth whoami +``` + +Credentials are stored at `~/.testsprite/credentials` (INI-style, mode `0600`). See [Configuration](#configuration) for profiles, environment overrides, and scopes. + +### 2. Run your first test + +```bash +# Describe a behavior, trigger it, and wait for a verdict — in one call +testsprite test create \ + --project proj_xxxxxxxx --type frontend \ + --plan-from ./checkout.plan.json \ + --run --wait --timeout 600 --output json +``` + +Exit `0` means the run passed; exit `1` means it failed. When it fails, pull the bundle (next section). + +## The complete agent loop + +This is the loop a coding agent runs on its own once you've onboarded it with `testsprite agent install`: + +```bash +# (one-time, per project) teach your agent the CLI +testsprite agent install claude + +# 1 — describe the behavior you want to guarantee, run it, wait +testsprite test create --project proj_8f0f6 --type frontend \ + --plan-from ./checkout-flow.plan.json --run --wait --output json +# → exits 1: the run failed + +# 2 — pull ONE self-consistent failure bundle to ./.testsprite/failure/ +# (code + failing step + screenshots + DOM + root-cause + recommended fix) +testsprite test failure get test_3a9f21c7 --out ./.testsprite/failure + +# 3 — the agent reads the bundle, edits the code, redeploys, then replays +testsprite test rerun test_3a9f21c7 --wait --output json +# → exits 0: passed. The test now lives in your durable suite. +``` + +Every artifact in the bundle shares one `snapshotId`; the CLI will not mix a failing step from one run with source code from another. Run any command with `--dry-run` first to learn its on-disk shape with zero setup. + +## Agent onboarding (`agent install`) + +`testsprite agent install` writes a ready-made skill/instruction file into your project so your coding agent knows the commands, the exit codes, and the failure-bundle layout — no prompt engineering required. It's a pure-local command: no network, no credentials. + +```bash +testsprite agent install claude # install the skill for Claude Code +testsprite agent install codex # install into AGENTS.md for Codex (managed-section) +testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc +testsprite agent install cline # .clinerules/testsprite-verify.md +testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md +testsprite agent list # list all 5 targets with status + mode + path +``` + +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental). + +The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. + +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity) backs up the existing file to `.bak` first. + +## Command reference + +Every command supports the [global flags](#global-flags), and every example below pairs a real call with a `--dry-run` companion that works on a fresh install with no auth. + +### Read commands + +#### `testsprite project list` + +List the projects visible to your API key. Cursor-paginated. + +```bash +testsprite project list --output json +testsprite project list --dry-run --output json +``` + +Common flags: + +- `--page-size ` — server hint for items per page; the cursor token comes back in `nextToken`. Passing `--page-size` without `--max-items` returns a single page. +- `--starting-token ` — opaque cursor from a previous response. +- `--max-items ` — client-side cap on total items across auto-paged pages. + +#### `testsprite project get ` + +Get a single project by id. Project ids look like `proj_xxxxxxxx` and come from `project list`. + +```bash +testsprite project get proj_xxxxxxxx --output json +testsprite project get proj_xxxxxxxx --dry-run --output json +``` + +#### `testsprite test list --project ` + +List tests under a project. `--project` is required. Cursor-paginated. + +```bash +testsprite test list --project proj_xxxxxxxx --output json +testsprite test list --project proj_xxxxxxxx --type frontend --created-from portal +testsprite test list --project proj_xxxxxxxx --dry-run --output json +``` + +Common flags: + +- `--type ` — filter by test type. +- `--created-from ` — filter by where the test was authored. +- `--status ` — filter by status. +- `--page-size`, `--starting-token`, `--max-items` — pagination, same shape as `project list`. + +#### `testsprite test get ` + +Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. + +```bash +testsprite test get test_xxxxxxxx --output json +testsprite test get test_xxxxxxxx --dry-run --output json +``` + +#### `testsprite test code get ` + +Print the generated test source. With `--out `, write it to a file instead of stdout (text mode writes the source body; JSON mode writes the wire envelope). + +```bash +testsprite test code get test_xxxxxxxx +testsprite test code get test_xxxxxxxx --out ./test_xxxxxxxx.spec.ts +testsprite test code get test_xxxxxxxx --dry-run --output json +``` + +#### `testsprite test steps ` + +List the latest steps for a test (with screenshot / DOM-snapshot pointers). Auto-paginates by default. + +```bash +testsprite test steps test_xxxxxxxx --output json +testsprite test steps test_xxxxxxxx --dry-run --output json +``` + +Common flags: `--page-size`, `--starting-token`, `--max-items` — same shape as the other lists. + +#### `testsprite test result ` + +Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). + +```bash +testsprite test result test_xxxxxxxx --output json +testsprite test result test_xxxxxxxx --include-analysis --output json +testsprite test result test_xxxxxxxx --dry-run --output json +``` + +With `--history`, the command lists a test's **prior runs** instead of the latest result — `{ runs: [...], nextCursor }`, where each run carries `runId`, `status`, `source` (`cli | portal | mcp | schedule | github_action`), `isRerun`, `createdFrom`, timestamps, `codeVersion`, and `failureKind`. Filter with `--source ` and `--since <24h|7d|ISO>`; paginate with `--page-size` (1–100, default 20) and `--cursor`. For one run's detail use `test wait `; for its failure bundle use `test artifact get `. + +```bash +testsprite test result test_xxxxxxxx --history --output json +testsprite test result test_xxxxxxxx --history --source cli --since 7d --output json +testsprite test result test_xxxxxxxx --history --dry-run --output json +``` + +#### `testsprite test failure get ` + +The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. + +```bash +# Print the wire envelope to stdout (good for piping into jq or an LLM) +testsprite test failure get test_xxxxxxxx --output json + +# Write the bundle as a directory under --out (atomic; .partial marker on crash) +testsprite test failure get test_xxxxxxxx --out ./.testsprite/failure/test_xxxxxxxx + +# Keep only the failed step plus its immediate neighbors (±1) +testsprite test failure get test_xxxxxxxx --out ./fail --failed-only + +# Dry-run prints the canned wire envelope; with --out it prints what would be +# written (no directory is created) +testsprite test failure get test_xxxxxxxx --dry-run --output json +testsprite test failure get test_xxxxxxxx --dry-run --out ./fail +``` + +Every artifact in the bundle shares one `snapshotId`; the CLI refuses to stitch data from different runs or code versions. Run `--dry-run` once to learn the on-disk shape, then run it for real. + +#### `testsprite test failure summary ` + +One-screen agent-friendly triage card (status, failure kind, root-cause hypothesis, recommended fix target) without downloading video, screenshots, or DOM snapshots. Sibling of `test failure get` — useful when an agent only needs to decide _what kind_ of failure it is looking at. + +```bash +testsprite test failure summary test_xxxxxxxx --output json +testsprite test failure summary test_xxxxxxxx --dry-run --output json +``` + +### Write commands + +Require the `write:tests` scope. + +#### `testsprite test create` + +Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. + +```bash +# Backend test from a code file +testsprite test create --project proj_xxxxxxxx --type backend --name "Login API" \ + --code-file ./login.py + +# Frontend test from an agent-supplied plan-steps document; trigger + wait inline +testsprite test create --plan-from ./checkout.plan.json --type frontend \ + --run --wait --timeout 600 --output json + +# Dry-run prints the canned wire envelope +testsprite test create --plan-from ./checkout.plan.json --dry-run --output json +``` + +#### `testsprite test create-batch` + +Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. + +```bash +testsprite test create-batch --plans ./plans.jsonl --run --max-concurrency 4 --output json +testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json +``` + +#### `testsprite test update ` + +Update test metadata (name, description). + +```bash +testsprite test update test_xxxxxxxx --name "Renamed test" --description "Updated" +testsprite test update test_xxxxxxxx --dry-run --output json +``` + +#### `testsprite test delete ` / `test delete-batch` + +Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. + +```bash +testsprite test delete test_xxxxxxxx --confirm +testsprite test delete-batch test_aaaa test_bbbb --confirm +testsprite test delete-batch --all --project proj_xxxxxxxx --confirm +testsprite test delete test_xxxxxxxx --dry-run --output json +``` + +#### `testsprite test code put ` + +Replace the generated test code with a new file. The CLI uses an etag (`codeVersion`) for optimistic-concurrency control: it auto-fetches the current version, or pass `--expected-version` to pin one, or `--force` to skip the guard. + +```bash +testsprite test code put test_xxxxxxxx --code-file ./test.spec.ts +testsprite test code put test_xxxxxxxx --code-file ./test.spec.ts --expected-version v3 +testsprite test code put test_xxxxxxxx --code-file ./test.spec.ts --dry-run --output json +``` + +#### `testsprite test plan put ` + +Replace a frontend test's plan-steps with a refined plan. `--expected-step-count` is an optional drift guard. + +```bash +testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --expected-step-count 8 +testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --output json +``` + +#### `testsprite project create` / `project update` + +Manage projects from the CLI. Both pre-flight `--target-url` against local addresses for fast feedback. + +```bash +testsprite project create --name "Checkout" --target-url https://staging.example.com +testsprite project update proj_xxxxxxxx --name "Checkout v2" +``` + +### Run commands + +Require the `run:tests` scope. + +#### `testsprite test run ` + +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). + +```bash +# Trigger and return immediately +testsprite test run test_xxxxxxxx --output json + +# Trigger against an environment URL and wait for terminal status +testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ + --wait --timeout 600 --output json + +# Dry-run prints a canned queued response (no network, no credentials) +testsprite test run test_xxxxxxxx --dry-run --output json +``` + +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr at `--verbose`); pass `--idempotency-key ` to control it explicitly. + +#### `testsprite test rerun [test-id...]` + +Re-execute one or more tests as a cheap **replay** — distinct from `test run`, which triggers a fresh agent run that may regenerate code and spend credits. A frontend rerun replays the saved script (verbatim unless AI heal-on-drift engages — see `--auto-heal`); a backend rerun re-runs the named test together with its producer/teardown dependency closure. Without `--wait`, prints the queued run(s) and exits 0; with `--wait`, polls to terminal with the same exit-code matrix as `test run --wait`. + +```bash +# Frontend test — verbatim replay +testsprite test rerun test_xxxxxxxx --wait --output json + +# Backend test — reruns the dependency closure (producers + teardowns) +testsprite test rerun test_be_xxxx --wait --output json + +# Backend test — just the named test, skip the closure +testsprite test rerun test_be_xxxx --skip-dependencies --output json + +# Rerun every test in a project (batch) +testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json + +# Several specific tests +testsprite test rerun test_aaaa test_bbbb --wait --output json +``` + +Flags: + +- `--all` — rerun every test in the resolved project; requires `--project `. +- `--wait`, `--timeout ` — block until terminal; same exit matrix as `test run --wait`. +- `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. +- `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. +- `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. +- `--idempotency-key ` — auto-minted when omitted. + +A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. + +#### `testsprite test wait ` + +Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. + +```bash +testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json +testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json +``` + +Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. + +#### `testsprite test artifact get ` + +Download the failure bundle for a specific `runId`. Same on-disk layout as `test failure get`, but addressed by `runId` instead of `testId`, so an agent can fetch the bundle for the exact run it just triggered — never a newer failure on the same test. Default `` is `./.testsprite/runs//`. The CLI enforces `meta.runId === ` as an integrity check; a mismatch exits 5 rather than silently writing the wrong bundle. + +```bash +testsprite test artifact get run_01hx3z9p8q4k2y7a --output json +testsprite test artifact get run_01hx3z9p8q4k2y7a --out ./.testsprite/runs/run_01hx3z9p8q4k2y7a +testsprite test artifact get run_01hx3z9p8q4k2y7a --failed-only +testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json +``` + +Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). + +## Configuration + +### Profiles & credentials + +Credentials live at `~/.testsprite/credentials` (INI-style, mode `0600`) — one section per profile. **Profile resolution order** (highest first): `--profile` flag → `TESTSPRITE_PROFILE` env → `default`. Within a profile, the `TESTSPRITE_API_KEY` / `TESTSPRITE_API_URL` env vars override the file, so CI can run without ever touching `~/.testsprite/credentials`. + +### Global flags + +These apply to every command: + +| Flag | Purpose | +| ----------------------------- | ----------------------------------------------------------------------------------------------- | +| `--profile ` | Pick a named profile (default: `default`) | +| `--endpoint-url ` | Override the API host | +| `--output json\|text` | JSON is the stable automation contract; text is human-friendly | +| `--request-timeout ` | Per-request wall-clock timeout (default 120, range 1–600) | +| `--verbose` | Human-readable HTTP retry / backoff / polling messages to stderr | +| `--debug` | Method / URL / request-id / latency / retry decisions to stderr (the API key is never included) | +| `--dry-run` | Run end-to-end with no network, credentials, or filesystem writes; emits canned data | + +### Environment variables + +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | + +### Scopes + +API-key scopes gate the write and run surfaces: + +| Scope | Required by | +| --------------- | -------------------------------------------------------------------- | +| `read:me` | `auth whoami` | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `run:tests` | `test run / rerun / wait / artifact get` | + +New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. + +## Output & scripting + +JSON is the stable, machine-readable contract; pipe it straight into `jq` or a coding agent: + +```bash +# Grab the runId of a freshly triggered run +RUN_ID=$(testsprite test run test_xxxxxxxx --output json | jq -r '.runId') + +# Wait on it and branch on the exit code +testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not pass" +``` + +## Exit codes + +| Code | Meaning | +| ---- | --------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | + +## Design principles + +1. **Resource-oriented.** Verbs (`list`, `get`) operate on resources (`project`, `test`, `run`). +2. **Scriptable.** Every command supports `--output json` for machine-readable output. +3. **Stateless.** No local database; the TestSprite backend is the source of truth. +4. **Composable.** Output is pipe-friendly and pairs well with `jq`. +5. **Agent-safe.** Reads that span multiple entities share a `snapshotId` and refuse to stitch data from different runs or code versions. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f4dadbe --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 TestSprite, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..097b583 --- /dev/null +++ b/README.md @@ -0,0 +1,190 @@ +
+ + + + + + TestSprite + + + +### The verification layer for the agentic coding era. + +AI ships code in minutes — verifying it hasn't. `testsprite` opens your live app, uses it like a real user, and shows your coding agent exactly what broke — so it fixes its own work before a bug ever reaches you. + +Proof, in public: verification beats model size. With this CLI in the loop, the cheapest model in the field shipped the most correct app on an open leaderboard — 89%, at half the cost of the priciest one. See the leaderboard → + +

+ npm version + npm downloads + Node >= 20 + License Apache 2.0 + CI +

+ +

+ Website + Docs + Follow on X + LinkedIn + Join our Discord +

+ +⭐ _Help us reach more developers and grow the TestSprite community. Star this repo!_ + +
+ +https://github.com/user-attachments/assets/ab3105d5-4d5c-4a7b-b639-610f80fddc0e + +

▶ Watch the launch video — the three hard limits every coding agent hits, and the loop that breaks them (4 min).

+ +--- + +## What is it? + +[TestSprite](https://www.testsprite.com) is the AI testing platform 100,000+ teams use to test their software, frontend and backend — in the cloud, against the live product, not mocks. This repo is its official CLI. + +It puts that platform in your coding agent's hands: the agent verifies every behavior it ships, and what broke comes back as **one self-consistent bundle** it can act on — no dashboard scraping. Humans drive the same surface from a terminal or CI. + +## ⭐️ Star the Repository + +testsprite-repo-star-fast + +If you find `testsprite` useful, a GitHub Star ⭐️ would be greatly appreciated — it helps other builders (and their agents) find the project, and stars notify you about new releases. + +## Quickstart + +Requires **Node.js ≥ 20**. (No global install? `npx @testsprite/testsprite-cli` works too.) + +```bash +npm install -g @testsprite/testsprite-cli +testsprite init +``` + +`testsprite init` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `antigravity`, `codex`, etc.). Non-interactive (CI / onboarding scripts): + +```bash +TESTSPRITE_API_KEY=sk-... testsprite init --from-env --yes --agent claude +``` + +From there, the loop runs on its own — an example session, typed by the coding agent: + +```bash +# 1 — describe the behavior you want to guarantee, run it, wait +testsprite test create --project proj_8f0f6 --type frontend \ + --plan-from ./checkout-flow.plan.json --run --wait --output json +# → exits 1: the run failed + +# 2 — pull ONE self-consistent failure bundle +testsprite test failure get test_3a9f21c7 --out ./.testsprite/failure + +# 3 — the agent reads the bundle, fixes the code, then replays +testsprite test rerun test_3a9f21c7 --wait --output json +# → exits 0: passed. The test now lives in your durable suite. +``` + +Prefer to configure each step by hand (or learn the surface offline with `--dry-run` first)? See [Manual setup](./DOCUMENTATION.md#manual-setup) and [Install & verify](./DOCUMENTATION.md#install--verify). + +## Commands + +| Group | Command | What it does | +| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| **Init** | `init` | One-shot onboarding: auth configure + whoami + agent install | +| **Auth** | `auth configure` | Store an API key at `~/.testsprite/credentials` | +| | `auth whoami` _(alias `status`)_ | Resolve the active profile to its user, key, env, and scopes | +| | `auth logout` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Onboard a coding agent (pure-local); targets: `claude`, `codex`, `cursor`, `cline`, `antigravity` | + +📚 **Full reference — every command, flag, and example:** [DOCUMENTATION.md](./DOCUMENTATION.md), including [configuration & profiles](./DOCUMENTATION.md#configuration), [scripting](./DOCUMENTATION.md#output--scripting), and [exit codes](./DOCUMENTATION.md#exit-codes). + +## Why a CLI for coding agents? + +- 🧪 **Tests like a real user.** Runs against a live browser or API in the cloud — real clicks, real navigation, real assertions. Not a mock. +- 🤖 **Agent-shaped output.** `test failure get` returns **one bundle** — the failing step, its neighbors, screenshots, DOM snapshots, the test source, a root-cause hypothesis, and a recommended fix target — all sharing a single `snapshotId`. The CLI _refuses_ to stitch data from two different runs, so an agent never reasons over a frankenstein context. +- ♻️ **A loop, not a one-shot.** `create → run → failure get → fix → rerun` — every pass is banked, not thrown away. +- 📐 **Scriptable & deterministic.** Stable `--output json` contract, predictable [exit codes](./DOCUMENTATION.md#exit-codes), and a `--dry-run` that exercises the full code path offline with canned data. +- 🔌 **One command to onboard your agent.** `testsprite agent install claude` drops a ready-made skill file into your repo so your coding agent knows how to drive the loop on its own. + +## How it works + +Every time your agent changes code, it asks one question: **is this behavior already covered by the suite?** + +- **Not yet covered** → `testsprite test create` — describe the new behavior, run it. +- **Already covered** → `testsprite test rerun` — replay the existing tests, so nothing that used to work breaks silently. +- **Something fails** → `testsprite test failure get` — one self-consistent bundle; the agent fixes the code and reruns. + +Every pass is banked into a durable suite, so coverage compounds as the project grows — a lasting record of every requirement it has ever gotten right, far bigger than any context window. + +```mermaid +flowchart TD + A["🤖 Your coding agent
Claude Code · Codex · Antigravity · Kimi · Cursor · Trae …"] + D{"behavior already
covered by the suite?"} + B["testsprite test create
new behavior → new test"] + R["testsprite test rerun
replay the existing tests"] + C{{"☁️ TestSprite testing agent
runs the test like a real user against
real browsers & real APIs on Cloud"}} + F["testsprite test failure get
ONE self-consistent bundle:
failing step · screenshots · DOM ·
root-cause · recommended fix"] + S[("📚 Durable integration suite
grows with every pass")] + + A -->|"writes / changes code"| D + D -->|"no — new behavior"| B + D -->|"yes"| R + B --> C + R --> C + C -->|"pass ✅"| S + C -->|"fail ❌"| F + F -->|"agent reads the bundle
& fixes the code"| A + S -.->|"defines what's covered"| D +``` + +The cloud is a black box on purpose: your agent describes intent and reads results. It never has to know _how_ the test was driven — only _what_ a real user experienced. + +## Proved in public + +On [**CoderCup**](https://codercup.ai) — an open leaderboard where frontier coding agents build the _same_ app under the _same_ rules, with TestSprite as the referee — the **cheapest** model in the field shipped the **most correct** app on the board: **89%**, at half the cost of the priciest one. + +That's the point of all of this: you no longer need the biggest, most expensive model to ship software you can trust — top-tier quality, without paying top-tier prices, within reach of every team. + +## Getting help + +- 📚 **CLI reference** — [DOCUMENTATION.md](./DOCUMENTATION.md) +- 🌐 **Platform docs** — [testsprite.com/docs](https://www.testsprite.com/docs) +- 🐛 **Issues & feature requests** — [GitHub issues](https://github.com/TestSprite/testsprite-cli/issues) +- 💬 **Quick questions** — [Discord](https://discord.gg/W4JDrZfdB), or `testsprite --help` / `testsprite test run --help` right in your terminal +- 📝 **Changelog** — [CHANGELOG.md](./CHANGELOG.md) + +## Contributing + +Contributions are welcome — the CLI is plain TypeScript/Node (≥ 20), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: + +```bash +git clone https://github.com/TestSprite/testsprite-cli.git +cd testsprite-cli && npm install +npm run build # tsc → dist/ ; try it: node dist/index.js --help +npm test # unit suite — no network or credentials needed +npm run lint:fix # ESLint +npm run typecheck # tsc --noEmit +``` + +Pull requests target the `dev` branch. The full guide — build from source, test tiers, CI gates, branches, project layout — is in [CONTRIBUTING.md](./CONTRIBUTING.md). + +**Support** — if you need any help, we're responsive on [Discord](https://discord.gg/W4JDrZfdB), and feel free to email us at [contact@testsprite.com](mailto:contact@testsprite.com) too. + +## License + +[Apache-2.0](./LICENSE) © TestSprite diff --git a/assets/testsprite-logo-dark.svg b/assets/testsprite-logo-dark.svg new file mode 100644 index 0000000..d519d71 --- /dev/null +++ b/assets/testsprite-logo-dark.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/assets/testsprite-logo-light.svg b/assets/testsprite-logo-light.svg new file mode 100644 index 0000000..9ecf240 --- /dev/null +++ b/assets/testsprite-logo-light.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/docs/cli-v1-agent-install/skill-template.md b/docs/cli-v1-agent-install/skill-template.md new file mode 100644 index 0000000..d051a39 --- /dev/null +++ b/docs/cli-v1-agent-install/skill-template.md @@ -0,0 +1,303 @@ +--- +name: testsprite-verify +description: TestSprite verification loop — after finishing a feature or fix in a TestSprite-tested repo, use the `testsprite` CLI to run the relevant TestSprite tests against the change and inspect any failure artifacts before reporting the work as done. Use whenever code has changed outside docs/config and is about to be reported complete — by running an existing test that covers the change, or by creating a new TestSprite test (a frontend plan, or a backend Python assertion) and running it to a terminal verdict. +--- + + + +# TestSprite Verification Loop + +The verification loop that flies your just-shipped feature through the +TestSprite CLI and reports back. + +You just finished a piece of work in a TestSprite-tested repo. Before you report +it done, **actually run the relevant TestSprite test(s)** through the `testsprite` +CLI and read the result. Spec review and unit tests catch correctness; only +running the test catches what breaks for a real user. + +## When to run + +Run after a feature or fix lands — one feature → one test run, at the moment it +lands, not batched at the end. Tests you create this way accumulate into the +project's TestSprite suite; before writing a new one, check `testsprite test list` +for an existing test that already covers the behavior and extend it instead of +duplicating. + +## When to skip + +The skip list is narrow: + +- Docs-only edits (`docs/**`, `*.md`, comments). +- Pure build/config edits (`tsconfig*`, lint/prettier config, lockfile bumps with + no behavior change). +- This repo isn't actually wired to TestSprite (no project linked, no creds). + Don't pull the user into a setup flow they didn't ask for — say so and stop. + +Otherwise, run it. + +## The one-test minimum + +Every shipped feature gets **at least one** TestSprite run to a terminal verdict +(`passed` / `failed` / `blocked` / `inconclusive`) before you call it done. What +counts: + +- `testsprite test create … --run --wait` returning a terminal verdict, **or** +- `testsprite test run --wait` against an existing test, **or** +- `testsprite test create-batch --plans plans.jsonl` to create FE tests, then + `testsprite test run --wait` on at least one of them returning a terminal verdict. + +What does **not** count: unit tests / typecheck / lint; drafting a plan without +`--run`; asking the user to run it for you. + +If you can't satisfy this — no creds, no valid target URL, repo not linked — +**say so explicitly**: "Feature shipped but I could not run any TestSprite test +because . Treat this as unverified until that's resolved." Don't claim done. + +## 1. Preflight + +```bash +testsprite --version # CLI installed? +testsprite auth whoami # credentials configured? +``` + +- `--version` fails → the CLI isn't installed. Tell the user to install the + TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. +- `auth whoami` fails → no credentials. Tell the user they can run + `testsprite auth configure`, then stop. + +## 2. Find the project + +In priority order: + +1. `$TESTSPRITE_PROJECT_ID` if set. +2. `.testsprite/config.json` in the repo root, if it has a `projectId`. +3. `testsprite project list --output json` → match a project whose `name` looks + like this repo (e.g. a `Portal` repo → the `Portal` project). +4. Still ambiguous → list the candidates and ask the user which to use (one short + question; picking the wrong project wastes a run). + +## 3. Decide what to test + +Look at the diff (`git diff --stat`, then the changed files) to understand what +user-facing behavior changed. Pick one sub-mode. For a brand-new feature the +right default is almost always (b) or (c); (a) is for tweaks to behavior an +existing test already covers. + +### (a) An existing test covers the change + +```bash +testsprite test list --project --output json +testsprite test list --project --status failed --output json # what's already red +``` + +Heuristic: a change to `src/components/CheckoutForm.tsx` is more likely covered +by "Checkout happy path" than "Login flow." Frontend change → `type: frontend`; +backend → `type: backend`. + +### (b) A new test for the change (most common) + +**Frontend — draft a `plan.json`.** You name the behavior and list steps in plain +language; you don't write browser code. + +```jsonc +{ + "projectId": "prj_abc", + "type": "frontend", + "name": "Booking date range change updates the estimated total", + "description": "When a user opens a listing and extends the booking date range, the booking-panel estimated total updates to reflect the new dates before payment.", + "priority": "p1", + "planSteps": [ + { "type": "action", "description": "Navigate to the homepage" }, + { "type": "action", "description": "Search for stays with a valid destination and date range" }, + { "type": "action", "description": "Open a listing from the search results" }, + { + "type": "action", + "description": "Select an initial short date range to view the estimated total", + }, + { "type": "action", "description": "Change the date range to a longer stay" }, + { + "type": "assertion", + "description": "Verify the estimated total on the booking panel updates from the initial value to reflect the new longer dates", + }, + ], +} +``` + +- `name` — an assertable behavior statement (subject + verb + outcome), not a noun + fragment. "Booking date range change updates the estimated total" tells the + agent what to verify; "date range" tells it nothing. +- `description` — the condition + expected outcome in one sentence; disambiguates + a short name. +- `priority` — `p0` (must-pass) / `p1` (important paths) / `p2` (edge) / `p3` + (cosmetic). Pick honestly. +- `planSteps` — `Array<{ type: 'action' | 'assertion', description: string }>`, + max 200 steps / 256 KB. Describe user intent, not selectors. + +**Backend — write the Python yourself and use `--code-file`.** There is no +server-side codegen on the CLI. Read the API surface that changed (OpenAPI, the +route handler, request/response shapes) and write a pytest-style assertion script +to a tempfile: + +```python +# /tmp/login-empty-password.py — runs against the project's target URL, creds injected. +import requests + +def test_login_rejects_empty_password(): + r = requests.post(f"{TARGET_URL}/login", json={"email": "a@b.c", "password": ""}) + assert r.status_code == 400 + assert r.json().get("error") == "invalid password" +``` + +**Show the user the drafted plan / code before creating it** — creating writes to +their project. One short confirmation; let them edit the tempfile first. + +#### Authoring planSteps the testing agent will execute reliably + +Most "the agent looks broken" outcomes trace back to plan wording, not agent +bugs. A loose plan produces a confidently-wrong `passed` (bare visibility +assertions) or a thrashing `blocked`. Write to these rules first: + +- **One verb per step.** `and` / `then` between two verbs → split. +- **Describe outcomes, not selectors.** Write `"Submit the form"`, not `"Click the +blue 'Submit' button"`. The agent does its own semantic match; a guessed label + causes brittle exact-match thrash. +- **`Navigate to` only on step 1** (or a login page). Use clicks for every later + transition — direct URLs break SPA routing. +- **Name the target entity type, not a fuzzy noun.** If a page has several + look-alike row kinds, "open a thing" lets the agent pick the wrong kind. Name + the type and the page/view you expect to land on. +- **1–2 assertions, at the end, on content existence** rather than UI copy / + formatting. (Quoting a literal you submitted earlier in the same plan, to verify + it round-trips, is fine.) +- **Assertion targets name the specific page region** (panel / tab / output area). + Otherwise the agent settles for "some element with that text is visible + anywhere," which passes for wrong reasons. +- **Keep `description` and `planSteps` in scope agreement** — the agent reads + `description` as the test's purpose and shrinks execution to match. If you change + the steps, re-align the description. +- **Skip flows the agent fails unpredictably on:** OAuth/SSO/2FA, file + upload/download, drag-and-drop, iframes, native dialogs, multi-tab, external + service state. Pick a different verification. +- **Test data must be self-contained** — create what you need within the plan, or + don't write the plan. + +### (c) A coverage set — multiple FE plans as a batch + +When a FE feature has distinct paths (happy + error + edge), draft a +`plans.jsonl`, one spec per line. Aim for 2–5 plans; more only with explicit user +confirmation. Max 50 plans / 5 MB per batch. + +```jsonc +// plans.jsonl — FE only +{"projectId":"prj_abc","type":"frontend","name":"login happy path","planSteps":[...]} +{"projectId":"prj_abc","type":"frontend","name":"login wrong password","planSteps":[...]} +{"projectId":"prj_abc","type":"frontend","name":"login empty email","planSteps":[...]} +``` + +Batch is **FE-only.** For 3 backend tests, run `test create --type backend +--project --code-file ` three times with three Python files. + +### (d) Refine an existing test + +- **FE** — replace the steps, then re-run: + ```bash + testsprite test plan put --steps refined.json + ``` +- **BE** — write updated Python and replace the code (optimistic concurrency): + ```bash + testsprite test code put --code-file refined.py --expected-version + ``` + +## 4. Run + +All variants use `--wait` for a synchronous verdict, `--target-url ` for +the deployment under test, and `--timeout 600` as a sane default. + +```bash +# (a) existing test +testsprite test run --target-url --wait --timeout 600 --output json + +# (b-FE) new FE test from plan +testsprite test create --plan-from plan.json --run --wait --target-url --timeout 600 + +# (b-BE) new BE test from Python (backend create needs --project) +testsprite test create --type backend --name "..." --project --code-file foo.py --run --wait --target-url --timeout 600 + +# (c) FE coverage set — create-batch only CREATES (FE-only); trigger each created test after +testsprite test create-batch --plans plans.jsonl --output json # prints the created test ids +# then trigger each id: testsprite test run --wait --target-url --timeout 600 + +# (d-FE) refine + re-run +testsprite test plan put --steps refined.json && \ +testsprite test run --target-url --wait --timeout 600 +``` + +Key behaviors: + +- `--target-url` must be an allowed project/environment URL. The CLI rejects + `localhost` / RFC1918 / link-local — local-only changes can't be run here. If + the feature is deployed only locally, say so and skip the run. +- `--wait` long-polls until terminal and handles its own backoff — don't wrap it + in a retry loop. +- Exit codes: `0` = passed; `1` = failed / blocked / cancelled; `7` = timeout. + Treat `7` as inconclusive (resume with `testsprite test wait `), not a + regression. +- Batch: `create-batch` only creates the tests (FE-only) — it does not run them + (`--run` is unsupported, exits 7). Trigger each created test with + `test run --wait` and read each verdict individually. +- `test code put` needs `--expected-version `; stale etag → exit 6, + re-fetch via `test get ` and retry. `--force` bypasses but is audit-logged. +- Idempotency: the CLI auto-mints an `Idempotency-Key`; replays within 24h return + the original test/run. + +## 4a. Read the result — plan, or product? + +Don't take the verdict at face value. After a run, pull the steps +(`testsprite test steps --output json`) and ask: **plan quality, or +actual product behavior?** + +The plan is the problem (most common — check first) when: + +- recorded action count < plan action count → the agent skipped steps it couldn't + disambiguate; +- assertions degraded to bare "Verify element is visible" with no subject; +- 5+ consecutive click attempts on similar targets with no progress. + +The product/environment is the problem (believe these) when: + +- `cause` / `error` names a concrete app observation ("the panel renders + read-only", "page 404s"); +- the trace blames infra (deploy lag, auth gate, missing fixture). + +Scope step counts to the **current run** — `test steps` is cumulative across runs; +filter on the run-id before counting. + +If it's the plan: tighten via `test plan put` and re-run **once** (two runs total), +then stop. Don't grind. If it's the product/env: report the verdict with the +agent's observation; don't auto-fix on the recommendation alone. If you genuinely +can't tell: report `inconclusive` with the signal that triggered the call and ask. + +## 5. On failure → download the artifact + +```bash +testsprite test artifact get --out ./.testsprite/runs// +``` + +Inspect the failure bundle (result, failed step + neighbors, video, root-cause +hypothesis, recommended fix target) before deciding whether your change caused it. + +## 6. Report + +When you tell the user the feature is done, include: + +- Which test(s) you ran — one line each: id, name, verdict. At least one terminal + verdict is required; zero means the feature is **not** done — surface that. +- The verdict. Don't report `passed` if §4a's sanity check tripped — surface as + `inconclusive` with the specific signal. +- If failed, a one-line summary of the bundle's root-cause hypothesis and + recommended fix target. **Don't auto-fix** on that alone — the recommendation + can be wrong; the human should look. diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..e415e8c --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,27 @@ +import js from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import prettier from 'eslint-config-prettier'; +import globals from 'globals'; + +export default tseslint.config( + { + ignores: ['dist/**', 'coverage/**', 'node_modules/**', 'perf/**', '.claude/worktrees/**'], + }, + js.configs.recommended, + ...tseslint.configs.recommended, + { + languageOptions: { + ecmaVersion: 2022, + sourceType: 'module', + globals: { + ...globals.node, + }, + }, + rules: { + '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + '@typescript-eslint/consistent-type-imports': 'error', + 'no-console': 'off', + }, + }, + prettier, +); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..860f65e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,4284 @@ +{ + "name": "@testsprite/testsprite-cli", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@testsprite/testsprite-cli", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "commander": "^12.1.0", + "valibot": "^1.4.1" + }, + "bin": { + "testsprite": "dist/index.js" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "@types/node": "^22.9.0", + "@vitest/coverage-v8": "^2.1.4", + "eslint": "^9.14.0", + "eslint-config-prettier": "^9.1.0", + "globals": "^15.12.0", + "msw": "^2.14.3", + "prettier": "^3.3.3", + "typescript": "^5.6.3", + "typescript-eslint": "^8.13.0", + "vitest": "^2.1.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", + "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", + "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@inquirer/ansi": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.5.tgz", + "integrity": "sha512-doc2sWgJpbFQ64UflSVd17ibMGDuxO1yKgOgLMwavzESnXjFWJqUeG8saYosqKpHp4kWiM5x1nXvEjbpx90gzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.0.12.tgz", + "integrity": "sha512-h9FgGun3QwVYNj5TWIZZ+slii73bMoBFjPfVIGtnFuL4t8gBiNDV9PcSfIzkuxvgquJKt9nr1QzszpBzTbH8Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.9", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "11.1.9", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.1.9.tgz", + "integrity": "sha512-BDE4fG22uYh1bGSifcj7JSx119TVYNViMhMu85usp4Fswrzh6M0DV3yld64jA98uOAa2GSQ4Bg4bZRm2d2cwSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.5", + "@inquirer/figures": "^2.0.5", + "@inquirer/type": "^4.0.5", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.5.tgz", + "integrity": "sha512-NsSs4kzfm12lNetHwAn3GEuH317IzpwrMCbOuMIVytpjnJ90YYHNwdRgYGuKmVxwuIqSgqk3M5qqQt1cDk0tGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + } + }, + "node_modules/@inquirer/type": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.5.tgz", + "integrity": "sha512-aetVUNeKNc/VriqXlw1NRSW0zhMBB0W4bNbWRJgzRl/3d0QNDQFfk0GO5SDdtjMZVg6o8ZKEiadd7SCCzoOn5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mswjs/interceptors": { + "version": "0.41.8", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.41.8.tgz", + "integrity": "sha512-pRLMNKTSGRoLq+KnEB/7OY5vijw1XmcheAAOiv6pj7W1FG32kAGqj1C/RK/cqxRGr1Fh+zBi8sDur8kj3EQv6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mswjs/interceptors/node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/deferred-promise": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-3.0.0.tgz", + "integrity": "sha512-XW375UK8/9SqUVNVa6M0yEy8+iTi4QN5VZ7aZuRFQmy76LRwI9wy5F4YIBU6T+eTe2/DNDo8tqu8RHlwLHM6RA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", + "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz", + "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz", + "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz", + "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz", + "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz", + "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz", + "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz", + "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz", + "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz", + "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz", + "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz", + "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz", + "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz", + "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz", + "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz", + "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz", + "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz", + "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz", + "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz", + "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz", + "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz", + "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz", + "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz", + "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz", + "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/set-cookie-parser": { + "version": "2.4.10", + "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.10.tgz", + "integrity": "sha512-GGmQVGpQWUe5qglJozEjZV/5dyxbOOZ0LHe/lqyWssB88Y4svNfst0uqBVscdDeIKl5Jy5+aPSvy7mI9tYRguw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/statuses": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz", + "integrity": "sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.1.tgz", + "integrity": "sha512-BOziFIfE+6osHO9FoJG4zjoHUcvI7fTNBSpdAwrNH0/TLvzjsk2oo8XSSOT2HhqUyhZPfHv4UOffoJ9oEEQ7Ag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.59.1", + "@typescript-eslint/type-utils": "8.59.1", + "@typescript-eslint/utils": "8.59.1", + "@typescript-eslint/visitor-keys": "8.59.1", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.59.1", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.1.tgz", + "integrity": "sha512-HDQH9O/47Dxi1ceDhBXdaldtf/WV9yRYMjbjCuNk3qnaTD564qwv61Y7+gTxwxRKzSrgO5uhtw584igXVuuZkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.59.1", + "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/typescript-estree": "8.59.1", + "@typescript-eslint/visitor-keys": "8.59.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.1.tgz", + "integrity": "sha512-+MuHQlHiEr00Of/IQbE/MmEoi44znZHbR/Pz7Opq4HryUOlRi+/44dro9Ycy8Fyo+/024IWtw8m4JUMCGTYxDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.59.1", + "@typescript-eslint/types": "^8.59.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.1.tgz", + "integrity": "sha512-LwuHQI4pDOYVKvmH2dkaJo6YZCSgouVgnS/z7yBPKBMvgtBvyLqiLy9Z6b7+m/TRcX1NFYUqZetI5Y+aT4GEfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/visitor-keys": "8.59.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.1.tgz", + "integrity": "sha512-/0nEyPbX7gRsk0Uwfe4ALwwgxuA66d/l2mhRDNlAvaj4U3juhUtJNq0DsY8M2AYwwb9rEq2hrC3IcIcEt++iJA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.1.tgz", + "integrity": "sha512-klWPBR2ciQHS3f++ug/mVnWKPjBUo7icEL3FAO1lhAR1Z1i5NQYZ1EannMSRYcq5qCv5wNALlXr6fksRHyYl7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/typescript-estree": "8.59.1", + "@typescript-eslint/utils": "8.59.1", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.1.tgz", + "integrity": "sha512-ZDCjgccSdYPw5Bxh+my4Z0lJU96ZDN7jbBzvmEn0FZx3RtU1C7VWl6NbDx94bwY3V5YsgwRzJPOgeY2Q/nLG8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.1.tgz", + "integrity": "sha512-OUd+vJS05sSkOip+BkZ/2NS8RMxrAAJemsC6vU3kmfLyeaJT0TftHkV9mcx2107MmsBVXXexhVu4F0TZXyMl4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.59.1", + "@typescript-eslint/tsconfig-utils": "8.59.1", + "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/visitor-keys": "8.59.1", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.1.tgz", + "integrity": "sha512-3pIeoXhCeYH9FSCBI8P3iNwJlGuzPlYKkTlen2O9T1DSeeg8UG8jstq6BLk+Mda0qup7mgk4z4XL4OzRaxZ8LA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.59.1", + "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/typescript-estree": "8.59.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.1.tgz", + "integrity": "sha512-LdDNl6C5iJExcM0Yh0PwAIBb9PrSiCsWamF/JyEZawm3kFDnRoaq3LGE4bpyRao/fWeGKKyw7icx0YxrLFC5Cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.59.1", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@vitest/coverage-v8": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.1.9.tgz", + "integrity": "sha512-Z2cOr0ksM00MpEfyVE8KXIYPEcBFxdbLSs56L8PO0QQMxt/6bDj45uQfxoc96v05KW3clk7vvgP0qfDit9DmfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.3.0", + "@bcoe/v8-coverage": "^0.2.3", + "debug": "^4.3.7", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-lib-source-maps": "^5.0.6", + "istanbul-reports": "^3.1.7", + "magic-string": "^0.30.12", + "magicast": "^0.3.5", + "std-env": "^3.8.0", + "test-exclude": "^7.0.1", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "2.1.9", + "vitest": "2.1.9" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, + "node_modules/@vitest/expect": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", + "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", + "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.12" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.9.tgz", + "integrity": "sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", + "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "2.1.9", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", + "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "magic-string": "^0.30.12", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", + "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^3.0.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", + "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "loupe": "^3.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", + "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.5", + "@eslint/js": "9.39.4", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-config-prettier": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.2.tgz", + "integrity": "sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ==", + "dev": true, + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.0.tgz", + "integrity": "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", + "dev": true, + "license": "ISC" + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "15.15.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-15.15.0.tgz", + "integrity": "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graphql": { + "version": "16.13.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.13.2.tgz", + "integrity": "sha512-5bJ+nf/UCpAjHM8i06fl7eLyVC9iuNAjm9qzkiu2ZGhM0VscSvS6WDPfAwkdkBuoXGM9FJSbKl6wylMwP9Ktig==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/headers-polyfill": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-5.0.1.tgz", + "integrity": "sha512-1TJ6Fih/b8h5TIcv+1+Hw0PDQWJTKDKzFZzcKOiW1wJza3XoAQlkCuXLbymPYB8+ZQyw8mHvdw560e8zVFIWyA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/set-cookie-parser": "^2.4.10", + "set-cookie-parser": "^3.0.1" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", + "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.23", + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/magicast": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.3.5.tgz", + "integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.25.4", + "@babel/types": "^7.25.4", + "source-map-js": "^1.2.0" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/msw": { + "version": "2.14.3", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.14.3.tgz", + "integrity": "sha512-kk8G5cocVlJ4wsKMGZegn2H6XLOEKjbA+nSJE2354e/SRp4mDicCHUYnMXpymzVcVDCs+GUAsmNqSn+yHv4T2A==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@inquirer/confirm": "^6.0.11", + "@mswjs/interceptors": "^0.41.3", + "@open-draft/deferred-promise": "^3.0.0", + "@types/statuses": "^2.0.6", + "cookie": "^1.1.1", + "graphql": "^16.13.2", + "headers-polyfill": "^5.0.1", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "path-to-regexp": "^6.3.0", + "picocolors": "^1.1.1", + "rettime": "^0.11.11", + "statuses": "^2.0.2", + "strict-event-emitter": "^0.5.1", + "tough-cookie": "^6.0.1", + "type-fest": "^5.5.0", + "until-async": "^3.0.2", + "yargs": "^17.7.2" + }, + "bin": { + "msw": "cli/index.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mswjs" + }, + "peerDependencies": { + "typescript": ">= 4.8.x" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/outvariant": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", + "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", + "dev": true, + "license": "MIT" + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.13", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.13.tgz", + "integrity": "sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prettier": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", + "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rettime": { + "version": "0.11.11", + "resolved": "https://registry.npmjs.org/rettime/-/rettime-0.11.11.tgz", + "integrity": "sha512-ILJRqVWBCTlg9r42fFgwVZx1gnFAcQF8mRoMkbgQfIrjEDf9nbBFDFx00oloOa+Q869FUtaYDXZvEfnecQSCoQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/rollup": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", + "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.60.2", + "@rollup/rollup-android-arm64": "4.60.2", + "@rollup/rollup-darwin-arm64": "4.60.2", + "@rollup/rollup-darwin-x64": "4.60.2", + "@rollup/rollup-freebsd-arm64": "4.60.2", + "@rollup/rollup-freebsd-x64": "4.60.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.2", + "@rollup/rollup-linux-arm-musleabihf": "4.60.2", + "@rollup/rollup-linux-arm64-gnu": "4.60.2", + "@rollup/rollup-linux-arm64-musl": "4.60.2", + "@rollup/rollup-linux-loong64-gnu": "4.60.2", + "@rollup/rollup-linux-loong64-musl": "4.60.2", + "@rollup/rollup-linux-ppc64-gnu": "4.60.2", + "@rollup/rollup-linux-ppc64-musl": "4.60.2", + "@rollup/rollup-linux-riscv64-gnu": "4.60.2", + "@rollup/rollup-linux-riscv64-musl": "4.60.2", + "@rollup/rollup-linux-s390x-gnu": "4.60.2", + "@rollup/rollup-linux-x64-gnu": "4.60.2", + "@rollup/rollup-linux-x64-musl": "4.60.2", + "@rollup/rollup-openbsd-x64": "4.60.2", + "@rollup/rollup-openharmony-arm64": "4.60.2", + "@rollup/rollup-win32-arm64-msvc": "4.60.2", + "@rollup/rollup-win32-ia32-msvc": "4.60.2", + "@rollup/rollup-win32-x64-gnu": "4.60.2", + "@rollup/rollup-win32-x64-msvc": "4.60.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-cookie-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.0.tgz", + "integrity": "sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/test-exclude": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.2.tgz", + "integrity": "sha512-u9E6A+ZDYdp7a4WnarkXPZOx8Ilz46+kby6p1yZ8zsGTz9gYa6FIS7lj2oezzNKmtdyyJNNmmXDppga5GB7kSw==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^10.4.1", + "minimatch": "^10.2.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/test-exclude/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-1.2.0.tgz", + "integrity": "sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", + "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tldts": { + "version": "7.0.30", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.30.tgz", + "integrity": "sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^7.0.30" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.0.30", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.30.tgz", + "integrity": "sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.6.0.tgz", + "integrity": "sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.59.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.59.1.tgz", + "integrity": "sha512-xqDcFVBmlrltH64lklOVp1wYxgJr6LVdg3NamBgH2OOQDLFdTKfIZXF5PfghrnXQKXZGTQs8tr1vL7fJvq8CTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.59.1", + "@typescript-eslint/parser": "8.59.1", + "@typescript-eslint/typescript-estree": "8.59.1", + "@typescript-eslint/utils": "8.59.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/until-async": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/until-async/-/until-async-3.0.2.tgz", + "integrity": "sha512-IiSk4HlzAMqTUseHHe3VhIGyuFmN90zMTpD3Z3y8jeQbzLIq500MVM7Jq2vUAnTKAFPJrqwkzr6PoTcPhGcOiw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/kettanaito" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/valibot": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/valibot/-/valibot-1.4.1.tgz", + "integrity": "sha512-klCmFTz2jeDluy9RwX+F884TCiogtdBJ/YaxSx1EOBYXa3NXNWj8kR1jjN8rzluwojJVWWaHJ4r1U5LfICnM3g==", + "license": "MIT", + "peerDependencies": { + "typescript": ">=5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", + "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.3.7", + "es-module-lexer": "^1.5.4", + "pathe": "^1.1.2", + "vite": "^5.0.0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", + "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "2.1.9", + "@vitest/mocker": "2.1.9", + "@vitest/pretty-format": "^2.1.9", + "@vitest/runner": "2.1.9", + "@vitest/snapshot": "2.1.9", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "debug": "^4.3.7", + "expect-type": "^1.1.0", + "magic-string": "^0.30.12", + "pathe": "^1.1.2", + "std-env": "^3.8.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.1", + "tinypool": "^1.0.1", + "tinyrainbow": "^1.2.0", + "vite": "^5.0.0", + "vite-node": "2.1.9", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/node": "^18.0.0 || >=20.0.0", + "@vitest/browser": "2.1.9", + "@vitest/ui": "2.1.9", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..804257e --- /dev/null +++ b/package.json @@ -0,0 +1,65 @@ +{ + "name": "@testsprite/testsprite-cli", + "version": "0.1.0", + "description": "Official TestSprite command-line interface", + "type": "module", + "main": "dist/index.js", + "bin": { + "testsprite": "dist/index.js" + }, + "files": [ + "dist", + "!dist/**/*.map" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json && node scripts/postbuild.mjs", + "clean": "rm -rf dist coverage", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "format": "prettier --write .", + "format:check": "prettier --check .", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage", + "test:e2e": "npm run build && vitest run -c vitest.e2e.config.ts" + }, + "engines": { + "node": ">=20" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/TestSprite/testsprite-cli.git" + }, + "bugs": { + "url": "https://github.com/TestSprite/testsprite-cli/issues" + }, + "keywords": [ + "testsprite", + "cli", + "testing", + "automation" + ], + "author": "TestSprite team", + "license": "Apache-2.0", + "dependencies": { + "commander": "^12.1.0", + "valibot": "^1.4.1" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "@types/node": "^22.9.0", + "@vitest/coverage-v8": "^2.1.4", + "eslint": "^9.14.0", + "eslint-config-prettier": "^9.1.0", + "globals": "^15.12.0", + "msw": "^2.14.3", + "prettier": "^3.3.3", + "typescript": "^5.6.3", + "typescript-eslint": "^8.13.0", + "vitest": "^2.1.4" + } +} diff --git a/scripts/postbuild.mjs b/scripts/postbuild.mjs new file mode 100644 index 0000000..9a79cae --- /dev/null +++ b/scripts/postbuild.mjs @@ -0,0 +1,18 @@ +#!/usr/bin/env node +// Post-build hook: make dist/index.js executable and copy asset tree. +// +// Replaces the previous `chmod +x dist/index.js` shell command, which +// fails on Windows (`'chmod' is not recognized`). On Windows Node only +// honors the writable/read-only bit so 0o755 is effectively a no-op for +// execute permissions, but the call is harmless. On POSIX it sets the +// executable bit as before. +// +// Asset copy: tsc only emits .ts → .js; .md assets (e.g. the agent skill +// body) are not copied by the compiler. This step mirrors src/assets/ into +// dist/assets/ so the built binary resolves them at the same relative path. +import { chmodSync, cpSync, rmSync } from 'node:fs'; +chmodSync(new URL('../dist/index.js', import.meta.url), 0o755); +rmSync(new URL('../dist/assets', import.meta.url), { recursive: true, force: true }); +cpSync(new URL('../src/assets', import.meta.url), new URL('../dist/assets', import.meta.url), { + recursive: true, +}); diff --git a/src/assets/agent-skill/testsprite-verify.codex.md b/src/assets/agent-skill/testsprite-verify.codex.md new file mode 100644 index 0000000..294038a --- /dev/null +++ b/src/assets/agent-skill/testsprite-verify.codex.md @@ -0,0 +1,119 @@ +# TestSprite Verification Loop + +After finishing a feature or fix in a TestSprite-tested repo, use the `testsprite` +CLI to run the relevant TestSprite tests against the change and inspect any failure +artifacts before reporting the work as done. Use whenever code has changed outside +docs/config and is about to be reported complete. + +## When to run + +Run after a feature or fix lands. Skip only for: docs-only edits, pure +build/config changes, or when the repo has no TestSprite project linked. + +## Core loop + +### 1. Preflight + +```bash +testsprite --version # CLI installed? +testsprite auth whoami # credentials valid? +``` + +If `--version` fails, tell the user to install the CLI and stop. +If `auth whoami` fails, tell the user to run `testsprite auth configure` and stop. + +### 2. Find the project + +In order: `$TESTSPRITE_PROJECT_ID` → `.testsprite/config.json` → `testsprite project list --output json`. + +### 3. Run + +```bash +# New frontend test from plan (most common) +testsprite test create --plan-from plan.json --run --wait \ + --target-url https://staging.example.com --timeout 600 --output json + +# Existing test +testsprite test run --target-url https://staging.example.com \ + --wait --timeout 600 --output json + +# New backend test from Python assertion file +testsprite test create --type backend --name "Login rejects empty password" \ + --project --code-file /tmp/test.py --run --wait --timeout 600 + +# Replay (cheaper than a fresh run — reuses saved test code) +testsprite test rerun --wait --output json + +# Backend tests sharing state: declare the dependency graph at create time; +# the wave engine orders runs (producers → consumers → teardown last) +testsprite test create --type backend --project --code-file /tmp/login.py \ + --name "login issues an auth token" --produces auth_token +testsprite test create --type backend --project --code-file /tmp/profile.py \ + --name "profile update accepts the token" --needs auth_token +testsprite test create --type backend --project --code-file /tmp/cleanup.py \ + --name "fixture user is deleted" --category teardown + +# Wave-ordered batch fresh run (BE tests, all or filtered) +testsprite test run --all --project [--filter ] \ + --wait --max-concurrency 4 --output json +``` + +**Key behaviors:** + +- `--target-url` must be publicly reachable (no localhost / RFC1918) and must + already have the change deployed (e.g. a CI preview deploy) — the CLI tests a + deployed URL, it doesn't host your environment. Running earlier verifies the + previous build. +- `--wait` long-polls until terminal. Do not wrap it in a retry loop. +- Exit `0` = passed; `1` = failed/blocked; `7` = timeout (resume with `test wait `). +- BE dependency flags (`--produces`/`--needs`/`--category`) are backend-only and + **create-only** — they can't be read back or edited later (delete + recreate to + change the graph). Don't hand-sequence `test run` calls to fake ordering; use + `test run --all` so the engine passes captured variables between waves. +- A BE `test rerun` dispatches the whole producer/teardown closure, side effects + included; `--skip-dependencies` reruns only the named test. If a producer failed + in the same closure, the consumer's failure is starvation (missing token/fixture) + — triage the producer first; it does not implicate your change. +- `create` and `--wait` output include a `dashboardUrl` — if the user wants to + inspect a test or run themselves, point them there. + +### 4. On failure — download the artifact + +```bash +testsprite test artifact get --out ./.testsprite/runs// +``` + +Inspect the bundle (failing step, screenshots, root-cause hypothesis) before +deciding whether your change caused the failure. + +### 5. One more tool — dry-run for learning + +Every command works without credentials under `--dry-run`: + +```bash +testsprite test run --dry-run --output json +testsprite test create --plan-from plan.json --dry-run --output json +``` + +## Exit-code quick reference + +| Code | Meaning | +| ---- | ------------------------------------------------- | +| 0 | Success (passed) | +| 1 | Failed / blocked / cancelled | +| 3 | Auth error | +| 4 | Not found | +| 5 | Validation error | +| 6 | Conflict (already running) | +| 7 | Timeout — resume: `testsprite test wait ` | +| 11 | Rate limited (retriable) | +| 12 | Insufficient credits | + +## Bootstrap (first-time setup) + +```bash +npm install -g @testsprite/testsprite-cli +testsprite init # configure + verify + install agent skill in one shot +``` + +Or step-by-step: `testsprite auth configure` → `testsprite auth whoami`. diff --git a/src/assets/agent-skill/testsprite-verify.skill.md b/src/assets/agent-skill/testsprite-verify.skill.md new file mode 100644 index 0000000..34a7dcc --- /dev/null +++ b/src/assets/agent-skill/testsprite-verify.skill.md @@ -0,0 +1,423 @@ +# TestSprite Verification Loop + +The verification loop that flies your just-shipped feature through the +TestSprite CLI and reports back. + +You just finished a piece of work in a TestSprite-tested repo. Before you report +it done, **actually run the relevant TestSprite test(s)** through the `testsprite` +CLI and read the result. Spec review and unit tests catch correctness; only +running the test catches what breaks for a real user. + +## When to run + +Run after a feature or fix lands — one feature → one test run, at the moment it +lands, not batched at the end. Tests you create this way accumulate into the +project's TestSprite suite; before writing a new one, check `testsprite test list` +for an existing test that already covers the behavior and extend it instead of +duplicating. + +The CLI tests a **deployed** URL — it doesn't build or host your environment. +Run the loop only once the change is live somewhere reachable (e.g. open the PR, +let CI deploy the preview/staging environment) and pass that URL as +`--target-url`. Running earlier verifies the previous build, not your change. + +## When to skip + +The skip list is narrow: + +- Docs-only edits (`docs/**`, `*.md`, comments). +- Pure build/config edits (`tsconfig*`, lint/prettier config, lockfile bumps with + no behavior change). +- This repo isn't actually wired to TestSprite (no project linked, no creds). + Don't pull the user into a setup flow they didn't ask for — say so and stop. + +Otherwise, run it. + +## The one-test minimum + +Every shipped feature gets **at least one** TestSprite run to a terminal verdict +(`passed` / `failed` / `blocked` / `inconclusive`) before you call it done. What +counts: + +- `testsprite test create … --run --wait` returning a terminal verdict, **or** +- `testsprite test run --wait` against an existing test, **or** +- `testsprite test create-batch --plans plans.jsonl --run --wait` (or + two-step: create-batch, then `test run --wait`) on at least one + test returning a terminal verdict. + +What does **not** count: unit tests / typecheck / lint; drafting a plan without +`--run`; asking the user to run it for you. + +If you can't satisfy this — no creds, no valid target URL, repo not linked — +**say so explicitly**: "Feature shipped but I could not run any TestSprite test +because . Treat this as unverified until that's resolved." Don't claim done. + +## 1. Preflight + +```bash +testsprite --version # CLI installed? +testsprite auth whoami # credentials configured? +``` + +- `--version` fails → the CLI isn't installed. Tell the user to install the + TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. +- `auth whoami` fails → no credentials. Tell the user they can run + `testsprite auth configure`, then stop. + +## 2. Find the project + +In priority order: + +1. `$TESTSPRITE_PROJECT_ID` if set. +2. `.testsprite/config.json` in the repo root, if it has a `projectId`. +3. `testsprite project list --output json` → match a project whose `name` looks + like this repo (e.g. a `Portal` repo → the `Portal` project). +4. Still ambiguous → list the candidates and ask the user which to use (one short + question; picking the wrong project wastes a run). + +## 3. Decide what to test + +Look at the diff (`git diff --stat`, then the changed files) to understand what +user-facing behavior changed. Pick one sub-mode. For a brand-new feature the +right default is almost always (b) or (c); (a) is for tweaks to behavior an +existing test already covers. + +### (a) An existing test covers the change + +```bash +testsprite test list --project --output json +testsprite test list --project --status failed --output json # what's already red +``` + +Heuristic: a change to `src/components/CheckoutForm.tsx` is more likely covered +by "Checkout happy path" than "Login flow." Frontend change → `type: frontend`; +backend → `type: backend`. + +### (b) A new test for the change (most common) + +**Frontend — draft a `plan.json`.** You name the behavior and list steps in plain +language; you don't write browser code. + +```jsonc +{ + "projectId": "prj_abc", + "type": "frontend", + "name": "Booking date range change updates the estimated total", + "description": "When a user opens a listing and extends the booking date range, the booking-panel estimated total updates to reflect the new dates before payment.", + "priority": "p1", + "planSteps": [ + { "type": "action", "description": "Navigate to the homepage" }, + { "type": "action", "description": "Search for stays with a valid destination and date range" }, + { "type": "action", "description": "Open a listing from the search results" }, + { + "type": "action", + "description": "Select an initial short date range to view the estimated total", + }, + { "type": "action", "description": "Change the date range to a longer stay" }, + { + "type": "assertion", + "description": "Verify the estimated total on the booking panel updates from the initial value to reflect the new longer dates", + }, + ], +} +``` + +- `name` — an assertable behavior statement (subject + verb + outcome), not a noun + fragment — e.g. "Booking date range change updates the estimated total" tells + the testing agent what to verify; "date range" tells it nothing. +- `description` — the condition + expected outcome in one sentence; disambiguates + a short name. +- `priority` — `p0` (must-pass) / `p1` (important paths) / `p2` (edge) / `p3` + (cosmetic). Pick honestly. +- `planSteps` — `Array<{ type: 'action' | 'assertion', description: string }>`, + max 200 steps / 256 KB. Describe user intent, not selectors. + +**Backend — write the Python yourself and use `--code-file`.** There is no +server-side codegen on the CLI. Read the API surface that changed (OpenAPI, the +route handler, request/response shapes) and write a pytest-style assertion script +to a tempfile: + +```python +# /tmp/login-empty-password.py — runs against the project's target URL, creds injected. +import requests + +def test_login_rejects_empty_password(): + r = requests.post(f"{TARGET_URL}/login", json={"email": "a@b.c", "password": ""}) + assert r.status_code == 400 + assert r.json().get("error") == "invalid password" +``` + +**Backend tests that share state declare dependencies at create time.** For a +one-off verification, prefer a single self-contained script (log in inside the +same file). But when the coverage set splits naturally into producer → consumer +→ cleanup — one test captures an auth token or created-resource id that others +reuse — declare the graph with the dependency flags instead of duplicating setup +in every file: + +```bash +# producer — captures a variable for later waves +testsprite test create --type backend --project --name "login issues an auth token" \ + --code-file /tmp/login.py --produces auth_token + +# consumer — scheduled in a wave after every producer it names +testsprite test create --type backend --project --name "profile update accepts the issued token" \ + --code-file /tmp/profile.py --needs auth_token + +# cleanup — always scheduled in the final wave +testsprite test create --type backend --project --name "fixture user is deleted" \ + --code-file /tmp/cleanup.py --category teardown +``` + +- `--produces ` / `--needs ` are repeatable. All three flags are + **backend-only** — combined with `--type frontend` the CLI exits 5. +- **Don't hand-sequence `test run` calls to fake ordering.** The wave engine owns + ordering: trigger the set with `test run --all` (§4) and producers run before + consumers, `teardown` last. Chaining `run A --wait && run B --wait` yourself + loses the engine's variable passing and conflicts with concurrent runs. +- **Declarations are currently create-only.** `test update` cannot amend them and + `test get` / `test list` don't echo them back, so note what each test + produces/needs as you create it; changing the graph later means delete + + recreate. + +**Show the user the drafted plan / code before creating it** — creating writes to +their project. One short confirmation; let them edit the tempfile first. + +#### Authoring planSteps the testing agent will execute reliably + +Execution reliability tracks plan wording. A loose plan produces a +confidently-wrong `passed` (bare visibility assertions) or a thrashing +`blocked`. Write to these rules first: + +- **One verb per step.** `and` / `then` between two verbs → split — e.g. + "Search for stays and open the first result" becomes two steps. +- **Describe outcomes, not selectors.** The testing agent locates targets + semantically — describe the intent and the match stays robust across copy + changes; a guessed literal label defeats that. E.g. write `"Submit the form"`, + not `"Click the blue 'Submit' button"`. +- **`Navigate to` only on step 1** (or a login page). Use clicks for every later + transition — direct URLs break SPA routing. +- **FE plans narrate a user journey through their actions.** Each action is + what a real user does (click, type, navigate, hover, scroll), forming a + sequence that exercises what gives the feature its value. The final assertion + verifies what the journey produced — the state the user arrived at, a + comparison against an earlier state, a derived count — not just that the + starting page rendered. +- **Name the target entity type, not a fuzzy noun.** If a page has several + look-alike row kinds, "open a thing" lets the testing agent pick the wrong + kind. Name the type and the page/view you expect to land on. +- **1–2 assertions, at the end, on whether the content is present _and actually + works_** (see _presence ≠ working_ below) rather than UI copy / formatting. + (Quoting a literal you submitted earlier in the same plan, to verify it + round-trips, is fine.) +- **Presence ≠ working.** A node can exist yet be broken or wrong — e.g. an + `` whose `src` 404s renders a broken-image box but still passes "element + present / has alt." Assert the thing actually **works and is correct**, not + just that the tag exists; trust the testing agent to judge correctness rather + than hard-coding the expected answer into the assertion. +- **Assertion targets name the specific page region** (panel / tab / output area). + Otherwise the testing agent settles for "some element with that text is + visible anywhere," which passes for wrong reasons. +- **For visual / CSS / layout-sensitive features, content-presence-only + assertions are dangerous.** "Verify the methodology section is visible" passes + even when the stylesheet is missing and three cards render as flat unstyled + text stacked vertically — the strings exist on the page, and a content-presence + assertion constrains nothing about layout. For any feature whose value depends on + layout, include at least one assertion that names geometry or relative + position (row, column, side-by-side, stacked, grid, above/below, span, card + background, border), not just content existence: + + ```jsonc + // BEFORE — content-presence-only (passes even on broken CSS) + { "type": "assertion", "description": "Verify the methodology section with three sub-score formulas is visible" } + + // AFTER — content + layout + { "type": "assertion", "description": "Verify the methodology section shows three card cells laid out side-by-side in a single horizontal row, each with a distinct card background and a heading; the three cards together span the full content width" } + ``` + +- **Keep `description` and `planSteps` in scope agreement** — the testing agent + reads `description` as the test's purpose and shrinks execution to match. If + you change the steps, re-align the description. +- **Keep flows that depend on state outside the test's control out of scope:** + third-party auth (OAuth/SSO/2FA), OS-native surfaces (file upload/download, + drag-and-drop, native dialogs), cross-context state (iframes, multi-tab, + external services). Pick a different verification. +- **Test data must be self-contained** — create what you need within the plan, or + don't write the plan. + +### (c) A coverage set — multiple FE plans as a batch + +When a FE feature has distinct paths (happy + error + edge), draft a +`plans.jsonl`, one spec per line. Aim for 2–5 plans; more only with explicit user +confirmation. Max 50 plans / 5 MB per batch. + +```jsonc +// plans.jsonl — FE only +{"projectId":"prj_abc","type":"frontend","name":"login happy path","planSteps":[...]} +{"projectId":"prj_abc","type":"frontend","name":"login wrong password","planSteps":[...]} +{"projectId":"prj_abc","type":"frontend","name":"login empty email","planSteps":[...]} +``` + +Batch is **FE-only.** For 3 backend tests, run `test create --type backend +--project --code-file ` three times with three Python files. + +### (d) Refine an existing test + +- **FE** — replace the steps, then re-run: + ```bash + testsprite test plan put --steps refined.json + ``` +- **BE** — write updated Python and replace the code (optimistic concurrency): + ```bash + testsprite test code put --code-file refined.py --expected-version + ``` + +## 4. Run + +All variants use `--wait` for a synchronous verdict, `--target-url ` for +the deployment under test, and `--timeout 600` as a sane default. + +```bash +# (a) existing test +testsprite test run --target-url --wait --timeout 600 --output json + +# (a-rerun) cheap replay of an existing test. FE: replays the saved script (auto-heal on — more +# lenient than a fresh run; for strict verification of a new change prefer `test run`). +# BE: dispatches the WHOLE dependency closure — producers and teardowns run too, not just , +# so expect their side effects (fixtures re-created, teardown deletes) and extra runs in history. +testsprite test rerun --wait --timeout 600 --output json +testsprite test rerun --skip-dependencies --wait --timeout 600 # BE: named test only + +# (b-FE) new FE test from plan +testsprite test create --plan-from plan.json --run --wait --target-url --timeout 600 + +# (b-BE) new BE test from Python (backend create needs --project) +testsprite test create --type backend --name "..." --project --code-file foo.py --run --wait --target-url --timeout 600 + +# (b-BE coverage set with dependencies) create each test without --run, then one wave-ordered batch +testsprite test run --all --project --wait --timeout 600 --output json +testsprite test run --all --project --filter --wait --timeout 600 # subset by name + +# (c) FE coverage set — single-call create + run (FE-only) +testsprite test create-batch --plans plans.jsonl --run --wait \ + --target-url --max-concurrency 3 --timeout 600 --output json +# Two-step alternative (inspect before running): +# testsprite test create-batch --plans plans.jsonl --output json +# then trigger each id: testsprite test run --wait --target-url --timeout 600 + +# (d-FE) refine + re-run +testsprite test plan put --steps refined.json && \ +testsprite test run --target-url --wait --timeout 600 + +# (d-BE) refine BE code + re-run. NOTE: if you reach for `test rerun` here instead, a BE rerun +# dispatches the full producer/teardown closure — use --skip-dependencies while iterating on one +# script (producers already verified this round), then finish with a closure run before reporting. +testsprite test code put --code-file refined.py --expected-version && \ +testsprite test run --target-url --wait --timeout 600 +``` + +Key behaviors: + +- `--target-url` must be an allowed project/environment URL. The CLI rejects + `localhost` / RFC1918 / link-local — local-only changes can't be run here. If + the feature is deployed only locally, say so and skip the run. +- `--wait` long-polls until terminal and handles its own backoff — don't wrap it + in a retry loop. +- Exit codes: `0` = passed; `1` = failed / blocked / cancelled; `7` = timeout. + Treat `7` as inconclusive (resume with `testsprite test wait `), not a + regression. +- Batch: `create-batch --run --wait` creates the tests (FE-only) and fans + out triggers in one call (bounded by `--max-concurrency`), emitting + `{ results: [...] }` that mirrors the single-test `test run --wait` + envelope. Exit `0` if every run passed; `1` for any failed/blocked/ + cancelled or trigger error; `7` only when EVERY run timed out. Drop + `--run` if you want to inspect the created tests before triggering. +- Run-trigger rate limit is 60/min/key, server-enforced; the CLI self-throttles + to 50/min. `--max-concurrency` (default 50) does NOT raise the server cap — + excess triggers return `RATE_LIMITED` (per-spec in batch JSON; exit 11 for + single `test run`). The test row stays created, the run never started. A full + `create-batch` (≤50 specs) fires all at once under the cap. The 50/min throttle + is per-invocation — separate batches/processes don't share it — so for >50 specs + in a minute you lean on the server's 60/min cap + `RATE_LIMITED` retries, not + client queuing. Re-list and check `draft` rows — "tests created" is + not the same as "runs triggered". +- **Backend execution is dependency-aware; frontend is not.** `test run --all +--project ` triggers a fresh wave-ordered batch over the project's backend + tests (producers → consumers → `teardown` last); FE tests in the project are + skipped with an advisory, and everything not dispatched is enumerated + (`conflicts` / `deferred` / `skippedFrontend`) so don't count `accepted[]` + alone. `test rerun ` expands the producer/teardown closure by default — + stderr summarizes `Reran N tests: 1 selected + P producer(s) + T teardown(s)`, + and failed members surface as `closureFailures[]` without flipping the exit + code. Pass `--skip-dependencies` only when this round already verified the + producers and you're iterating fast on one script. +- `test code put` needs `--expected-version `; stale etag → exit 6, + re-fetch via `test get ` and retry. `--force` bypasses but is audit-logged. +- Idempotency: the CLI auto-mints an `Idempotency-Key`; replays within 24h return + the original test/run. + +## 4a. Read the result — plan, or product? + +Don't take the verdict at face value. After a run, pull the steps +(`testsprite test steps --output json`) and ask: **plan quality, or +actual product behavior?** + +The plan is the problem (most common — check first) when: + +- recorded action count < plan action count → steps the plan left ambiguous were + skipped; +- assertions degraded to bare "Verify element is visible" with no subject; +- 5+ consecutive click attempts on similar targets with no progress. + +The product/environment is the problem (believe these) when: + +- `cause` / `error` names a concrete app observation ("the panel renders + read-only", "page 404s"); +- the trace blames infra (deploy lag, auth gate, missing fixture). + +Scope step counts to the **current run** — `test steps` is cumulative across runs; +filter on the run-id before counting. + +**Read the testing agent's failure summary skeptically — it can conflate two +failure phases.** When a flow produces an artifact that then runs against an external +target, "the backend returned an error" may mean the action endpoint itself +failed (feature broken) **or** the action succeeded and the produced artifact +failed later at runtime (orthogonal to your feature). Cross-check the step trace +and runtime output: if the runtime output references a literal you submitted +earlier in the plan, the action succeeded and the error is runtime-phase. + +**BE closure runs: root-cause the earliest failed wave, not the selected test.** +When a rerun or `run --all` expanded a dependency closure and a **producer** +failed, every consumer downstream of it fails or blocks by starvation (missing +token / missing fixture). The selected test's red verdict does not implicate +your feature — report it as failed-downstream, pull the producer's artifact +first, and only blame the consumer once its producers pass in the same run. + +If it's the plan: tighten via `test plan put` and re-run **once** (two runs total), +then stop. Don't grind. If it's the product/env: report the verdict with the +testing agent's observation; don't auto-fix on the recommendation alone. If you +genuinely can't tell: report `inconclusive` with the signal that triggered the +call and ask. + +## 5. On failure → download the artifact + +```bash +testsprite test artifact get --out ./.testsprite/runs// +``` + +Inspect the failure bundle (result, failed step + neighbors, video, root-cause +hypothesis, recommended fix target) before deciding whether your change caused it. + +## 6. Report + +When you tell the user the feature is done, include: + +- Which test(s) you ran — one line each: id, name, verdict. At least one terminal + verdict is required; zero means the feature is **not** done — surface that. +- The verdict. Don't report `passed` if §4a's sanity check tripped — surface as + `inconclusive` with the specific signal. +- If the user wants to inspect a test or run themselves, point them at the + dashboard link from the CLI output (`dashboardUrl` in JSON; a `Dashboard:` line + in text mode) — `test create` and `--wait` terminal output carry one. +- If failed, a one-line summary of the bundle's root-cause hypothesis and + recommended fix target. **Don't auto-fix** on that alone — the recommendation + can be wrong; the human should look. diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts new file mode 100644 index 0000000..bb558f3 --- /dev/null +++ b/src/commands/agent.test.ts @@ -0,0 +1,1957 @@ +import { existsSync, mkdtempSync, readFileSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError } from '../lib/errors.js'; +import { + MANAGED_SECTION_BEGIN, + MANAGED_SECTION_END, + renderForTarget, + TARGETS, + type AgentTarget, +} from '../lib/agent-targets.js'; +import type { AgentDeps, AgentFs, InstallResult, ListResult } from './agent.js'; +import { AGENTS_MD_CODEX_BUDGET_BYTES, createAgentCommand, runInstall, runList } from './agent.js'; + +// --------------------------------------------------------------------------- +// In-memory AgentFs backed by a Map +// --------------------------------------------------------------------------- + +function makeMemFs(): { + store: Map; + fs: AgentFs; + mkdirCalls: string[]; + writeCalls: string[]; + seedFile: (p: string, content: string) => void; + seedDir: (p: string) => void; + seedSymlink: (p: string) => void; +} { + const store = new Map(); // regular files: path -> content + const dirs = new Set(); // directories + const symlinks = new Set(); // symlinks (we only need to know it IS one) + const mkdirCalls: string[] = []; + const writeCalls: string[] = []; + + // Record `p` and all of its ancestors as directories, modelling a real fs + // tree so the per-component lstat walk in inspectTargetPath can traverse. + const addAncestorDirs = (p: string) => { + let cur = path.dirname(p); + while (cur !== path.dirname(cur)) { + dirs.add(cur); + cur = path.dirname(cur); + } + dirs.add(cur); // filesystem root + }; + + const agentFs: AgentFs = { + async lstat(p: string) { + if (symlinks.has(p)) return { isFile: false, isSymbolicLink: true }; + if (store.has(p)) return { isFile: true, isSymbolicLink: false }; + if (dirs.has(p)) return { isFile: false, isSymbolicLink: false }; + return null; + }, + async readFile(p: string) { + const v = store.get(p); + if (v === undefined) throw Object.assign(new Error(`ENOENT: ${p}`), { code: 'ENOENT' }); + return v; + }, + async writeFile(p: string, data: string, opts?: { exclusive?: boolean }) { + if (opts?.exclusive && (store.has(p) || dirs.has(p) || symlinks.has(p))) { + throw Object.assign(new Error(`EEXIST: ${p}`), { code: 'EEXIST' }); + } + writeCalls.push(p); + store.set(p, data); + addAncestorDirs(p); + }, + async mkdir(p: string) { + mkdirCalls.push(p); + dirs.add(p); + addAncestorDirs(p); + }, + }; + + return { + store, + fs: agentFs, + mkdirCalls, + writeCalls, + seedFile: (p, content) => { + store.set(p, content); + addAncestorDirs(p); + }, + seedDir: p => { + dirs.add(p); + addAncestorDirs(p); + }, + seedSymlink: p => { + symlinks.add(p); + addAncestorDirs(p); + }, + }; +} + +// --------------------------------------------------------------------------- +// Captured output helper +// --------------------------------------------------------------------------- + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { capture: CapturedOutput; deps: Pick } { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const CWD = '/test-project'; +const ALL_TARGETS = Object.keys(TARGETS) as AgentTarget[]; +/** own-file targets (not managed-section — codex has different install semantics) */ +const OWN_FILE_TARGETS = ALL_TARGETS.filter(t => TARGETS[t].mode === 'own-file'); + +// --------------------------------------------------------------------------- +// runInstall — fresh install per target +// --------------------------------------------------------------------------- + +describe('runInstall — fresh install', () => { + it.each(OWN_FILE_TARGETS)('writes correct file for own-file target %s', async t => { + const { store, fs: agentFs, mkdirCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [t], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { path: relPath, content } = renderForTarget(t); + const abs = path.resolve(CWD, relPath); + + // File was written to store + expect(store.get(abs)).toBe(content); + + // mkdir was called for the parent directory + expect(mkdirCalls.some(d => d === path.dirname(abs))).toBe(true); + + // stdout contains 'written' + expect(capture.stdout.join('\n')).toContain('written'); + expect(capture.stdout.join('\n')).toContain(relPath); + }); + + it('written content equals renderForTarget exactly', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(CWD, relPath); + expect(store.get(abs)).toBe(content); + }); + + it('writes to the correct matrix paths (claude and antigravity)', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'antigravity'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + const antigravityAbs = path.resolve(CWD, TARGETS.antigravity.path); + expect(store.has(claudeAbs)).toBe(true); + expect(store.has(antigravityAbs)).toBe(true); + }); + + it('cline landing path .clinerules/testsprite-verify.md', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['cline'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const abs = path.resolve(CWD, '.clinerules/testsprite-verify.md'); + expect(store.has(abs)).toBe(true); + }); + + it('cursor landing path .cursor/rules/testsprite-verify.mdc', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['cursor'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const abs = path.resolve(CWD, '.cursor/rules/testsprite-verify.mdc'); + expect(store.has(abs)).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — idempotency +// --------------------------------------------------------------------------- + +describe('runInstall — idempotency (skipped)', () => { + it('re-run with identical content → action skipped, no extra write', async () => { + const { store, fs: agentFs, writeCalls, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + // Pre-seed with the canonical content + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(CWD, relPath); + seedFile(abs, content); + + const writeCountBefore = writeCalls.length; + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // No new write happened + expect(writeCalls.length).toBe(writeCountBefore); + expect(capture.stdout.join('\n')).toContain('skipped'); + // Content unchanged + expect(store.get(abs)).toBe(content); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — conflict without --force +// --------------------------------------------------------------------------- + +describe('runInstall — conflict (blocked)', () => { + it('exits 6 when file differs and --force not set', async () => { + const { store, fs: agentFs, writeCalls, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const { path: relPath } = renderForTarget('claude'); + const abs = path.resolve(CWD, relPath); + seedFile(abs, 'DIFFERENT CONTENT'); + + const writeCountBefore = writeCalls.length; + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(6); + + // File not written + expect(writeCalls.length).toBe(writeCountBefore); + expect(store.get(abs)).toBe('DIFFERENT CONTENT'); + + // stderr has a --force hint + expect(capture.stderr.join('\n')).toContain('--force'); + // stdout has action:blocked + expect(capture.stdout.join('\n')).toContain('blocked'); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — --force +// --------------------------------------------------------------------------- + +describe('runInstall — --force', () => { + it('backs up to .bak and writes canonical content', async () => { + const { store, fs: agentFs, writeCalls, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(CWD, relPath); + const oldContent = 'OLD CONTENT'; + seedFile(abs, oldContent); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: true, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // .bak was written via writeFile + expect(writeCalls).toContain(`${abs}.bak`); + + // .bak has old content + expect(store.get(`${abs}.bak`)).toBe(oldContent); + + // File now has canonical content + expect(store.get(abs)).toBe(content); + + // Action reported as updated + expect(capture.stdout.join('\n')).toContain('updated'); + }); + + it('double --force preserves the first backup and writes a numbered .bak.1', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { deps: deps1 } = makeCapture(); + + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(CWD, relPath); + const firstEdit = 'FIRST EDIT'; + seedFile(abs, firstEdit); + + // First --force + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: true, + }, + { cwd: CWD, fs: agentFs, ...deps1 }, + ); + + // .bak holds firstEdit; abs holds canonical + expect(store.get(`${abs}.bak`)).toBe(firstEdit); + expect(store.get(abs)).toBe(content); + + // Now mutate the file again (simulate user editing after first --force) + const secondEdit = 'SECOND EDIT'; + seedFile(abs, secondEdit); + + const { deps: deps2 } = makeCapture(); + // Second --force + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: true, + }, + { cwd: CWD, fs: agentFs, ...deps2 }, + ); + + // First backup preserved (not clobbered); the second lands at .bak.1. + expect(store.get(`${abs}.bak`)).toBe(firstEdit); + expect(store.get(`${abs}.bak.1`)).toBe(secondEdit); + // abs still holds canonical + expect(store.get(abs)).toBe(content); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — --dry-run +// --------------------------------------------------------------------------- + +describe('runInstall — --dry-run', () => { + it('writes nothing to fs; emits banner + would-write lines on stderr', async () => { + const { store, fs: agentFs, writeCalls, mkdirCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: true, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // No writes + expect(writeCalls.length).toBe(0); + expect(mkdirCalls.length).toBe(0); + expect(store.size).toBe(0); + + const stderrOut = capture.stderr.join('\n'); + // Banner present + expect(stderrOut).toContain('[dry-run] no files written'); + // Would-write line present + expect(stderrOut).toContain('would write'); + // bytes count present (positive integer) + expect(stderrOut).toMatch(/\(\d+ bytes\)/); + + // stdout contains 'dry-run' action + expect(capture.stdout.join('\n')).toContain('dry-run'); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — --dir override +// --------------------------------------------------------------------------- + +describe('runInstall — --dir override', () => { + it('writes under --dir instead of cwd', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const customDir = '/custom-dir'; + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: customDir, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(customDir, relPath); + expect(store.get(abs)).toBe(content); + // Not written under CWD + const cwdAbs = path.resolve(CWD, relPath); + expect(store.has(cwdAbs)).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — unknown target +// --------------------------------------------------------------------------- + +describe('runInstall — unknown target', () => { + it('throws exit 5 with supported list, nothing written', async () => { + const { store, fs: agentFs, writeCalls } = makeMemFs(); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['bogus'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect((thrown as ApiError).exitCode).toBe(5); + // localValidationError puts the detail in nextAction; message is always 'Invalid request.' + expect((thrown as ApiError).nextAction).toContain('bogus'); + expect(writeCalls.length).toBe(0); + expect(store.size).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — multi-target +// --------------------------------------------------------------------------- + +describe('runInstall — multi-target', () => { + it('writes both claude and cursor when both specified', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'cursor'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + expect(store.has(path.resolve(CWD, TARGETS.claude.path))).toBe(true); + expect(store.has(path.resolve(CWD, TARGETS.cursor.path))).toBe(true); + expect(capture.stdout.join('\n')).toContain('written'); + }); + + it('comma-separated in single --target value works', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude,cursor'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + expect(store.has(path.resolve(CWD, TARGETS.claude.path))).toBe(true); + expect(store.has(path.resolve(CWD, TARGETS.cursor.path))).toBe(true); + }); + + it('mixed: one blocked, one fresh — fresh is still written, overall exits 6', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + // Pre-seed claude with different content + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + seedFile(claudeAbs, 'DIFFERENT CONTENT'); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'cursor'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + // Overall exit 6 + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(6); + + // cursor was still written + const cursorAbs = path.resolve(CWD, TARGETS.cursor.path); + expect(store.has(cursorAbs)).toBe(true); + + // claude not overwritten + expect(store.get(claudeAbs)).toBe('DIFFERENT CONTENT'); + + // stdout has both blocked and written + const stdoutOut = capture.stdout.join('\n'); + expect(stdoutOut).toContain('blocked'); + expect(stdoutOut).toContain('written'); + }); + + it('de-duplicates repeated targets', async () => { + const { fs: agentFs, writeCalls } = makeMemFs(); + const { deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // Only one write + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(writeCalls.filter(p => p === claudeAbs).length).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — empty target (TTY / non-TTY) +// --------------------------------------------------------------------------- + +describe('runInstall — empty target', () => { + it('non-TTY with no target throws exit 5', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect((thrown as ApiError).exitCode).toBe(5); + }); + + it('TTY with injected prompt returning "claude" installs claude', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const promptFn = vi.fn().mockResolvedValue('claude'); + + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: true, prompt: promptFn, ...deps }, + ); + + expect(promptFn).toHaveBeenCalledOnce(); + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + }); + + it('TTY with empty prompt answer defaults to claude', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const promptFn = vi.fn().mockResolvedValue(''); // empty => default to claude + + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: true, prompt: promptFn, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// runList +// --------------------------------------------------------------------------- + +describe('runList', () => { + it('returns all five targets with correct status', async () => { + const { capture, deps } = makeCapture(); + + await runList({ profile: 'default', output: 'text', debug: false, dryRun: false }, deps); + + const out = capture.stdout.join('\n'); + expect(out).toContain('claude'); + expect(out).toContain('cursor'); + expect(out).toContain('cline'); + expect(out).toContain('antigravity'); + expect(out).toContain('codex'); + expect(out).toContain('ga'); + expect(out).toContain('experimental'); + // All matrix paths present + expect(out).toContain(TARGETS.claude.path); + expect(out).toContain(TARGETS.cursor.path); + expect(out).toContain(TARGETS.cline.path); + expect(out).toContain(TARGETS.antigravity.path); + expect(out).toContain(TARGETS.codex.path); + }); + + it('JSON mode emits array of {target, status, path, mode}', async () => { + const { capture, deps } = makeCapture(); + + await runList({ profile: 'default', output: 'json', debug: false, dryRun: false }, deps); + + const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; + expect(Array.isArray(json)).toBe(true); + expect(json).toHaveLength(5); + const targets = json.map(r => r.target); + expect(targets).toContain('claude'); + expect(targets).toContain('cursor'); + expect(targets).toContain('cline'); + expect(targets).toContain('antigravity'); + expect(targets).toContain('codex'); + const claudeEntry = json.find(r => r.target === 'claude'); + expect(claudeEntry?.status).toBe('ga'); + expect(claudeEntry?.path).toBe(TARGETS.claude.path); + // codex entry has mode: managed-section + const codexEntry = json.find(r => r.target === 'codex'); + expect(codexEntry?.mode).toBe('managed-section'); + }); + + it('text mode has a header row', async () => { + const { capture, deps } = makeCapture(); + + await runList({ profile: 'default', output: 'text', debug: false, dryRun: false }, deps); + + const lines = capture.stdout.join('\n').split('\n'); + expect(lines[0]).toMatch(/TARGET/i); + expect(lines[0]).toMatch(/STATUS/i); + expect(lines[0]).toMatch(/PATH/i); + }); +}); + +// --------------------------------------------------------------------------- +// JSON vs text output shapes for install +// --------------------------------------------------------------------------- + +describe('runInstall — output modes', () => { + it('JSON mode emits array of {target, path, action}', async () => { + const { fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const json = JSON.parse(capture.stdout.join('\n')) as InstallResult[]; + expect(Array.isArray(json)).toBe(true); + expect(json[0]).toMatchObject({ target: 'claude', action: 'written' }); + expect(json[0]?.path).toBe(TARGETS.claude.path); + }); + + it('text mode emits one line per target with padded columns', async () => { + const { fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const line = capture.stdout.join('\n'); + // Should contain target name, action, and path + expect(line).toContain('claude'); + expect(line).toContain('written'); + expect(line).toContain(TARGETS.claude.path); + }); +}); + +// --------------------------------------------------------------------------- +// createAgentCommand wiring (parseAsync smoke tests) +// --------------------------------------------------------------------------- + +describe('createAgentCommand wiring', () => { + it('agent install with unknown target via parseAsync → throws CLIError exit 5', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, fs: agentFs, ...deps }); + // We need a parent for optsWithGlobals to work + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + let thrown: unknown; + try { + await parent.parseAsync(['node', 'ts', 'agent', 'install', '--target=bogus', `--dir=${CWD}`]); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeDefined(); + const isValidationErr = + (thrown instanceof ApiError && thrown.exitCode === 5) || + (thrown instanceof CLIError && thrown.exitCode === 5); + expect(isValidationErr).toBe(true); + }); + + it('agent list via parseAsync → stdout contains all targets', async () => { + const { deps, capture } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, ...deps }); + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + await parent.parseAsync(['node', 'ts', 'agent', 'list']); + + const out = capture.stdout.join('\n'); + expect(out).toContain('claude'); + expect(out).toContain('antigravity'); + }); +}); + +// --------------------------------------------------------------------------- +// All four own-file targets installed at once +// --------------------------------------------------------------------------- + +describe('runInstall — all four own-file targets', () => { + it('installs all four own-file targets in one invocation', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'cursor', 'cline', 'antigravity'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + for (const t of OWN_FILE_TARGETS) { + const abs = path.resolve(CWD, TARGETS[t].path); + expect(store.has(abs)).toBe(true); + } + + const out = capture.stdout.join('\n'); + expect(out).toContain('written'); + }); +}); + +// --------------------------------------------------------------------------- +// Dry-run for all four own-file targets +// --------------------------------------------------------------------------- + +describe('runInstall — dry-run all own-file targets', () => { + it('writes nothing for any of the four own-file targets', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: true, + target: ['claude', 'cursor', 'cline', 'antigravity'], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + expect(store.size).toBe(0); + const stderrOut = capture.stderr.join('\n'); + // Banner appears once + expect(stderrOut).toContain('[dry-run] no files written'); + // Four would-write lines + const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); + expect(wouldWriteLines.length).toBe(4); + }); +}); + +// --------------------------------------------------------------------------- +// Default AgentFs adapter (real disk I/O) — covers the adapter functions +// --------------------------------------------------------------------------- + +describe('runInstall — default AgentFs (real disk)', () => { + it('writes file to real tmpdir when no fs is injected', async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-default-')); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: tmpRoot, + }, + { ...deps }, // no fs injected → uses defaultAgentFs + ); + + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(tmpRoot, relPath); + // File exists on real disk + expect(readFileSync(abs, 'utf8')).toBe(content); + expect(capture.stdout.join('\n')).toContain('written'); + }); + + it('skips on idempotent re-run with real disk', async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-idem-')); + const { deps: deps1 } = makeCapture(); + + // First install + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: tmpRoot, + }, + { ...deps1 }, + ); + + const { capture: cap2, deps: deps2 } = makeCapture(); + // Second install → should skip + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: tmpRoot, + }, + { ...deps2 }, + ); + + expect(cap2.stdout.join('\n')).toContain('skipped'); + }); + + it('blocked on real disk when file differs, --force backs up and overwrites', async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-force-')); + const { deps: deps1 } = makeCapture(); + + // First install + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: tmpRoot, + }, + { ...deps1 }, + ); + + // Mutate the file + const { path: relPath, content } = renderForTarget('claude'); + const abs = path.resolve(tmpRoot, relPath); + const oldContent = 'MODIFIED BY USER'; + // Use default fs to write the modified content + const nodeFs = await import('node:fs/promises'); + await nodeFs.writeFile(abs, oldContent, 'utf8'); + + // --force re-run + const { capture: cap3, deps: deps3 } = makeCapture(); + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: true, + dir: tmpRoot, + }, + { ...deps3 }, + ); + + expect(cap3.stdout.join('\n')).toContain('updated'); + // .bak has old content + expect(readFileSync(`${abs}.bak`, 'utf8')).toBe(oldContent); + // File has canonical content + expect(readFileSync(abs, 'utf8')).toBe(content); + }); + + it('refuses to write through a symlinked parent dir (real disk) — exit 5', async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); + const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); + // `.claude` is a real symlink to a directory outside the project root. + symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: false, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // Nothing was created through the symlink, outside --dir. + expect(existsSync(path.join(outside, 'skills'))).toBe(false); + }); + + it('refuses to overwrite a symlinked target file (real disk) with --force — exit 5', async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); + const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); + const { path: relPath } = renderForTarget('claude'); + const abs = path.resolve(tmpRoot, relPath); + const nodeFs = await import('node:fs/promises'); + await nodeFs.mkdir(path.dirname(abs), { recursive: true }); + // SKILL.md is a real symlink to a file outside the project root. + const outsideFile = path.join(outsideDir, 'secret.txt'); + await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); + symlinkSync(outsideFile, abs, 'file'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + force: true, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // The outside file was NOT overwritten (nor clobbered via the .bak path). + expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); + }); +}); + +// --------------------------------------------------------------------------- +// Path safety — non-regular-file + intermediate guards +// --------------------------------------------------------------------------- + +const BASE_OPTS = { + profile: 'default' as const, + output: 'text' as const, + debug: false, + dryRun: false, +}; + +describe('runInstall — path safety', () => { + it('rethrows a non-ENOENT error from fs.lstat as-is', async () => { + // inspectTargetPath lstats the first component; a non-ENOENT failure + // (e.g. EPERM) must propagate unchanged rather than be swallowed. + const permError = Object.assign(new Error('EPERM: operation not permitted'), { + code: 'EPERM', + }); + const badFs: AgentFs = { + lstat: async () => { + throw permError; + }, + readFile: async () => '', + writeFile: async () => undefined, + mkdir: async () => undefined, + }; + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false }, + { cwd: CWD, fs: badFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBe(permError); + }); + + it('exit 5 when the landing path already exists as a directory', async () => { + const { fs: agentFs, writeCalls, seedDir } = makeMemFs(); + seedDir(path.resolve(CWD, TARGETS.claude.path)); // SKILL.md path is a directory + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('not a regular file'); + expect(writeCalls.length).toBe(0); + }); + + it('--force does NOT bypass the directory-at-landing-path guard', async () => { + const { fs: agentFs, writeCalls, seedDir } = makeMemFs(); + seedDir(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect(writeCalls.length).toBe(0); + }); + + it('exit 5 when an intermediate path component is a regular file, not a directory', async () => { + const { fs: agentFs, writeCalls, seedFile } = makeMemFs(); + // `.claude` exists as a FILE, so `.claude/skills/...` cannot be created. + seedFile(path.resolve(CWD, '.claude'), 'oops'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('not a directory'); + expect(writeCalls.length).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// Path safety — symlink escape (regression for the adversarial-review finding) +// --------------------------------------------------------------------------- + +describe('runInstall — symlink safety', () => { + it('refuses (exit 5) when a parent path component is a symlink', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // A planted `.claude` symlink (e.g. -> /etc) would let writes escape --dir. + seedSymlink(path.resolve(CWD, '.claude')); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + + it('refuses (exit 5) when the target file is a symlink, even with --force', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // SKILL.md itself is a symlink (e.g. -> ~/.bashrc); parents modelled as dirs. + seedSymlink(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); // never wrote a .bak nor through the link + }); + + it('does not write through a symlinked .bak slot — backs up to a numbered slot', async () => { + const { store, fs: agentFs, seedFile, seedSymlink } = makeMemFs(); + const abs = path.resolve(CWD, TARGETS.claude.path); + seedFile(abs, 'DIFFERENT CONTENT'); // real file that differs -> overwrite + seedSymlink(`${abs}.bak`); // a planted symlink at the default .bak slot + const { content } = renderForTarget('claude'); + const { deps } = makeCapture(); + + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // Exclusive create never writes through the symlink slot; backup -> .bak.1. + expect(store.has(`${abs}.bak`)).toBe(false); // symlink slot untouched + expect(store.get(`${abs}.bak.1`)).toBe('DIFFERENT CONTENT'); + expect(store.get(abs)).toBe(content); + }); +}); + +// --------------------------------------------------------------------------- +// Backup collision (regression for round-2 finding: don't clobber backups) +// --------------------------------------------------------------------------- + +describe('runInstall — backup collision', () => { + it('--force does not clobber a pre-existing regular .bak; uses .bak.1', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const abs = path.resolve(CWD, TARGETS.claude.path); + seedFile(abs, 'CURRENT EDIT'); + seedFile(`${abs}.bak`, 'PRECIOUS USER BACKUP'); // a backup the user already has + const { content } = renderForTarget('claude'); + const { capture, deps } = makeCapture(); + + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + expect(store.get(`${abs}.bak`)).toBe('PRECIOUS USER BACKUP'); // preserved + expect(store.get(`${abs}.bak.1`)).toBe('CURRENT EDIT'); // our backup + expect(store.get(abs)).toBe(content); + // The actual backup path is reported to the user. + expect(capture.stderr.join('\n')).toContain('.bak.1'); + }); + + it('fresh install: a target that races in after the path check → exit 6', async () => { + const eexist = Object.assign(new Error('EEXIST'), { code: 'EEXIST' }); + const racyFs: AgentFs = { + lstat: async () => null, // the path check sees nothing + readFile: async () => '', + writeFile: async (_p: string, _d: string, o?: { exclusive?: boolean }) => { + if (o?.exclusive) throw eexist; // but it exists by the time we write + }, + mkdir: async () => undefined, + }; + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false }, + { cwd: CWD, fs: racyFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(6); + }); +}); + +// --------------------------------------------------------------------------- +// runInstall — managed-section (codex target) +// --------------------------------------------------------------------------- + +describe('runInstall — codex managed-section: create (AGENTS.md absent)', () => { + it('creates AGENTS.md with just the section when file is absent', async () => { + const { store, fs: agentFs, writeCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const abs = path.resolve(CWD, TARGETS.codex.path); + expect(store.has(abs)).toBe(true); + const written = store.get(abs)!; + // Section sentinels are present + expect(written).toContain(MANAGED_SECTION_BEGIN); + expect(written).toContain(MANAGED_SECTION_END); + // action reported + const out = capture.stdout.join('\n'); + expect(out).toContain('section-installed'); + expect(writeCalls.some(p => p === abs)).toBe(true); + }); +}); + +describe('runInstall — codex managed-section: append (AGENTS.md exists, no sentinels)', () => { + it('appends the section to existing AGENTS.md content', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + const existingContent = '# My Project\n\nSome existing agent notes.\n'; + seedFile(agentsAbs, existingContent); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Original content preserved + expect(written).toContain('# My Project'); + expect(written).toContain('Some existing agent notes.'); + // Section appended + expect(written).toContain(MANAGED_SECTION_BEGIN); + expect(written).toContain(MANAGED_SECTION_END); + // Action reported as section-installed (first-time append = install) + expect(capture.stdout.join('\n')).toContain('section-installed'); + }); + + it('inserts a blank line separator when existing content has no trailing blank line', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // No trailing newline after last line + seedFile(agentsAbs, '# Existing\n'); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Separator between existing and section + const beginIdx = written.indexOf(MANAGED_SECTION_BEGIN); + const before = written.slice(0, beginIdx); + // Should end with two newlines (one from the original, one separator) + expect(before.endsWith('\n\n')).toBe(true); + }); +}); + +describe('runInstall — codex managed-section: replace (sentinels already present)', () => { + it('replaces the section when content differs, preserves surrounding text', async () => { + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Pre-seed with an outdated section + const oldSection = `${MANAGED_SECTION_BEGIN}\nOLD CONTENT\n${MANAGED_SECTION_END}\n`; + const beforeText = '# My Project\n\n'; + const afterText = '\n## Other section\n'; + seedFile(agentsAbs, `${beforeText}${oldSection}${afterText}`); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Before-text preserved + expect(written).toContain('# My Project'); + // After-text preserved + expect(written).toContain('## Other section'); + // Old content replaced + expect(written).not.toContain('OLD CONTENT'); + // New section present + expect(written).toContain(MANAGED_SECTION_BEGIN); + expect(written).toContain(MANAGED_SECTION_END); + // Action reported + expect(capture.stdout.join('\n')).toContain('section-updated'); + }); +}); + +describe('runInstall — codex managed-section: unchanged (byte-identical section)', () => { + it('reports section-unchanged and makes no write when content matches', async () => { + const { store, fs: agentFs, seedFile, writeCalls } = makeMemFs(); + const { deps: deps1 } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + + // First install to get the canonical section bytes + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps1 }, + ); + + // Re-read from the store to get the canonical content + const canonicalContent = store.get(agentsAbs)!; + // Re-seed to simulate no change + seedFile(agentsAbs, canonicalContent); + + const writesBeforeSecondInstall = writeCalls.length; + + const { capture: cap2, deps: deps2 } = makeCapture(); + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps2 }, + ); + + expect(writeCalls.length).toBe(writesBeforeSecondInstall); + expect(cap2.stdout.join('\n')).toContain('section-unchanged'); + // Content untouched + expect(store.get(agentsAbs)).toBe(canonicalContent); + }); +}); + +describe('runInstall — codex managed-section: corrupt sentinel → exit 5', () => { + it('throws exit 5 when BEGIN is present but END is missing', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Only BEGIN, no END + seedFile( + agentsAbs, + `# My Project\n\n${MANAGED_SECTION_BEGIN}\n# Partial section without end\n`, + ); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toMatch(/malformed|corrupt|sentinel/i); + }); + + it('throws exit 5 when END appears before BEGIN', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // END before BEGIN — reversed sentinels + seedFile( + agentsAbs, + `# My Project\n\n${MANAGED_SECTION_END}\nsome content\n${MANAGED_SECTION_BEGIN}\n`, + ); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + }); +}); + +describe('runInstall — codex managed-section: --dry-run', () => { + it('writes nothing; reports managed section in dry-run output', async () => { + const { store, fs: agentFs, writeCalls, mkdirCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { ...BASE_OPTS, dryRun: true, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + expect(store.size).toBe(0); + expect(writeCalls.length).toBe(0); + expect(mkdirCalls.length).toBe(0); + + const stderrOut = capture.stderr.join('\n'); + expect(stderrOut).toContain('[dry-run] no files written'); + expect(stderrOut).toContain('managed section'); + // stdout has dry-run action + expect(capture.stdout.join('\n')).toContain('dry-run'); + }); + + it('dry-run + AGENTS.md already present: still writes nothing', async () => { + const { store, fs: agentFs, seedFile, writeCalls } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + seedFile(agentsAbs, '# Existing notes\n'); + const contentBefore = store.get(agentsAbs); + + await runInstall( + { ...BASE_OPTS, dryRun: true, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // Content unchanged + expect(store.get(agentsAbs)).toBe(contentBefore); + expect(writeCalls.length).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// [P2] AGENTS.md 32 KiB budget warning +// Codex has a documented ~32 KiB load budget per AGENTS.md. When the would-be +// file content exceeds that threshold we emit a [warn] to stderr. We still +// write (warn, not refusal). Small files must not trigger the warning. +// --------------------------------------------------------------------------- + +describe('[codex-P2] AGENTS.md 32 KiB budget warning', () => { + it('no warning when existing file is small (well under 32 KiB)', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + seedFile(agentsAbs, '# Small project notes\n'); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // No budget warning should appear + const warnLines = capture.stderr.filter(l => l.includes('[warn]') && l.includes('KiB')); + expect(warnLines).toHaveLength(0); + }); + + it('emits [warn] on stderr when resulting file would exceed 32 KiB budget', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Seed a near-full existing file: 31.5 KiB of content (just under the budget on its own). + // After appending the ~3.5 KiB codex section the total will exceed 32 KiB. + const nearFullContent = '# Project notes\n' + 'x'.repeat(AGENTS_MD_CODEX_BUDGET_BYTES - 512); + seedFile(agentsAbs, nearFullContent); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + // At least one stderr line must be a budget warning + const warnLines = capture.stderr.filter(l => l.includes('[warn]') && l.includes('KiB')); + expect(warnLines.length).toBeGreaterThanOrEqual(1); + + // The warning must mention the resulting byte size and the 32 KiB budget + const warnText = warnLines[0]!; + expect(warnText).toMatch(/\d+ bytes/); + expect(warnText).toContain('32 KiB'); + + // The write must still happen (warning is advisory, not a refusal) + expect(capture.stderr.some(l => l.includes('section-'))).toBe(false); // action only in stdout + expect(capture.stdout.join('\n')).toMatch(/section-installed|section-updated/); + }); +}); + +// --------------------------------------------------------------------------- +// [codex-P2] Sentinel standalone-line matching — Finding 1 hardening +// +// Sentinels must only be recognised when they appear as STANDALONE lines. +// An inline mention inside prose (e.g. documentation quoting the marker) must +// NOT be treated as a managed block. Duplicate standalone pairs must be +// rejected as corrupt (exit 5). +// --------------------------------------------------------------------------- + +describe('[codex-P2] sentinel standalone-line matching', () => { + it('inline-only mention → treated as no sentinels → appends section', async () => { + // The file mentions the sentinel string INSIDE a prose paragraph, not as a + // standalone line. The classifier must see this as "no sentinels" and append. + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Inline mention: sentinel string is embedded in a longer sentence. + const inlineMentionContent = + '# My Project\n\n' + + `You can identify our managed block by looking for the marker ${MANAGED_SECTION_BEGIN} in the text.\n` + + `Likewise the closing marker is ${MANAGED_SECTION_END} but both are only in this paragraph.\n`; + seedFile(agentsAbs, inlineMentionContent); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Original prose preserved + expect(written).toContain('You can identify our managed block'); + // Section was appended (not replacing the prose) + const beginCount = written.split(MANAGED_SECTION_BEGIN).length - 1; + const endCount = written.split(MANAGED_SECTION_END).length - 1; + // The inline mention + the newly written standalone sentinel = 2 occurrences each + expect(beginCount).toBe(2); + expect(endCount).toBe(2); + // Action must be 'section-installed' (first-time append) + expect(capture.stdout.join('\n')).toContain('section-installed'); + }); + + it('inline mention + real standalone block → replaces only the standalone block', async () => { + // The file has one inline mention AND a real standalone sentinel pair. + // Only the standalone pair is the managed block; the inline mention is + // left untouched. + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + const oldSection = `${MANAGED_SECTION_BEGIN}\nOLD MANAGED CONTENT\n${MANAGED_SECTION_END}\n`; + const content = + '# My Project\n\n' + + // Inline mention (NOT a standalone line — embedded in a paragraph) + `See the marker ${MANAGED_SECTION_BEGIN} for details.\n\n` + + oldSection + + '\n## Other section\n'; + seedFile(agentsAbs, content); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Inline prose mention preserved + expect(written).toContain(`See the marker ${MANAGED_SECTION_BEGIN} for details.`); + // Old content replaced + expect(written).not.toContain('OLD MANAGED CONTENT'); + // Surrounding sections preserved + expect(written).toContain('## Other section'); + // Action must be 'section-updated' + expect(capture.stdout.join('\n')).toContain('section-updated'); + }); + + it('duplicate standalone BEGIN lines → corrupt → exit 5', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Two standalone BEGIN lines → ambiguous; must be rejected as corrupt. + const content = + `${MANAGED_SECTION_BEGIN}\nSection content\n${MANAGED_SECTION_END}\n\n` + + `${MANAGED_SECTION_BEGIN}\nDuplicate second block\n${MANAGED_SECTION_END}\n`; + seedFile(agentsAbs, content); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toMatch(/malformed|corrupt|sentinel|ambiguous/i); + }); + + it('duplicate standalone END lines → corrupt → exit 5', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // One BEGIN but two standalone END lines. + const content = + `${MANAGED_SECTION_BEGIN}\nContent\n${MANAGED_SECTION_END}\n` + + `Stray content\n${MANAGED_SECTION_END}\n`; + seedFile(agentsAbs, content); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + }); + + it('CRLF file with real standalone block → replaces correctly', async () => { + // Regression: sentinel lines in a CRLF file have a trailing \r that must + // be stripped before comparison so they still match. + const { store, fs: agentFs, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Build a CRLF file: each line ends with \r\n. + const oldSection = `${MANAGED_SECTION_BEGIN}\r\nOLD CRLF CONTENT\r\n${MANAGED_SECTION_END}\r\n`; + const crlfContent = `# My Project\r\n\r\n${oldSection}\r\n## Other\r\n`; + seedFile(agentsAbs, crlfContent); + + await runInstall( + { ...BASE_OPTS, target: ['codex'], force: false }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const written = store.get(agentsAbs)!; + // Old CRLF content must have been replaced + expect(written).not.toContain('OLD CRLF CONTENT'); + // After-section preserved + expect(written).toContain('## Other'); + // Action reported + expect(capture.stdout.join('\n')).toContain('section-updated'); + }); +}); + +// --------------------------------------------------------------------------- +// [B-E2E-04] Fix 4 regression — codex --dry-run warns when existing file +// + section would exceed the 32 KiB Codex budget +// --------------------------------------------------------------------------- + +describe('[B-E2E-04] codex --dry-run: over-budget warning (Fix 4 regression)', () => { + const BASE_OPTS_DRY = { + profile: 'default' as const, + output: 'text' as const, + debug: false, + dryRun: true, + force: false, + }; + + it('emits [warn] on stderr when existing AGENTS.md + section > 32 KiB; writes nothing', async () => { + const { store, fs: agentFs, seedFile, writeCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + + // Seed an AGENTS.md that is large enough to push the total over budget. + // AGENTS_MD_CODEX_BUDGET_BYTES = 32768. + // We use 31 KiB of existing content so that existing + section > 32 KiB. + const bigExisting = '#'.repeat(31 * 1024); + seedFile(agentsAbs, bigExisting); + + await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); + + // Must not write anything in dry-run + expect(writeCalls.length).toBe(0); + expect(store.get(agentsAbs)).toBe(bigExisting); // unchanged + + // Must emit a [warn] about the budget + const stderrOut = capture.stderr.join('\n'); + expect(stderrOut).toMatch(/\[warn\].*bytes.*Codex/i); + }); + + it('does NOT warn when AGENTS.md is absent (fresh install would be under budget)', async () => { + const { fs: agentFs, writeCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); + + // Dry-run = no writes + expect(writeCalls.length).toBe(0); + + // No over-budget warning when file is absent (section alone is ≤ 32 KiB) + const stderrOut = capture.stderr.join('\n'); + expect(stderrOut).not.toMatch(/\[warn\].*Codex/i); + }); + + it('reports AGENTS_MD_CODEX_BUDGET_BYTES constant publicly (sanity)', () => { + expect(AGENTS_MD_CODEX_BUDGET_BYTES).toBe(32768); + }); +}); + +// --------------------------------------------------------------------------- +// [P2 regression] codex --dry-run: symlinked AGENTS.md must fail-close (exit 5) +// before any readFile call — dry-run must apply the same symlink guard as the +// real install path (inspectTargetPath runs first in both modes). +// --------------------------------------------------------------------------- + +describe('[P2] codex --dry-run: symlink fail-close (same guard as real install)', () => { + const BASE_OPTS_DRY = { + profile: 'default' as const, + output: 'text' as const, + debug: false, + dryRun: true, + force: false, + }; + + it('symlinked AGENTS.md + --dry-run → exit 5, readFile never called', async () => { + // The final AGENTS.md path is a symlink. In dry-run the old code called + // readFile BEFORE inspectTargetPath, so the symlink was followed silently. + // The fix runs inspectTargetPath first in both modes; this test is the + // regression gate. + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + const { deps } = makeCapture(); + + // Seed AGENTS.md as a symlink at the codex landing path. + seedSymlink(path.resolve(CWD, TARGETS.codex.path)); + + // Also confirm readFile is never called by using a custom fs + let readFileCalled = false; + const spyFs: AgentFs = { + ...agentFs, + async readFile(p: string) { + readFileCalled = true; + return agentFs.readFile(p); + }, + }; + + let thrown: unknown; + try { + await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: spyFs, ...deps }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + // readFile must never have been called on the symlinked path + expect(readFileCalled).toBe(false); + }); + + it('regular AGENTS.md + --dry-run: lstat check passes, readFile called normally', async () => { + // Confirm normal operation is not disrupted: a regular AGENTS.md file + // is lstat-checked (not a symlink), then readFile runs for the budget check. + const { fs: agentFs, writeCalls, seedFile } = makeMemFs(); + const { capture, deps } = makeCapture(); + + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + // Seed a regular file (not a symlink) + seedFile(agentsAbs, '# My Project\nSome content.\n'); + + await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); + + // No writes (dry-run) + expect(writeCalls.length).toBe(0); + // Stdout reports dry-run action + expect(capture.stdout.join('\n')).toContain('dry-run'); + }); +}); + +// --------------------------------------------------------------------------- +// [P3 round-2] codex --dry-run budget: measure the COMPOSED result, not +// existing+section (replace must not double-count the old block), and surface +// non-ENOENT read failures instead of treating them as absence. +// --------------------------------------------------------------------------- + +describe('[P3 round-2] codex --dry-run: composed-size precision + read-failure surfacing', () => { + const BASE_OPTS_DRY = { + profile: 'default' as const, + output: 'text' as const, + debug: false, + dryRun: true, + force: false, + }; + + it('replace path: no warn when the composed file is under budget even though existing+section is over', async () => { + const { store, fs: agentFs, seedFile, writeCalls } = makeMemFs(); + const { capture, deps } = makeCapture(); + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + + // Old managed section with a 6 KiB body + 26 KiB of user prose. + // existing (~32.9 KiB) + new section (~4.8 KiB) > 32 KiB → the OLD formula + // would warn; the composed replace result (26 KiB + new section) is + // comfortably under budget → no warn expected. + const oldSection = `${MANAGED_SECTION_BEGIN}\n${'o'.repeat(6 * 1024)}\n${MANAGED_SECTION_END}\n`; + const userProse = `# My own AGENTS.md\n${'u'.repeat(26 * 1024)}\n`; + seedFile(agentsAbs, `${userProse}\n${oldSection}`); + + await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); + + expect(writeCalls.length).toBe(0); + expect(store.get(agentsAbs)).toBe(`${userProse}\n${oldSection}`); + expect(capture.stderr.join('\n')).not.toMatch(/\[warn\].*Codex/i); + }); + + it('corrupt sentinel + --dry-run → exit 5 (same outcome the real install would report), no writes', async () => { + const { fs: agentFs, seedFile, writeCalls } = makeMemFs(); + const { deps } = makeCapture(); + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + seedFile(agentsAbs, `user content\n${MANAGED_SECTION_BEGIN}\nno end sentinel here\n`); + + await expect( + runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + expect(writeCalls.length).toBe(0); + }); + + it('non-ENOENT read failure (EACCES) on --dry-run → exit 5, not silent success', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const { deps } = makeCapture(); + const agentsAbs = path.resolve(CWD, TARGETS.codex.path); + seedFile(agentsAbs, 'unreadable'); + + const denied = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); + const realRead = agentFs.readFile.bind(agentFs); + agentFs.readFile = async (p: string) => + p === agentsAbs ? Promise.reject(denied) : realRead(p); + + await expect( + runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + }); +}); diff --git a/src/commands/agent.ts b/src/commands/agent.ts new file mode 100644 index 0000000..c27e2b8 --- /dev/null +++ b/src/commands/agent.ts @@ -0,0 +1,770 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { Command } from 'commander'; +import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { CLIError, localValidationError } from '../lib/errors.js'; +import type { OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { promptText } from '../lib/prompt.js'; +import { + type AgentTarget, + TARGETS, + loadSkillBody, + loadCodexSkillBody, + renderForTarget, + MANAGED_SECTION_BEGIN, + MANAGED_SECTION_END, +} from '../lib/agent-targets.js'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** + * Codex loads AGENTS.md files lazily and has a documented 32 KiB load budget + * per file. Content beyond that offset is silently truncated. We warn (but do + * not refuse to write) when a managed-section write would produce a file larger + * than this threshold so operators have early visibility. + */ +export const AGENTS_MD_CODEX_BUDGET_BYTES = 32768; // 32 KiB + +// --------------------------------------------------------------------------- +// Filesystem port (injectable for tests) +// --------------------------------------------------------------------------- + +export interface AgentFs { + // lstat semantics: does NOT follow symlinks (null = ENOENT). Critical for the + // path-safety walk — fs writes follow symlinks, so we must be able to see them. + lstat(p: string): Promise<{ isFile: boolean; isSymbolicLink: boolean } | null>; + readFile(p: string): Promise; + // exclusive: fail with EEXIST if the path already exists. O_EXCL|O_CREAT does + // not follow a final symlink, so exclusive writes never clobber or traverse a + // planted symlink — used for backups and fresh installs. + writeFile(p: string, data: string, opts?: { exclusive?: boolean }): Promise; + mkdir(p: string): Promise; // recursive +} + +const defaultAgentFs: AgentFs = { + async lstat(p: string): Promise<{ isFile: boolean; isSymbolicLink: boolean } | null> { + try { + const s = await fs.lstat(p); + return { isFile: s.isFile(), isSymbolicLink: s.isSymbolicLink() }; + } catch (err: unknown) { + if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'ENOENT') { + return null; + } + throw err; + } + }, + async readFile(p: string): Promise { + return fs.readFile(p, 'utf8'); + }, + async writeFile(p: string, data: string, opts?: { exclusive?: boolean }): Promise { + await fs.writeFile(p, data, { encoding: 'utf8', flag: opts?.exclusive ? 'wx' : 'w' }); + }, + async mkdir(p: string): Promise { + await fs.mkdir(p, { recursive: true }); + }, +}; + +// --------------------------------------------------------------------------- +// Path safety +// --------------------------------------------------------------------------- + +/** + * Walk each component of `relPath` beneath `root`, refusing to traverse or + * write through a symlink. `fs.mkdir`/`writeFile` follow symlinks, so a planted + * symlink at any existing path component (e.g. `.claude` -> /etc, or the final + * `SKILL.md` -> ~/.bashrc) could place or clobber files outside `--dir`. The + * lexical containment guard in `runInstall` is a string compare and cannot see + * this; only an `lstat`-per-component walk can. Fail-closed: any symlink is + * rejected (exit 5). + * + * Returns the target's `{ isFile }` when it already exists, or `null` when it + * (or any ancestor) does not yet exist — in which case the missing tail is + * created fresh and cannot be a pre-planted symlink. A small TOCTOU window + * remains between this check and the write; that is acceptable for a local, + * single-user CLI and avoids non-portable O_NOFOLLOW / rename gymnastics. + */ +async function inspectTargetPath( + agentFs: AgentFs, + root: string, + relPath: string, +): Promise<{ isFile: boolean } | null> { + const segments = relPath.split(/[/\\]+/).filter(Boolean); + let current = root; + let finalIsFile = false; + for (const [i, seg] of segments.entries()) { + current = path.join(current, seg); + const ls = await agentFs.lstat(current); + if (ls === null) { + // This component and everything below it does not exist yet. + return null; + } + if (ls.isSymbolicLink) { + const shown = segments.slice(0, i + 1).join('/'); + throw new CLIError( + `refusing to write through a symlink: "${shown}" — installing here could place files outside --dir. Remove the symlink or choose a different --dir.`, + 5, + ); + } + if (i < segments.length - 1 && ls.isFile) { + const shown = segments.slice(0, i + 1).join('/'); + throw new CLIError(`cannot create ${relPath}: "${shown}" exists and is not a directory.`, 5); + } + finalIsFile = ls.isFile; + } + return { isFile: finalIsFile }; +} + +/** + * Back up the current bytes at `abs` next to it without clobbering any existing + * backup or writing through a symlink. Exclusive create (`wx`) fails with + * EEXIST on an existing regular file OR symlink, so we walk `.bak`, `.bak.1`, + * `.bak.2`, … until a free slot is found. Returns the absolute path used. + */ +async function writeBackup(agentFs: AgentFs, abs: string, existing: string): Promise { + for (let n = 0; n < 100; n++) { + const candidate = n === 0 ? `${abs}.bak` : `${abs}.bak.${n}`; + try { + await agentFs.writeFile(candidate, existing, { exclusive: true }); + return candidate; + } catch (err) { + if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'EEXIST') { + continue; + } + throw err; + } + } + throw new CLIError( + `refusing to back up ${path.basename(abs)}: too many existing .bak files — clean them up and re-run.`, + 6, + ); +} + +// --------------------------------------------------------------------------- +// Managed-section helpers (codex target) +// --------------------------------------------------------------------------- + +/** + * Build the section block to inject (sentinels + body + trailing newline). + * Uses \n throughout; the caller handles CRLF normalisation. + */ +function buildSection(body: string): string { + return `${MANAGED_SECTION_BEGIN}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; +} + +/** + * Managed-section install result — what happened to AGENTS.md. + * + * 'create' — file did not exist; write the section as a new file. + * 'append' — file exists, no sentinels; append section at end. + * 'replace' — file exists with sentinels; replace section content in-place. + * 'unchanged' — file exists with sentinels and content is byte-identical. + * 'corrupt' — BEGIN sentinel without matching END; refuse to touch the file. + */ +type SectionState = + | { kind: 'create' } + | { kind: 'append'; existing: string } + | { kind: 'replace'; existing: string; before: string; after: string } + | { kind: 'unchanged' } + | { kind: 'corrupt' }; + +/** + * Inspect an existing AGENTS.md and classify the managed-section state. + * + * Sentinel-matching rules (P2 hardening): + * - Only STANDALONE sentinel lines count (a line that consists solely of the + * marker, optionally followed by whitespace/CR before the LF). This prevents + * inline mentions in prose (e.g. documentation quoting the markers) from + * being mis-classified as a managed block. + * - Multiple standalone BEGIN or END lines → ambiguous → corrupt (exit 5). + * - CRLF files are handled by stripping trailing \r from each line before + * comparison. + */ +function classifySection(existing: string, section: string): SectionState { + // Split on LF; strip trailing CR so CRLF files normalise correctly. + const lines = existing.split('\n'); + + // Collect line INDICES (0-based) where the sentinel appears as the whole line + // (trimEnd removes trailing CR and spaces). + const beginLines: number[] = []; + const endLines: number[] = []; + + for (let i = 0; i < lines.length; i++) { + const stripped = (lines[i] ?? '').trimEnd(); + if (stripped === MANAGED_SECTION_BEGIN) beginLines.push(i); + else if (stripped === MANAGED_SECTION_END) endLines.push(i); + } + + const hasBegin = beginLines.length > 0; + const hasEnd = endLines.length > 0; + + if (!hasBegin && !hasEnd) { + // No standalone sentinels — append path. + return { kind: 'append', existing }; + } + + // Duplicate standalone sentinels are ambiguous — treat as corrupt. + if (beginLines.length > 1) { + return { kind: 'corrupt' }; + } + if (endLines.length > 1) { + return { kind: 'corrupt' }; + } + + if (hasBegin && !hasEnd) { + // BEGIN present but no standalone END — corrupt. + return { kind: 'corrupt' }; + } + + if (!hasBegin && hasEnd) { + // END present but no standalone BEGIN — corrupt. + return { kind: 'corrupt' }; + } + + const beginLineIdx = beginLines[0]!; + const endLineIdx = endLines[0]!; + + if (endLineIdx < beginLineIdx) { + // END appears before BEGIN — corrupt. + return { kind: 'corrupt' }; + } + + // Both sentinels present, in the right order, with no duplicates. + // Reconstruct byte offsets from line positions so we can slice the original + // string (preserving its exact byte content for the before/after split). + // + // lineStart[i] = byte offset of the first character of line i. + let byteOffset = 0; + const lineStart: number[] = []; + for (const line of lines) { + lineStart.push(byteOffset); + byteOffset += line.length + 1; // +1 for the '\n' that split() removed + } + + const beginByteIdx = lineStart[beginLineIdx]!; + + // The END sentinel line ends at: lineStart[endLineIdx] + raw line length. + // We want to include the trailing '\n' after END when present. + const endLineRawLength = (lines[endLineIdx] ?? '').length; + const endOfEndByte = lineStart[endLineIdx]! + endLineRawLength; + // Include one trailing newline after END if present. + const charAfterEnd = existing[endOfEndByte]; + const trailingNewline = charAfterEnd === '\n' ? 1 : charAfterEnd === '\r' ? 2 : 0; + + const before = existing.slice(0, beginByteIdx); + const after = existing.slice(endOfEndByte + trailingNewline); + const currentSection = existing.slice(beginByteIdx, endOfEndByte + trailingNewline); + + if (currentSection === section) { + return { kind: 'unchanged' }; + } + + return { kind: 'replace', existing, before, after }; +} + +/** + * Compose the new AGENTS.md content for the 'append' and 'replace' paths. + * + * 'append': ensure a single blank line separator between existing content + * and the section (but don't add two blank lines if the file already ends + * with one). + * 'replace': splice the new section between `before` and `after`. + */ +function composeManagedFile( + state: SectionState & { kind: 'append' | 'replace' }, + section: string, +): string { + if (state.kind === 'append') { + const existing = state.existing; + const sep = existing.length === 0 || existing.endsWith('\n\n') ? '' : '\n'; + return `${existing}${sep}${section}`; + } + // replace + return `${state.before}${section}${state.after}`; +} + +// --------------------------------------------------------------------------- +// Deps +// --------------------------------------------------------------------------- + +export interface AgentDeps { + cwd?: string; + fs?: AgentFs; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + isTTY?: boolean; + prompt?: (question: string) => Promise; +} + +// --------------------------------------------------------------------------- +// Result types +// --------------------------------------------------------------------------- + +export type InstallAction = + | 'written' + | 'skipped' + | 'blocked' + | 'updated' + | 'dry-run' + | 'section-installed' + | 'section-updated' + | 'section-unchanged'; + +export interface InstallResult { + target: AgentTarget; + path: string; // repo-relative matrix path + action: InstallAction; +} + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +type CommonOptions = FactoryCommonOptions; + +interface InstallOptions extends CommonOptions { + target: string[]; + dir?: string; + force: boolean; +} + +// --------------------------------------------------------------------------- +// runInstall +// --------------------------------------------------------------------------- + +export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Promise { + const agentFs = deps.fs ?? defaultAgentFs; + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + // 1. Parse targets + const rawTargets = opts.target + .flatMap(s => s.split(',')) + .map(s => s.trim()) + .filter(Boolean); + + let resolvedTargetStrings: string[]; + + if (rawTargets.length === 0) { + const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); + if (!isTTY) { + throw localValidationError( + 'target', + `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, + ); + } + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); + } else { + resolvedTargetStrings = rawTargets; + } + + // 2. Validate targets + const validTargets = Object.keys(TARGETS) as AgentTarget[]; + for (const t of resolvedTargetStrings) { + if (!validTargets.includes(t as AgentTarget)) { + throw localValidationError( + 'target', + `unknown target "${t}"; supported: ${validTargets.join(', ')}`, + ); + } + } + + // De-duplicate while preserving first-seen order + const seen = new Set(); + const targets = resolvedTargetStrings.filter(t => { + if (seen.has(t)) return false; + seen.add(t); + return true; + }) as AgentTarget[]; + + // 3. Resolve dir + const dir = opts.dir ?? deps.cwd ?? process.cwd(); + const root = path.resolve(dir); + + // 4. Load skill bodies (lazy — only touch disk if a target actually needs it) + let ownFileBody: string | undefined; + let codexBody: string | undefined; + + const results: InstallResult[] = []; + + // Track bytes for dry-run output + const dryRunLines: { abs: string; bytes: number; note: string }[] = []; + + // 5. Process each target + for (const t of targets) { + const spec = TARGETS[t]; + const relPath = spec.path; + const abs = path.resolve(root, relPath); + + // Path safety: ensure abs is inside root (defense against .. in relPath or dir) + if (abs !== root && !abs.startsWith(root + path.sep)) { + throw new CLIError(`refusing to write outside --dir: ${relPath}`, 5); + } + + // ----------------------------------------------------------------------- + // managed-section mode (codex target) + // ----------------------------------------------------------------------- + if (spec.mode === 'managed-section') { + if (codexBody === undefined) codexBody = loadCodexSkillBody(); + const section = buildSection(codexBody); + + if (opts.dryRun) { + // Dry-run: report what would happen without writing disk. + // + // [P2] Apply the SAME symlink fail-close guard as the real install path. + // Without this, a symlinked AGENTS.md gets followed in dry-run even + // though the real install would refuse (exit 5). Run inspectTargetPath + // first; only lstat-check the final file (not write) after that. + const dryRunSt = await inspectTargetPath(agentFs, root, relPath); + if (dryRunSt !== null && !dryRunSt.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } + + // We DO read the existing file (if present) to compute the + // would-be byte count and emit the 32 KiB budget warning — without + // this the warning was silently absent on --dry-run runs (Fix 4). + // + // [P3 round-2] Measure the ACTUAL composed result via the same + // classifySection + composeManagedFile pipeline the real install + // uses — `existing + section` double-counts the old block on the + // replace path and misses the append separator. Read failures other + // than ENOENT are surfaced (EACCES/EIO must not read as "absent" — + // absence is already represented by dryRunSt === null). + const bytes = Buffer.byteLength(section, 'utf8'); + let wouldBeContent = section; + if (dryRunSt !== null) { + let existing: string | null = null; + try { + existing = await agentFs.readFile(abs); + } catch (err) { + if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'ENOENT') { + existing = null; // raced away between lstat and read → would-be = create + } else { + throw new CLIError( + `cannot read ${relPath} for dry-run: ${err instanceof Error ? err.message : String(err)}`, + 5, + ); + } + } + if (existing !== null) { + const state = classifySection(existing, section); + if (state.kind === 'corrupt') { + // The real install would refuse with exit 5 — dry-run reports + // the same outcome rather than a misleading success. + throw new CLIError( + `${relPath} contains a malformed TestSprite sentinel (BEGIN without END or vice-versa). ` + + `Manually remove the partial sentinel block and re-run.`, + 5, + ); + } + wouldBeContent = + state.kind === 'unchanged' + ? existing + : state.kind === 'create' + ? section + : composeManagedFile(state, section); + } + } + const wouldBeBytes = Buffer.byteLength(wouldBeContent, 'utf8'); + if (wouldBeBytes > AGENTS_MD_CODEX_BUDGET_BYTES) { + stderrFn( + `[warn] ${relPath} will be ${wouldBeBytes} bytes after this write — Codex may not load content beyond its 32 KiB (${AGENTS_MD_CODEX_BUDGET_BYTES} byte) budget. Trim AGENTS.md to stay within the limit.`, + ); + } + dryRunLines.push({ abs, bytes, note: 'managed section' }); + results.push({ target: t, path: relPath, action: 'dry-run' }); + continue; + } + + // Inspect the target path via lstat walk (symlink-safe, same as own-file). + const st = await inspectTargetPath(agentFs, root, relPath); + + if (st !== null && !st.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } + + /** + * [P2] Emit a stderr warn when the would-be file content exceeds Codex's + * 32 KiB load budget. We still write — this is a warn, not a refusal — + * but the operator needs early visibility so they can trim AGENTS.md. + */ + function warnIfOverBudget(wouldBeContent: string): void { + const byteLen = Buffer.byteLength(wouldBeContent, 'utf8'); + if (byteLen > AGENTS_MD_CODEX_BUDGET_BYTES) { + stderrFn( + `[warn] ${relPath} will be ${byteLen} bytes after this write — Codex may not load content beyond its 32 KiB (${AGENTS_MD_CODEX_BUDGET_BYTES} byte) budget. Trim AGENTS.md to stay within the limit.`, + ); + } + } + + if (st === null) { + // File absent → create AGENTS.md containing just the section. + warnIfOverBudget(section); + await agentFs.mkdir(path.dirname(abs)); + try { + await agentFs.writeFile(abs, section, { exclusive: true }); + } catch (err) { + if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'EEXIST') { + throw new CLIError( + `${relPath} appeared after the path check — re-run, or pass --force to overwrite.`, + 6, + ); + } + throw err; + } + results.push({ target: t, path: relPath, action: 'section-installed' }); + } else { + const existing = await agentFs.readFile(abs); + const state = classifySection(existing, section); + + if (state.kind === 'corrupt') { + // BEGIN without matching END (or vice-versa) — never destroy user content. + throw new CLIError( + `${relPath} contains a malformed TestSprite sentinel (BEGIN without END or vice-versa). ` + + `Manually remove the partial sentinel block and re-run.`, + 5, + ); + } + + if (state.kind === 'unchanged') { + results.push({ target: t, path: relPath, action: 'section-unchanged' }); + } else if (state.kind === 'create') { + // Shouldn't happen (st !== null means file exists), but guard anyway. + warnIfOverBudget(section); + await agentFs.writeFile(abs, section); + results.push({ target: t, path: relPath, action: 'section-installed' }); + } else { + // 'append' or 'replace' — write the new content. + // --force has no special meaning for managed-section: we always merge + // rather than replacing the whole file, so force is effectively always + // on for the section (user content is never at risk). + const newContent = composeManagedFile(state, section); + warnIfOverBudget(newContent); + await agentFs.writeFile(abs, newContent); + const action: InstallAction = + state.kind === 'append' ? 'section-installed' : 'section-updated'; + results.push({ target: t, path: relPath, action }); + } + } + continue; + } + + // ----------------------------------------------------------------------- + // own-file mode (all other targets) + // ----------------------------------------------------------------------- + if (ownFileBody === undefined) ownFileBody = loadSkillBody(); + const content = renderForTarget(t, ownFileBody).content; + + if (opts.dryRun) { + const bytes = Buffer.byteLength(content, 'utf8'); + dryRunLines.push({ abs, bytes, note: '' }); + results.push({ target: t, path: relPath, action: 'dry-run' }); + continue; + } + + // Inspect the target path: refuse to traverse or write through a symlink + // (fs writes follow symlinks, which would let a planted symlink escape + // --dir), and reject a non-regular-file landing path. The lexical guard + // above is necessary but not sufficient — it cannot see symlinks. + const st = await inspectTargetPath(agentFs, root, relPath); + + if (st !== null && !st.isFile) { + throw new CLIError(`${relPath} exists but is not a regular file — remove it and re-run.`, 5); + } + + if (st === null) { + // Path does not exist — create it. inspectTargetPath verified every + // existing ancestor is a real directory; exclusive create (wx) then + // ensures a file or symlink that races in after the check is not followed + // or silently overwritten. + await agentFs.mkdir(path.dirname(abs)); + try { + await agentFs.writeFile(abs, content, { exclusive: true }); + } catch (err) { + if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'EEXIST') { + throw new CLIError( + `${relPath} appeared after the path check — re-run, or pass --force to overwrite.`, + 6, + ); + } + throw err; + } + results.push({ target: t, path: relPath, action: 'written' }); + } else { + const existing = await agentFs.readFile(abs); + if (existing === content) { + // Byte-identical — skip + results.push({ target: t, path: relPath, action: 'skipped' }); + } else if (!opts.force) { + // Differs and no --force → blocked + results.push({ target: t, path: relPath, action: 'blocked' }); + } else { + // Differs and --force → back up the current bytes to a fresh slot + // (never clobbering an existing backup or following a symlink), then + // overwrite. The overwrite itself can follow a symlink swapped in after + // the check — an accepted TOCTOU residual for a local, single-user CLI. + const backupPath = await writeBackup(agentFs, abs, existing); + await agentFs.writeFile(abs, content); + if (opts.output === 'text') { + stderrFn(`backed up ${relPath} to ${path.relative(root, backupPath)}`); + } + results.push({ target: t, path: relPath, action: 'updated' }); + } + } + } + + // 6. Dry-run output + if (opts.dryRun) { + stderrFn('[dry-run] no files written — preview only'); + for (const { abs, bytes, note } of dryRunLines) { + const suffix = note ? ` (${note}, ${bytes} bytes)` : ` (${bytes} bytes)`; + stderrFn(`[dry-run] would write ${abs}${suffix}`); + } + } + + // 7. Blocked hints + for (const r of results) { + if (r.action === 'blocked') { + stderrFn( + `${r.path} exists and differs from the canonical skill — re-run with --force to overwrite (the existing file is backed up to .bak).`, + ); + } + } + + // 8. Print results + out.print(results, data => { + const items = data as InstallResult[]; + return items.map(r => `${r.target.padEnd(12)} ${r.action.padEnd(12)} ${r.path}`).join('\n'); + }); + + // 9. Exit with 6 if any blocked + if (results.some(r => r.action === 'blocked')) { + throw new CLIError( + 'one or more targets already exist and differ; re-run with --force to overwrite (a .bak is kept).', + 6, + ); + } +} + +// --------------------------------------------------------------------------- +// runList +// --------------------------------------------------------------------------- + +export interface ListResult { + target: AgentTarget; + status: string; + mode: string; + path: string; +} + +export async function runList(opts: CommonOptions, deps: AgentDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + + const results: ListResult[] = ( + Object.entries(TARGETS) as [AgentTarget, { status: string; mode: string; path: string }][] + ).map(([t, spec]) => ({ + target: t, + status: spec.status, + mode: spec.mode, + path: spec.path, + })); + + out.print(results, data => { + const items = data as ListResult[]; + const header = `${'TARGET'.padEnd(14)} ${'STATUS'.padEnd(12)} ${'MODE'.padEnd(18)} PATH`; + const rows = items.map( + r => `${r.target.padEnd(14)} ${r.status.padEnd(12)} ${r.mode.padEnd(18)} ${r.path}`, + ); + return [header, ...rows].join('\n'); + }); +} + +// --------------------------------------------------------------------------- +// Command factory +// --------------------------------------------------------------------------- + +function collect(v: string, prev: string[]): string[] { + return prev.concat(v); +} + +export function createAgentCommand(deps: AgentDeps = {}): Command { + const agent = new Command('agent').description( + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Codex)', + ); + + agent + .command('install') + .description( + 'Write the TestSprite verification-loop skill file into a project for a coding agent', + ) + .option( + '--target ', + 'Agent target(s): claude, cursor, cline, antigravity, codex (comma-separated or repeated)', + collect, + [], + ) + .option('--dir ', 'Project root to write into (default: cwd)') + .option( + '--force', + 'For own-file targets: overwrite existing file (a .bak backup is kept). ' + + 'For codex (managed-section): replaces the section unconditionally; user content outside the section is never destroyed.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async (cmdOpts: { target: string[]; dir?: string; force?: boolean }, command: Command) => { + await runInstall( + { + ...resolveCommonOptions(command), + target: cmdOpts.target, + dir: cmdOpts.dir, + force: Boolean(cmdOpts.force), + }, + deps, + ); + }, + ); + + agent + .command('list') + .description('List supported agent targets, their status, and landing paths') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (_o, command: Command) => { + await runList(resolveCommonOptions(command), deps); + }); + + return agent; +} + +// --------------------------------------------------------------------------- +// Per-file helpers (per convention: copy from auth.ts) +// --------------------------------------------------------------------------- + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + }; +} + +function makeOutput(mode: OutputMode, deps: AgentDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts new file mode 100644 index 0000000..ce8365a --- /dev/null +++ b/src/commands/auth.test.ts @@ -0,0 +1,933 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError } from '../lib/errors.js'; +import { readProfile, writeProfile } from '../lib/credentials.js'; +import type { AuthDeps, MeResponse } from './auth.js'; +import { createAuthCommand, runConfigure, runLogout, runWhoami } from './auth.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; + prelude: string[]; +} + +function makeCapture(): { + capture: CapturedOutput; + deps: Pick; +} { + const capture: CapturedOutput = { stdout: [], stderr: [], prelude: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + preludeWrite: chunk => capture.prelude.push(chunk), + }, + }; +} + +const sampleMe: MeResponse = { + userId: 'u-1', + keyId: 'k-1', + scopes: ['read:projects', 'read:tests'], + env: 'development', +}; + +/** Mock fetchImpl that returns 200 for /me — used by runConfigure tests to satisfy the pre-write ping. */ +const meOkFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), +) as unknown as AuthDeps['fetchImpl']; + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-auth-')), 'credentials'); +}); + +describe('runConfigure', () => { + it('writes the env-supplied key when --from-env is set', async () => { + const { capture, deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-from-env', TESTSPRITE_API_URL: 'https://from-env' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })).toEqual({ + apiKey: 'sk-from-env', + apiUrl: 'https://from-env', + }); + expect(capture.stdout.join('\n')).toContain('configured'); + }); + + it('--from-env without TESTSPRITE_API_URL uses the built-in default endpoint', async () => { + const { deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://api.testsprite.com', + ); + }); + + it('--endpoint-url overrides TESTSPRITE_API_URL when --from-env is set', async () => { + const { deps } = makeCapture(); + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + endpointUrl: 'https://flag-wins.example.com', + }, + { + ...deps, + env: { + TESTSPRITE_API_KEY: 'sk', + TESTSPRITE_API_URL: 'https://env-loses.example.com', + }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://flag-wins.example.com', + ); + }); + + it('throws VALIDATION_ERROR when --from-env is set but key is missing', async () => { + const { deps } = makeCapture(); + await expect( + runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { ...deps, env: {}, credentialsPath }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('prompts only for the API key (never the endpoint) and defaults to prod', async () => { + const { capture, deps } = makeCapture(); + // Prompt object exposes ONLY `secret`. If runConfigure tried to prompt for + // the endpoint it would call an undefined `text` and throw — so a passing + // test proves the endpoint is never prompted. + const prompt = { + secret: vi.fn(async () => 'sk-typed'), + }; + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { ...deps, env: {}, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })).toEqual({ + apiKey: 'sk-typed', + apiUrl: 'https://api.testsprite.com', + }); + expect(capture.prelude.join('')).toContain('Configuring profile "default"'); + }); + + it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { + const { capture, deps } = makeCapture(); + const prompt = { secret: vi.fn(async () => 'sk-typed') }; + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { + ...deps, + env: { TESTSPRITE_API_URL: 'https://api.example.com:8443' }, + credentialsPath, + prompt, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })).toEqual({ + apiKey: 'sk-typed', + apiUrl: 'https://api.example.com:8443', + }); + // An env-supplied endpoint is explicit → no inherit advisory. + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('interactive path inherits an existing non-default profile endpoint without prompting (+ advisory)', async () => { + const { capture, deps } = makeCapture(); + // Pre-existing dev profile — re-running configure interactively must keep it + // (the internal dogfooding flow) without ever prompting for the endpoint. + writeProfile( + 'dev', + { apiKey: 'sk-old', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const prompt = { secret: vi.fn(async () => 'sk-typed') }; + await runConfigure( + { profile: 'dev', output: 'text', debug: false, fromEnv: false }, + { ...deps, env: {}, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + expect(readProfile('dev', { path: credentialsPath })).toEqual({ + apiKey: 'sk-typed', + apiUrl: 'https://api.example.com:8443', + }); + expect(capture.stderr.join('\n')).toContain( + '[advisory] Inheriting api_url from existing profile: https://api.example.com:8443', + ); + }); + + it('throws when interactive secret comes back empty', async () => { + const { deps } = makeCapture(); + const prompt = { secret: vi.fn(async () => ' ') }; + await expect( + runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { ...deps, env: {}, credentialsPath, prompt }, + ), + ).rejects.toBeInstanceOf(CLIError); + }); + + it('honors --endpoint-url without prompting for the endpoint', async () => { + const { capture, deps } = makeCapture(); + const prompt = { secret: vi.fn(async () => 'sk-1') }; + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: false, + endpointUrl: 'https://override.example', + }, + { ...deps, env: {}, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://override.example', + ); + // Explicit endpoint → no inherit advisory. + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + // P4: pre-write /me ping behaviour + it('P4 — writes profile and prints success when API key is accepted', async () => { + const { capture, deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-good' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-good'); + expect(capture.stdout.join('\n')).toContain('configured'); + }); + + it('P4 — does NOT write profile and throws CLIError when API key is rejected (401)', async () => { + const { capture, deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: {}, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + await expect( + runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + // Profile must NOT be written. + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + // Stderr must mention the rejection. + expect(capture.stderr.join('\n')).toContain('profile NOT updated'); + }); + + // piece-3 bootstrap tip tests + it('piece-3 — text mode success emits the agent-install tip on stderr', async () => { + const { capture, deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-good' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(capture.stderr.join('\n')).toContain('agent install --target=claude'); + }); + + it('piece-3 — json mode success does NOT emit the agent-install tip', async () => { + const { capture, deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-good' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(capture.stderr.join('\n')).not.toContain('agent install'); + }); + + it('piece-3 — dry-run does NOT emit the agent-install tip', async () => { + const { capture, deps } = makeCapture(); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true, dryRun: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-good' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(capture.stderr.join('\n')).not.toContain('agent install'); + }); + + it('piece-3 — key-rejected path does NOT emit the agent-install tip', async () => { + const { capture, deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: {}, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + await expect( + runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + expect(capture.stderr.join('\n')).not.toContain('agent install'); + }); + + // Regression (2026-05-25): auth configure inherits existing profile apiUrl + it('dogfood-2026-05-25 — --from-env without TESTSPRITE_API_URL inherits existing profile api_url AND validates against it', async () => { + const { capture, deps } = makeCapture(); + // Pre-write an existing profile with a custom (non-default) endpoint. + writeProfile( + 'default', + { apiKey: 'sk-old', apiUrl: 'https://api.example.com' }, + { path: credentialsPath }, + ); + // codex-review P2 (2026-05-28): capture the URL the /me ping was made against + // so the regression actually exercises "validation hit the inherited URL, not + // DEFAULT_API_URL". meOkFetch was URL-agnostic, so the original test would + // have passed even if runConfigure called against prod and then wrote the + // inherited dev URL — exactly the bug we're trying to prevent. + const seenFetchUrls: string[] = []; + const urlAwareFetch: AuthDeps['fetchImpl'] = vi.fn(async (input: unknown) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.toString() : String(input); + seenFetchUrls.push(url); + return new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as unknown as AuthDeps['fetchImpl']; + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-new' }, + credentialsPath, + fetchImpl: urlAwareFetch, + }, + ); + // The new profile should reuse the inherited dev endpoint. + expect(readProfile('default', { path: credentialsPath })).toEqual({ + apiKey: 'sk-new', + apiUrl: 'https://api.example.com', + }); + // The /me ping MUST have been issued against the inherited dev URL — this is + // the actual regression guard (the original bug was validating against prod). + expect(seenFetchUrls.length).toBeGreaterThan(0); + expect(seenFetchUrls[0]).toMatch(/^https:\/\/api\.example\.com\//); + // And no fetch should have hit the default endpoint. + expect(seenFetchUrls.some(u => /^https:\/\/api\.testsprite\.com\//.test(u))).toBe(false); + // An advisory message must appear on stderr so the user is not confused. + expect(capture.stderr.join('\n')).toContain( + '[advisory] Inheriting api_url from existing profile: https://api.example.com', + ); + }); + + it('dogfood-2026-05-25 — --endpoint-url flag overrides existing profile api_url', async () => { + const { capture, deps } = makeCapture(); + writeProfile( + 'default', + { apiKey: 'sk-old', apiUrl: 'https://api.example.com' }, + { path: credentialsPath }, + ); + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + endpointUrl: 'https://custom.example.com', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-new' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://custom.example.com', + ); + // No advisory when --endpoint-url was supplied explicitly. + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('dogfood-2026-05-25 — no advisory when inherited url equals DEFAULT_API_URL', async () => { + const { capture, deps } = makeCapture(); + // Existing profile has the default prod endpoint — no advisory needed. + writeProfile( + 'default', + { apiKey: 'sk-old', apiUrl: 'https://api.testsprite.com' }, + { path: credentialsPath }, + ); + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-new' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('dogfood-2026-05-25 — fresh machine (no existing profile) still falls back to DEFAULT_API_URL', async () => { + const { deps } = makeCapture(); + // No existing profile written — credentialsPath points at an empty temp dir. + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-new' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://api.testsprite.com', + ); + }); + + it('dogfood-2026-05-25 — rejection error message includes the resolved endpoint URL', async () => { + const { deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_dd', + details: {}, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + // No existing profile → falls back to DEFAULT_API_URL. + const error = await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(CLIError); + const msg = (error as CLIError).message; + // Must contain the endpoint so the user knows which host rejected the key. + expect(msg).toContain('https://api.testsprite.com'); + // Must contain the TESTSPRITE_API_URL hint. + expect(msg).toContain('TESTSPRITE_API_URL'); + }); + + it('treats an empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile, never "")', async () => { + const { capture, deps } = makeCapture(); + // An exported-but-empty env var (`export TESTSPRITE_API_URL=`) must not + // short-circuit the `??` chain to an empty endpoint; it should fall through + // to the existing profile's api_url. + writeProfile( + 'default', + { apiKey: 'sk-old', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const seenFetchUrls: string[] = []; + const urlAwareFetch: AuthDeps['fetchImpl'] = vi.fn(async (input: unknown) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.toString() : String(input); + seenFetchUrls.push(url); + return new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as unknown as AuthDeps['fetchImpl']; + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-new', TESTSPRITE_API_URL: ' ' }, + credentialsPath, + fetchImpl: urlAwareFetch, + }, + ); + // Empty env → inherit the profile's dev endpoint, never "". + expect(readProfile('default', { path: credentialsPath })?.apiUrl).toBe( + 'https://api.example.com:8443', + ); + // The /me ping must have gone to the inherited dev URL, not an empty/relative one. + expect(seenFetchUrls[0]).toMatch(/^https:\/\/api\.example\.com:8443\//); + // And the inherit advisory must still fire (empty env is treated as absent). + expect(capture.stderr.join('\n')).toContain( + '[advisory] Inheriting api_url from existing profile: https://api.example.com:8443', + ); + }); +}); + +describe('runWhoami', () => { + function makeFetch(response: Response | Error): typeof fetch { + if (response instanceof Error) { + return vi.fn(async () => { + throw response; + }) as unknown as typeof fetch; + } + return vi.fn(async () => response) as unknown as typeof fetch; + } + + function meResponse(): Response { + return new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + it('calls GET /me using the configured profile and prints text output', async () => { + writeProfile( + 'default', + { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { path: credentialsPath }, + ); + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + expect(input.toString()).toBe('https://api.example.com/api/cli/v1/me'); + const headers = new Headers(init?.headers); + expect(headers.get('x-api-key')).toBe('sk-stored'); + expect(headers.get('authorization')).toBeNull(); + return meResponse(); + }); + const me = await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: fetchImpl as unknown as typeof fetch }, + ); + expect(me).toEqual(sampleMe); + expect(capture.stdout.join('\n')).toContain('userId: u-1'); + expect(capture.stdout.join('\n')).toContain('scopes: read:projects, read:tests'); + }); + + it('emits JSON when --output json', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + const printed = JSON.parse(capture.stdout.join('')); + expect(printed).toEqual(sampleMe); + }); + + it('L1788: text output includes the resolved endpoint URL', async () => { + writeProfile( + 'default', + { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { path: credentialsPath }, + ); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: vi.fn( + async () => + new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as typeof fetch, + }, + ); + const out = capture.stdout.join('\n'); + // Endpoint must be surfaced so the user can confirm which env they are in. + expect(out).toContain('endpoint:'); + expect(out).toContain('api.example.com'); + }); + + it('L1788: JSON output does NOT add endpoint (raw /me envelope is passed through)', async () => { + writeProfile( + 'default', + { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { path: credentialsPath }, + ); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: vi.fn( + async () => + new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as typeof fetch, + }, + ); + // JSON mode outputs the raw server envelope — endpoint is only in text. + const parsed = JSON.parse(capture.stdout.join('')) as Record; + expect(parsed.userId).toBe('u-1'); + // endpoint is not part of the wire response, so must not appear in JSON output. + expect(parsed).not.toHaveProperty('endpoint'); + }); + + it('L1866: renders email + name in text mode when the backend supplies them', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithEmail = new Response( + JSON.stringify({ ...sampleMe, email: 'alice@example.com', displayName: 'Alice' }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meWithEmail) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('email: alice@example.com'); + expect(out).toContain('name: Alice'); + expect(out).toContain('userId: u-1'); + }); + + it('L1866: omits email/name lines when the backend does not return them', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('email:'); + expect(out).not.toContain('name:'); + expect(out).toContain('userId: u-1'); + }); + + it('L1866: passes email through verbatim in JSON mode when present', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithEmail = new Response(JSON.stringify({ ...sampleMe, email: 'alice@example.com' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meWithEmail) }, + ); + const printed = JSON.parse(capture.stdout.join('')) as MeResponse; + expect(printed.email).toBe('alice@example.com'); + }); + + it('throws AUTH_REQUIRED locally when no profile/key is configured', async () => { + const { deps } = makeCapture(); + await expect( + runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ), + ).rejects.toMatchObject({ code: 'AUTH_REQUIRED' }); + }); + + it('falls back to TESTSPRITE_API_KEY env when the file has no key', async () => { + const { deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-env' }, + credentialsPath, + fetchImpl: makeFetch(meResponse()), + }, + ); + }); + + it('emits debug events to stderr when debug is enabled', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'json', debug: true }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + const stderr = capture.stderr.join('\n'); + expect(stderr).toContain('"kind":"request"'); + expect(stderr).toContain('"kind":"response"'); + expect(stderr).not.toContain('x-api-key'); + }); + + it('forwards server AUTH_INVALID with exit code 3', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const errorBody = { + error: { + code: 'AUTH_INVALID', + message: 'Bad key.', + nextAction: 'rotate it', + requestId: 'req_x', + details: { reason: 'revoked' }, + }, + }; + await expect( + runWhoami( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: makeFetch(new Response(JSON.stringify(errorBody), { status: 401 })), + }, + ), + ).rejects.toMatchObject({ code: 'AUTH_INVALID', exitCode: 3 }); + }); + + // C2: warn in text mode when key is missing write:tests or run:tests + it('C2 — text mode warns when key lacks write:tests and run:tests', async () => { + const readOnlyMe: MeResponse = { + ...sampleMe, + scopes: ['read:projects', 'read:tests'], + }; + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const fetchImpl = makeFetch( + new Response(JSON.stringify(readOnlyMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('note:'); + expect(out).toContain('write:tests'); + expect(out).toContain('run:tests'); + }); + + it('C2 — text mode shows NO warning when key has both write:tests and run:tests', async () => { + const fullMe: MeResponse = { + ...sampleMe, + scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], + }; + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const fetchImpl = makeFetch( + new Response(JSON.stringify(fullMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('note:'); + }); + + it('C2 — JSON mode does NOT include the scope warning (clean envelope)', async () => { + const readOnlyMe: MeResponse = { + ...sampleMe, + scopes: ['read:projects', 'read:tests'], + }; + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const fetchImpl = makeFetch( + new Response(JSON.stringify(readOnlyMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl }, + ); + // JSON stdout must be the raw MeResponse envelope — no warning field. + const parsed = JSON.parse(capture.stdout.join('')) as Record; + expect(parsed).not.toHaveProperty('note'); + // The warning does NOT appear in stdout when JSON mode is active. + expect(capture.stdout.join('\n')).not.toContain('note:'); + }); +}); + +describe('runLogout', () => { + it('removes the profile and reports success', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('dev', { apiKey: 'sk-dev' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runLogout( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath }, + ); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(readProfile('dev', { path: credentialsPath })).toBeDefined(); + expect(capture.stdout.join('\n')).toContain('Removed credentials'); + }); + + it('reports no_credentials when the profile is not present', async () => { + const { capture, deps } = makeCapture(); + await runLogout( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath }, + ); + expect(capture.stdout.join('\n')).toContain('No credentials stored'); + }); +}); + +describe('createAuthCommand wiring', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exposes the expected subcommands', () => { + const auth = createAuthCommand(); + expect(auth.commands.map(c => c.name()).sort()).toEqual(['configure', 'logout', 'whoami']); + }); + + it('configure --from-env writes the profile and exits 0', async () => { + const { deps } = makeCapture(); + const auth = createAuthCommand({ + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-wired' }, + credentialsPath, + fetchImpl: meOkFetch, + }); + auth.exitOverride(); + auth.commands.forEach(c => c.exitOverride()); + await auth.parseAsync(['configure', '--from-env'], { from: 'user' }); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-wired'); + }); + + it('whoami uses injected fetch and exits 0', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const fetchImpl = vi.fn( + async () => + new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as typeof fetch; + const auth = createAuthCommand({ ...deps, credentialsPath, env: {}, fetchImpl }); + auth.exitOverride(); + auth.commands.forEach(c => c.exitOverride()); + await auth.parseAsync(['whoami'], { from: 'user' }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('L1802: `status` alias resolves to the whoami action', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const fetchImpl = vi.fn( + async () => + new Response(JSON.stringify(sampleMe), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as typeof fetch; + const auth = createAuthCommand({ ...deps, credentialsPath, env: {}, fetchImpl }); + auth.exitOverride(); + auth.commands.forEach(c => c.exitOverride()); + await auth.parseAsync(['status'], { from: 'user' }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('logout removes the profile', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const auth = createAuthCommand({ ...deps, credentialsPath }); + auth.exitOverride(); + auth.commands.forEach(c => c.exitOverride()); + await auth.parseAsync(['logout'], { from: 'user' }); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); +}); + +describe('createAuthCommand surface', () => { + it('returns an ApiError type that maps to the right exit code', () => { + const err = ApiError.authRequired(); + expect(err.exitCode).toBe(3); + }); +}); diff --git a/src/commands/auth.ts b/src/commands/auth.ts new file mode 100644 index 0000000..c9e9c58 --- /dev/null +++ b/src/commands/auth.ts @@ -0,0 +1,337 @@ +import { Command } from 'commander'; +import { + emitDryRunBanner, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import type { ErrorCode } from '../lib/errors.js'; +import { ApiError, CLIError } from '../lib/errors.js'; +import { facadeBaseUrl } from '../lib/facade.js'; +import type { FetchImpl } from '../lib/http.js'; +import { HttpClient } from '../lib/http.js'; +import { + defaultCredentialsPath, + deleteProfile, + readProfile, + writeProfile, +} from '../lib/credentials.js'; +import { loadConfig } from '../lib/config.js'; +import type { OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { promptSecret } from '../lib/prompt.js'; + +export interface MeResponse { + userId: string; + keyId: string; + scopes: string[]; + env: 'development' | 'staging' | 'production'; + /** + * Human-readable email for the bound account. Forward-compat: the + * backend `/me` projection does not surface it yet, so it is optional + * and absent-safe — rendered only when present (dogfood L1866). Keep + * `userId` as the machine-stable join key regardless. + */ + email?: string; + /** Human-readable display name for the bound account. Absent-safe (dogfood L1866). */ + displayName?: string; +} + +export interface AuthDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + prompt?: { + secret: (question: string) => Promise; + }; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + preludeWrite?: (chunk: string) => void; +} + +type CommonOptions = FactoryCommonOptions; + +interface ConfigureOptions extends CommonOptions { + fromEnv: boolean; +} + +const DEFAULT_API_URL = 'https://api.testsprite.com'; +const FROM_ENV_MISSING_KEY = + 'TESTSPRITE_API_KEY is not set in the environment. Set it and re-run with --from-env, or omit --from-env to enter the key interactively.'; + +export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): Promise { + const env = deps.env ?? process.env; + const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); + const out = makeOutput(opts.output, deps); + const prelude = deps.preludeWrite ?? ((chunk: string) => process.stdout.write(chunk)); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // Normalize the env endpoint: an empty / whitespace-only TESTSPRITE_API_URL is + // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a + // shell profile) is non-nullish and would short-circuit the `??` chains below to + // an empty endpoint instead of falling through to the profile / prod default. + const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + + // Dry-run: do not prompt, do not read env, do not write credentials. + // Print the canned success shape so an agent sees exactly the JSON it + // would get on a real configure (modulo the endpoint string). + if (opts.dryRun) { + emitDryRunBanner(stderr); + const apiUrl = opts.endpointUrl ?? envApiUrl ?? DEFAULT_API_URL; + stderr(`[dry-run] would write credentials for profile="${opts.profile}" to ${credentialsPath}`); + out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { + const d = data as { profile: string; apiUrl: string }; + return `Profile "${d.profile}" configured (dry-run). Endpoint: ${d.apiUrl}`; + }); + return; + } + + let apiKey: string | undefined; + + // Read the existing profile once — used for apiUrl inheritance in both + // --from-env and interactive paths when no explicit URL is supplied. + const existingProfile = readProfile(opts.profile, { path: credentialsPath }); + + // The API endpoint is resolved SILENTLY and is never prompted for. Public + // users must not be asked to configure an endpoint at install time — the prod + // default is always correct for them. Internal/staging point elsewhere via the + // global `--endpoint-url` flag or the `TESTSPRITE_API_URL` env var. + // + // Precedence: --endpoint-url flag > TESTSPRITE_API_URL > existing profile's + // api_url > built-in prod default. The existing-profile fallback mirrors + // lib/config.ts runtime resolution so a machine already pointed at a non-default + // api_url doesn't silently validate a new key against the default endpoint. + const resolvedFromProfile = existingProfile?.apiUrl; + const apiUrl = opts.endpointUrl ?? envApiUrl ?? resolvedFromProfile ?? DEFAULT_API_URL; + + if (opts.fromEnv) { + apiKey = env.TESTSPRITE_API_KEY?.trim(); + if (!apiKey) throw validationError('TESTSPRITE_API_KEY', FROM_ENV_MISSING_KEY); + } else { + const promptApi = deps.prompt ?? { secret: (q: string) => promptSecret(q) }; + prelude(`Configuring profile "${opts.profile}".\n`); + // Only the API key is prompted — the endpoint defaults to prod (see above). + apiKey = (await promptApi.secret('TestSprite API key: ')).trim(); + if (!apiKey) throw new CLIError('No API key provided.', 5); + } + + // Advisory: when the endpoint was silently inherited from an existing profile + // (not an explicit flag/env), surface it before validating the key against it + // so a key is never checked against an unexpected host without the user noticing. + if ( + !opts.endpointUrl && + !envApiUrl && + resolvedFromProfile && + resolvedFromProfile !== DEFAULT_API_URL + ) { + stderr(`[advisory] Inheriting api_url from existing profile: ${resolvedFromProfile}`); + } + + // Verify the key is accepted before persisting. Build an HttpClient + // directly (bypassing loadConfig) so we can test the candidate key+url + // before it is written to disk. This ensures we never overwrite a + // working profile with a bad key. + const pingClient = new HttpClient({ + baseUrl: facadeBaseUrl(apiUrl), + apiKey, + fetchImpl: deps.fetchImpl, + }); + try { + await pingClient.get('/me'); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + stderr(`API key rejected by ${apiUrl}: ${message} — profile NOT updated`); + const exitCode = err instanceof ApiError ? err.exitCode : 3; + // Include the resolved endpoint in the thrown message so the user knows + // which host rejected the key. This prevents the "invalid or revoked" + // message from being ambiguous when the key is valid for a different env. + throw new CLIError( + `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`, + exitCode, + ); + } + + writeProfile(opts.profile, { apiKey, apiUrl }, { path: credentialsPath }); + + out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { + const d = data as { profile: string; apiUrl: string }; + return `Profile "${d.profile}" configured. Endpoint: ${d.apiUrl}`; + }); + + // Self-bootstrap: nudge the user toward wiring their coding agent to TestSprite. + // stderr + text-only + real-run-only so JSON consumers and dry-run previews stay clean. + if (opts.output === 'text' && !opts.dryRun) { + stderr( + 'Tip: run `testsprite agent install --target=claude` to let your coding ' + + 'agent run TestSprite tests automatically (also: cursor, cline, antigravity).', + ); + } +} + +export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const env = deps.env ?? process.env; + + // Resolve the endpoint URL so it can be surfaced in text output. + // Dry-run uses the flag/env/default chain without touching credentials. + // Real path uses the same loadConfig the HttpClient factory uses so the + // displayed URL always matches where requests actually go (dogfood L1788). + let resolvedEndpoint: string; + if (opts.dryRun) { + resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; + } else { + const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath, + }); + resolvedEndpoint = config.apiUrl; + } + + // Dry-run + real path both go through the shared factory. + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + + const me = await client.get('/me'); + out.print(me, data => { + const m = data as MeResponse; + const lines = [ + `userId: ${m.userId}`, + // Human-readable identity, rendered only when the backend supplies it + // (dogfood L1866) — confirm the account at a glance before a billable run. + ...(m.displayName ? [`name: ${m.displayName}`] : []), + ...(m.email ? [`email: ${m.email}`] : []), + `keyId: ${m.keyId}`, + // Show the resolved endpoint so the user knows which env they are bound to + // without having to infer from the `env` field alone (dogfood L1788). + `endpoint: ${resolvedEndpoint}`, + `env: ${m.env}`, + `scopes: ${m.scopes.join(', ')}`, + ]; + // C2: warn in text mode when key cannot write/run + const missingScopes = (['write:tests', 'run:tests'] as const).filter( + s => !m.scopes.includes(s), + ); + if (missingScopes.length > 0) { + lines.push( + `note: this key cannot run write or test-trigger commands. Missing scopes: ${missingScopes.join(', ')}. Ask your account owner to extend it via the portal.`, + ); + } + return lines.join('\n'); + }); + return me; +} + +export async function runLogout(opts: CommonOptions, deps: AuthDeps = {}): Promise { + const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderr); + stderr( + `[dry-run] would remove credentials for profile="${opts.profile}" from ${credentialsPath}`, + ); + out.print({ profile: opts.profile, status: 'logged_out' }, data => { + const d = data as { profile: string; status: string }; + return `Removed credentials for profile "${d.profile}" (dry-run).`; + }); + return; + } + + const removed = deleteProfile(opts.profile, { path: credentialsPath }); + out.print({ profile: opts.profile, status: removed ? 'logged_out' : 'no_credentials' }, data => { + const d = data as { profile: string; status: string }; + return d.status === 'logged_out' + ? `Removed credentials for profile "${d.profile}".` + : `No credentials stored for profile "${d.profile}".`; + }); +} + +export function createAuthCommand(deps: AuthDeps = {}): Command { + const auth = new Command('auth').description('Manage TestSprite credentials'); + + auth + .command('configure') + .description('Configure an API key for the active profile') + .option( + '--from-env', + 'Read TESTSPRITE_API_KEY (and optionally TESTSPRITE_API_URL) from the environment instead of prompting', + false, + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: { fromEnv?: boolean }, command: Command) => { + await runConfigure( + { ...resolveCommonOptions(command), fromEnv: Boolean(cmdOpts.fromEnv) }, + deps, + ); + }); + + auth + .command('whoami') + .alias('status') // muscle-memory alias; `auth status` resolves to `auth whoami` (dogfood L1802) + .description('Show the user, API key, and scopes bound to the active profile') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (_cmdOpts, command: Command) => { + await runWhoami(resolveCommonOptions(command), deps); + }); + + auth + .command('logout') + .description('Remove credentials for the active profile') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (_cmdOpts, command: Command) => { + await runLogout(resolveCommonOptions(command), deps); + }); + + return auth; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * Returns `undefined` when the flag was not supplied (factory falls back to + * the env var / default). Silently clamps out-of-range values — the + * factory applies the same clamp so there is no double-clamp risk. + */ +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return undefined; + return Math.round(n * 1000); // seconds → milliseconds +} + +function makeOutput(mode: OutputMode, deps: AuthDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} + +function validationError(field: string, message: string): ApiError { + return ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR' satisfies ErrorCode, + message, + nextAction: `Set ${field} and re-run.`, + requestId: 'local', + details: { field, reason: 'missing' }, + }, + }); +} diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts new file mode 100644 index 0000000..1673810 --- /dev/null +++ b/src/commands/init.test.ts @@ -0,0 +1,828 @@ +/** + * Unit tests for `testsprite init` — all deps injected, no disk or network. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError } from '../lib/errors.js'; +import { resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import type { MeResponse } from './auth.js'; +import type { AgentFs } from './agent.js'; +import type { InitDeps } from './init.js'; +import { runInit } from './init.js'; +import { TARGETS, type AgentTarget } from '../lib/agent-targets.js'; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const ME: MeResponse = { + userId: 'u-test', + keyId: 'k-test', + scopes: ['read:projects', 'write:tests', 'run:tests'], + env: 'development', + email: 'test@example.com', + displayName: 'Test User', +}; + +/** Mock fetch that returns 200 /me response for any request. */ +function makeOkFetch(): InitDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as InitDeps['fetchImpl']; +} + +/** Mock fetch that returns 401 for any request (simulates bad key). */ +function makeAuthFailFetch(): InitDeps['fetchImpl'] { + return vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'Invalid API key', + nextAction: 'Provide a valid key.', + requestId: 'r-1', + }, + }), + { + status: 401, + headers: { 'content-type': 'application/json' }, + }, + ), + ) as unknown as InitDeps['fetchImpl']; +} + +// --------------------------------------------------------------------------- +// In-memory AgentFs +// --------------------------------------------------------------------------- + +function makeMemFs(): { + store: Map; + fs: AgentFs; + writeCalls: string[]; + mkdirCalls: string[]; +} { + const store = new Map(); + const dirs = new Set(); + const writeCalls: string[] = []; + const mkdirCalls: string[] = []; + + const addAncestors = (p: string) => { + let cur = path.dirname(p); + while (cur !== path.dirname(cur)) { + dirs.add(cur); + cur = path.dirname(cur); + } + dirs.add(cur); + }; + + const agentFs: AgentFs = { + async lstat(p: string) { + if (store.has(p)) return { isFile: true, isSymbolicLink: false }; + if (dirs.has(p)) return { isFile: false, isSymbolicLink: false }; + return null; + }, + async readFile(p: string) { + const v = store.get(p); + if (v === undefined) throw Object.assign(new Error(`ENOENT: ${p}`), { code: 'ENOENT' }); + return v; + }, + async writeFile(p: string, data: string, opts?: { exclusive?: boolean }) { + if (opts?.exclusive && (store.has(p) || dirs.has(p))) { + throw Object.assign(new Error(`EEXIST: ${p}`), { code: 'EEXIST' }); + } + writeCalls.push(p); + store.set(p, data); + addAncestors(p); + }, + async mkdir(p: string) { + mkdirCalls.push(p); + dirs.add(p); + addAncestors(p); + }, + }; + + return { store, fs: agentFs, writeCalls, mkdirCalls }; +} + +// --------------------------------------------------------------------------- +// Output capture +// --------------------------------------------------------------------------- + +interface Captured { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { captured: Captured; deps: Pick } { + const captured: Captured = { stdout: [], stderr: [] }; + return { + captured, + deps: { + stdout: line => captured.stdout.push(line), + stderr: line => captured.stderr.push(line), + }, + }; +} + +// --------------------------------------------------------------------------- +// Base options factories +// --------------------------------------------------------------------------- + +const CWD = '/test-project'; + +function makeBaseOpts(overrides: Partial[0]> = {}) { + return { + profile: 'default', + output: 'text' as const, + debug: false, + dryRun: false, + fromEnv: false, + agent: 'claude' as AgentTarget, + noAgent: false, + force: false, + yes: false, + ...overrides, + }; +} + +// --------------------------------------------------------------------------- +// Setup / teardown +// --------------------------------------------------------------------------- + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-init-')), 'credentials'); + resetDryRunBannerForTesting(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// --------------------------------------------------------------------------- +// 1. Happy path — interactive (text + json output) +// --------------------------------------------------------------------------- + +describe('runInit — happy path (interactive)', () => { + it('text mode: prompts key → configure → whoami banner → install → summary', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + const secretPrompt = vi.fn(async () => 'sk-test-key'); + + await runInit(makeBaseOpts(), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + prompt: { secret: secretPrompt }, + isTTY: true, + cwd: CWD, + fs: agentFs, + }); + + // Secret was prompted once + expect(secretPrompt).toHaveBeenCalledOnce(); + + // GET /me was called (configure + whoami = 2 calls minimum) + expect(fetchMock).toHaveBeenCalled(); + + const stdout = captured.stdout.join('\n'); + expect(stdout).toContain('TestSprite initialized.'); + expect(stdout).toContain('profile:'); + expect(stdout).toContain('Next steps:'); + expect(stdout).toContain('testsprite test list'); + }); + + it('json mode: emits structured InitSummary object', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ output: 'json', apiKey: 'sk-json-test' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + // The last stdout line (or join) should be parseable JSON + const jsonOut = captured.stdout.join('\n'); + const parsed = JSON.parse(jsonOut) as Record; + expect(parsed.status).toBe('initialized'); + expect(parsed.profile).toBe('default'); + expect(typeof parsed.apiUrl).toBe('string'); + expect(Array.isArray(parsed.scopes)).toBe(true); + expect(parsed.agent).not.toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// 2. --yes --api-key: zero prompts, default claude agent +// --------------------------------------------------------------------------- + +describe('runInit — --yes --api-key (non-interactive)', () => { + it('completes with zero prompts, uses claude as agent target', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + const secretPrompt = vi.fn(async () => 'should-never-be-called'); + + await runInit(makeBaseOpts({ apiKey: 'sk-test', yes: true }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + prompt: { secret: secretPrompt }, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + // Prompt never called + expect(secretPrompt).not.toHaveBeenCalled(); + + const stdout = captured.stdout.join('\n'); + expect(stdout).toContain('claude'); + expect(stdout).toContain('initialized'); + }); +}); + +// --------------------------------------------------------------------------- +// 3. --no-agent: install NOT called, summary shows agent: null +// --------------------------------------------------------------------------- + +describe('runInit — --no-agent', () => { + it('skips agent install; summary has agent: null in JSON', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs, writeCalls } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ apiKey: 'sk-test', noAgent: true, output: 'json' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + // No skill file written + expect(writeCalls.length).toBe(0); + + const jsonOut = captured.stdout.join('\n'); + const parsed = JSON.parse(jsonOut) as Record; + expect(parsed.agent).toBeNull(); + }); + + it('text mode shows "skipped (--no-agent)"', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ apiKey: 'sk-test', noAgent: true }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const stdout = captured.stdout.join('\n'); + expect(stdout).toContain('skipped (--no-agent)'); + }); +}); + +// --------------------------------------------------------------------------- +// 4. --agent cursor: passes target:'cursor' to runInstall +// --------------------------------------------------------------------------- + +describe('runInit — --agent cursor', () => { + it('installs cursor skill at the correct matrix path', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs, writeCalls } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ apiKey: 'sk-test', agent: 'cursor' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const cursorAbsPath = path.resolve(CWD, TARGETS.cursor.path); + expect(writeCalls).toContain(cursorAbsPath); + + const stdout = captured.stdout.join('\n'); + expect(stdout).toContain('cursor'); + }); +}); + +// --------------------------------------------------------------------------- +// 5. --dry-run: zero fetch calls, zero fs writes +// --------------------------------------------------------------------------- + +describe('runInit — --dry-run', () => { + it('makes no fetch calls and no fs writes', async () => { + const { deps } = makeCapture(); + const { fs: agentFs, writeCalls, mkdirCalls } = makeMemFs(); + const fetchMock = vi.fn( + async () => new Response('{}', { status: 200 }), + ) as unknown as InitDeps['fetchImpl']; + + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + // No network + expect(fetchMock).not.toHaveBeenCalled(); + // No file writes + expect(writeCalls).toHaveLength(0); + expect(mkdirCalls).toHaveLength(0); + }); + + it('emits dry-run banners and preview lines on stderr', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry' }), { + ...deps, + fetchImpl: vi.fn(async () => new Response('{}')) as unknown as InitDeps['fetchImpl'], + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const stderr = captured.stderr.join('\n'); + expect(stderr).toContain('[dry-run]'); + expect(stderr).toContain('preview only'); + }); + + it('dry-run --no-agent: still no fetch, no writes, summary shows agent: null', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs, writeCalls } = makeMemFs(); + const fetchMock = vi.fn(async () => new Response('{}')) as unknown as InitDeps['fetchImpl']; + + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry', noAgent: true, output: 'json' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(writeCalls).toHaveLength(0); + + const jsonOut = captured.stdout.join('\n'); + const parsed = JSON.parse(jsonOut) as Record; + expect(parsed.agent).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// 6. No TTY + no key + no --from-env → exit 5 +// --------------------------------------------------------------------------- + +describe('runInit — no TTY + no key source → exit 5', () => { + it('throws CLIError with exit 5 when non-interactive and no key available', async () => { + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInit(makeBaseOpts(), { + ...deps, + isTTY: false, + }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + const msg = (thrown as CLIError).message; + expect(msg).toContain('--api-key'); + }); +}); + +// --------------------------------------------------------------------------- +// 6b. Codex-review fixes — dry-run bypass, key precedence, endpoint, JSON guard +// --------------------------------------------------------------------------- + +describe('runInit — codex-review hardening', () => { + it('--dry-run bypasses the no-key guard in non-interactive mode (no throw, no fetch)', async () => { + const { captured, deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + // No TTY, no apiKey, no fromEnv — but dry-run must still preview, not exit 5. + await runInit(makeBaseOpts({ dryRun: true, noAgent: true }), { + ...deps, + fetchImpl, + credentialsPath, + isTTY: false, + }); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(captured.stderr.some(l => l.includes('[dry-run]'))).toBe(true); + }); + + it('--api-key wins over --from-env (configure uses the explicit key, not env)', async () => { + const { deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + // env has NO TESTSPRITE_API_KEY; if --from-env wrongly won, runConfigure would + // read undefined and throw. Success proves --api-key took precedence. + await runInit(makeBaseOpts({ apiKey: 'sk-wins', fromEnv: true, noAgent: true }), { + ...deps, + env: {}, + fetchImpl, + credentialsPath, + isTTY: false, + }); + expect(fetchImpl).toHaveBeenCalled(); + }); + + it('whoami banner uses --api-key, not a stale TESTSPRITE_API_KEY in env (E2E 2026-06-09)', async () => { + const { captured, deps } = makeCapture(); + // Key-aware fetch: only the real key gets a 200 + identity; the stale env key 401s. + // The bug was: runWhoami read env.TESTSPRITE_API_KEY (stale) → 401 → misleading + // production/no-email banner even though configure wrote the correct key. + const fetchImpl = vi.fn(async (_url: string, init: { headers?: Record }) => { + const key = init.headers?.['x-api-key'] ?? init.headers?.['X-API-Key']; + if (key === 'sk-real') { + return new Response(JSON.stringify(ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'AUTH_INVALID', message: 'Invalid API key', requestId: 'r' }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ); + }) as unknown as InitDeps['fetchImpl']; + + await runInit(makeBaseOpts({ apiKey: 'sk-real', noAgent: true, output: 'json' }), { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-stale-bogus' }, + fetchImpl, + credentialsPath, + isTTY: false, + }); + const summary = JSON.parse(captured.stdout.join('\n')) as { + email?: string; + env: string; + scopes: string[]; + }; + // Real-key identity must surface — NOT the 401 placeholder (production/no-email/[]). + expect(summary.email).toBe(ME.email); + expect(summary.env).toBe('development'); + expect(summary.scopes.length).toBeGreaterThan(0); + }); + + it('summary reports the endpoint from TESTSPRITE_API_URL, not a flat prod default', async () => { + const { captured, deps } = makeCapture(); + await runInit(makeBaseOpts({ apiKey: 'sk-env-url', noAgent: true, output: 'json' }), { + ...deps, + env: { TESTSPRITE_API_URL: 'https://api.example.com:8443' }, + fetchImpl: makeOkFetch(), + credentialsPath, + isTTY: false, + }); + const summary = JSON.parse(captured.stdout.join('\n')) as { apiUrl: string }; + expect(summary.apiUrl).toBe('https://api.example.com:8443'); + }); + + it('--output json with an interactive prompt (no key source) → exit 5 (protects JSON stdout)', async () => { + const { deps } = makeCapture(); + let thrown: unknown; + try { + await runInit(makeBaseOpts({ output: 'json' }), { + ...deps, + fetchImpl: makeOkFetch(), + credentialsPath, + isTTY: true, // interactive: would otherwise promptSecret → stdout + }); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message.toLowerCase()).toContain('json'); + }); +}); + +// --------------------------------------------------------------------------- +// 7. Bad key → runConfigure throws → auth error propagates (exit 3) +// --------------------------------------------------------------------------- + +describe('runInit — bad API key', () => { + it('propagates auth error from runConfigure (exit 3)', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + + let thrown: unknown; + try { + await runInit(makeBaseOpts({ apiKey: 'sk-bad' }), { + ...deps, + fetchImpl: makeAuthFailFetch(), + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + } catch (err) { + thrown = err; + } + + // runConfigure throws a CLIError wrapping the auth failure + expect(thrown).toBeDefined(); + const exitCode = + thrown instanceof CLIError + ? thrown.exitCode + : thrown instanceof ApiError + ? thrown.exitCode + : -1; + expect(exitCode).toBe(3); + }); +}); + +// --------------------------------------------------------------------------- +// 8. Summary JSON shape +// --------------------------------------------------------------------------- + +describe('runInit — summary JSON shape', () => { + it('JSON summary has all required top-level fields', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ apiKey: 'sk-shape', output: 'json' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const parsed = JSON.parse(captured.stdout.join('\n')) as Record; + expect(parsed).toHaveProperty('profile'); + expect(parsed).toHaveProperty('apiUrl'); + expect(parsed).toHaveProperty('env'); + expect(parsed).toHaveProperty('scopes'); + expect(parsed).toHaveProperty('agent'); + expect(parsed).toHaveProperty('status', 'initialized'); + }); + + it('JSON summary agent field has target and action', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + + await runInit(makeBaseOpts({ apiKey: 'sk-agent-shape', output: 'json' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const parsed = JSON.parse(captured.stdout.join('\n')) as { + agent: { target: string; action: string } | null; + }; + expect(parsed.agent).not.toBeNull(); + expect(parsed.agent?.target).toBe('claude'); + expect(typeof parsed.agent?.action).toBe('string'); + }); +}); + +// --------------------------------------------------------------------------- +// 9. --from-env reads TESTSPRITE_API_KEY +// --------------------------------------------------------------------------- + +describe('runInit — --from-env', () => { + it('reads key from env, no prompt', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + const secretPrompt = vi.fn(async () => 'should-not-be-called'); + + await runInit(makeBaseOpts({ fromEnv: true }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + env: { TESTSPRITE_API_KEY: 'sk-from-env-key' }, + prompt: { secret: secretPrompt }, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + expect(secretPrompt).not.toHaveBeenCalled(); + const stdout = captured.stdout.join('\n'); + expect(stdout).toContain('initialized'); + }); +}); + +// --------------------------------------------------------------------------- +// 10. All valid agent targets +// --------------------------------------------------------------------------- + +describe('runInit — all agent targets', () => { + const allTargets = Object.keys(TARGETS) as AgentTarget[]; + + for (const target of allTargets) { + it(`target=${target}: installs to correct matrix path`, async () => { + const { deps } = makeCapture(); + const { fs: agentFs, writeCalls } = makeMemFs(); + const fetchMock = makeOkFetch(); + + // Reset banner state per-test since module-level state persists + resetDryRunBannerForTesting(); + + // Fresh credentials path per target + const localCreds = join( + mkdtempSync(join(tmpdir(), `testsprite-init-target-${target}-`)), + 'credentials', + ); + + await runInit(makeBaseOpts({ apiKey: 'sk-target', agent: target }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const expectedPath = path.resolve(CWD, TARGETS[target].path); + expect(writeCalls).toContain(expectedPath); + }); + } +}); + +// --------------------------------------------------------------------------- +// [B-E2E-05] Fix 5 regression — --no-agent + --agent conflict warning +// --------------------------------------------------------------------------- + +describe('[B-E2E-05] runInit: --no-agent + --agent conflict emits [warn] on stderr', () => { + // Commander sets opts.agent=false when --no-agent is passed (negation flag). + // When both --agent and --no-agent are in rawArgs, the CLI should + // emit a [warn] and apply last-flag-wins semantics. + // runInit receives the pre-resolved noAgent boolean and agent value from + // Commander; the conflict is detected via a rawArgs scan in the command action. + // These tests exercise runInit with rawArgConflict=true injected as the flag. + + it('warns on stderr when rawArgConflict=true and noAgent wins', async () => { + // Simulate: user passed --agent cursor --no-agent (--no-agent last → noAgent=true) + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + const localCreds = join(mkdtempSync(join(tmpdir(), 'testsprite-init-fix5a-')), 'credentials'); + + // Pass rawArgConflict signal: noAgent=true wins (--no-agent was last) + // runInit exposes a rawArgConflict option that the command action passes + // when it detects both --agent and --no-agent in rawArgs. + await runInit(makeBaseOpts({ apiKey: 'sk-conflict', noAgent: true, rawArgConflict: true }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: agentFs, + }); + + const warnLine = captured.stderr.find(l => l.includes('[warn]') && l.includes('--no-agent')); + expect(warnLine).toBeDefined(); + }); + + it('warns on stderr when rawArgConflict=true and --agent wins', async () => { + // Simulate: user passed --no-agent --agent cursor (--agent last → agent='cursor') + const { captured, deps } = makeCapture(); + const { fs: agentFs, writeCalls } = makeMemFs(); + const fetchMock = makeOkFetch(); + const localCreds = join(mkdtempSync(join(tmpdir(), 'testsprite-init-fix5b-')), 'credentials'); + + await runInit( + makeBaseOpts({ + apiKey: 'sk-conflict2', + agent: 'cursor', + noAgent: false, + rawArgConflict: true, + }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: agentFs, + }, + ); + + const warnLine = captured.stderr.find(l => l.includes('[warn]') && l.includes('--no-agent')); + expect(warnLine).toBeDefined(); + + // --agent cursor wins → cursor file should be written + const cursorPath = path.resolve(CWD, TARGETS.cursor.path); + expect(writeCalls).toContain(cursorPath); + }); + + it('no warning when only --agent is passed (no conflict)', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + const fetchMock = makeOkFetch(); + const localCreds = join(mkdtempSync(join(tmpdir(), 'testsprite-init-fix5c-')), 'credentials'); + + await runInit( + // rawArgConflict not set (default undefined/false) + makeBaseOpts({ apiKey: 'sk-no-conflict', agent: 'claude' }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: agentFs, + }, + ); + + const warnLine = captured.stderr.find(l => l.includes('[warn]') && l.includes('--no-agent')); + expect(warnLine).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// [B-E2E-06] Fix 6 regression — install failure emits info message about creds +// --------------------------------------------------------------------------- + +describe('[B-E2E-06] runInit: install failure → info message on stderr + re-throws', () => { + // When the agent install step fails (e.g. bad --dir path), credentials are + // already saved. The CLI should emit an [info] saying credentials are saved + // and suggesting 'testsprite agent install' before re-throwing. + + it('emits [info] about saved credentials when install throws, then re-throws', async () => { + const { captured, deps } = makeCapture(); + const fetchMock = makeOkFetch(); + const localCreds = join(mkdtempSync(join(tmpdir(), 'testsprite-init-fix6-')), 'credentials'); + + // Inject an AgentFs that throws on writeFile to simulate install failure + const failFs: AgentFs = { + async lstat() { + return null; + }, + async readFile() { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }, + async writeFile() { + throw new Error('ENOENT: no such directory'); + }, + async mkdir() { + throw new Error('ENOENT: no such directory'); + }, + }; + + let caughtErr: unknown; + try { + await runInit(makeBaseOpts({ apiKey: 'sk-install-fail', agent: 'claude' }), { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: failFs, + }); + } catch (err) { + caughtErr = err; + } + + // Must re-throw + expect(caughtErr).toBeDefined(); + + // Must emit an [info] mentioning credentials were saved + agent install hint + const infoLine = captured.stderr.find( + l => l.includes('[info]') && (l.includes('credentials') || l.includes('agent install')), + ); + expect(infoLine).toBeDefined(); + }); +}); diff --git a/src/commands/init.ts b/src/commands/init.ts new file mode 100644 index 0000000..c6b48c0 --- /dev/null +++ b/src/commands/init.ts @@ -0,0 +1,493 @@ +/** + * `testsprite init` — one-shot onboarding orchestrator. + * + * Chains, in order: + * 1. runConfigure — writes the API-key profile (validates via GET /me first) + * 2. runWhoami — fetches identity for the post-configure banner + * 3. runInstall — installs the TestSprite verification-loop skill (unless --no-agent) + * 4. Summary print — JSON object or human text block + * + * Hard constraint: orchestrate existing exported primitives; never fork them. + */ + +import { Command } from 'commander'; +import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { CLIError } from '../lib/errors.js'; +import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import type { AuthDeps, MeResponse } from './auth.js'; +import { runConfigure, runWhoami } from './auth.js'; +import type { AgentDeps, AgentFs, InstallResult } from './agent.js'; +import { runInstall } from './agent.js'; +import { TARGETS, type AgentTarget } from '../lib/agent-targets.js'; +import type { FetchImpl } from '../lib/http.js'; +import { readProfile } from '../lib/credentials.js'; + +/** Mirrors auth.ts's DEFAULT_API_URL (kept in sync; auth.ts owns the canonical value). */ +const DEFAULT_API_URL = 'https://api.testsprite.com'; + +/** + * Resolve the endpoint the summary should report, using the SAME precedence + * `runConfigure` uses to pick (and persist) the endpoint: + * --endpoint-url > TESTSPRITE_API_URL env > existing profile apiUrl > prod default. + * Reporting a flat prod default would falsely claim a prod target after + * configuring staging/dev (codex). On the real path this runs AFTER the profile + * is written, so the persisted apiUrl is reflected faithfully. + */ +function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { + const env = deps.env ?? process.env; + const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + let existing: string | undefined; + try { + existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; + } catch { + existing = undefined; + } + return opts.endpointUrl ?? envApiUrl ?? existing ?? DEFAULT_API_URL; +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type CommonOptions = FactoryCommonOptions; + +/** + * InitDeps merges AuthDeps and AgentDeps. Because the `prompt` field differs + * between them (`{secret: fn}` in AuthDeps vs a plain function in AgentDeps), + * we compose manually and expose `agentPrompt` for the agent install step. + */ +export interface InitDeps { + // Shared output/environment + env?: NodeJS.ProcessEnv; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + + // AuthDeps-specific + credentialsPath?: string; + fetchImpl?: FetchImpl; + /** Injected for auth configure: { secret: (q) => Promise } */ + prompt?: AuthDeps['prompt']; + preludeWrite?: (chunk: string) => void; + + // AgentDeps-specific + cwd?: string; + fs?: AgentFs; + isTTY?: boolean; + /** Injected for agent install prompt (plain function, not {secret: fn}) */ + agentPrompt?: (question: string) => Promise; +} + +interface InitOptions extends CommonOptions { + apiKey?: string; + fromEnv: boolean; + agent: string; + noAgent: boolean; + force: boolean; + dir?: string; + yes: boolean; + /** Set by the command action when both --agent and --no-agent appear in rawArgs. */ + rawArgConflict?: boolean; +} + +export interface InitSummary { + profile: string; + apiUrl: string; + env: string; + email?: string; + scopes: string[]; + agent: { target: string; action: string } | null; + status: 'initialized'; +} + +// --------------------------------------------------------------------------- +// Helpers to split deps into the two primitive shapes +// --------------------------------------------------------------------------- + +/** + * Build AuthDeps from InitDeps. `stdout` is intentionally suppressed here + * because runInit owns the final output — runConfigure's success message + * and runWhoami's identity block are replaced by the init summary. + * stderr (advisory messages, errors) flows through. + */ +function toAuthDeps(deps: InitDeps, apiKey?: string): AuthDeps { + return { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stdout: _suppressedStdout, + stderr: deps.stderr, + // Forward the preludeWrite so injected tests can capture the "Configuring + // profile..." line, but default to a no-op so tests that don't care don't + // see it on real process.stdout. + preludeWrite: deps.preludeWrite ?? _suppressedStdout, + // If an explicit API key was provided, override the prompt so configure + // never actually prompts the user. + prompt: apiKey ? { secret: async (_q: string) => apiKey } : deps.prompt, + }; +} + +/** + * Build AgentDeps from InitDeps. `stdout` is suppressed for the same reason — + * runInit owns output. The result is parsed from the captured JSON in the + * caller, not forwarded to user stdout. + */ +function toAgentDeps(deps: InitDeps, captureStdout?: (line: string) => void): AgentDeps { + return { + cwd: deps.cwd, + fs: deps.fs, + stdout: captureStdout ?? _suppressedStdout, + stderr: deps.stderr, + isTTY: deps.isTTY, + prompt: deps.agentPrompt, + }; +} + +// Discards stdout lines from sub-commands so runInit owns the output surface. +function _suppressedStdout(_line: string): void { + // intentionally empty +} + +// --------------------------------------------------------------------------- +// runInit +// --------------------------------------------------------------------------- + +export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = new Output(opts.output, { stdout: deps.stdout, stderr: deps.stderr }); + + // ------------------------------------------------------------------------- + // Fix 5: emit conflict warning when both --agent and --no-agent were given + // ------------------------------------------------------------------------- + if (opts.rawArgConflict) { + const effectiveLabel = opts.noAgent ? '--no-agent' : `--agent ${opts.agent}`; + stderrFn( + `[warn] both --no-agent and --agent supplied; using ${effectiveLabel} (last flag wins)`, + ); + } + + // ------------------------------------------------------------------------- + // Non-interactive guard: no TTY + no key source → exit 5 immediately + // ------------------------------------------------------------------------- + const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); + const hasKeySource = Boolean(opts.apiKey) || opts.fromEnv; + // Non-interactive guard: no TTY + no key source → exit 5. Skipped under + // --dry-run, which is documented to work without credentials or network. + if (!isTTY && !hasKeySource && !opts.dryRun) { + throw new CLIError( + 'No API key available in non-interactive mode. ' + + 'Pass --api-key , --from-env (reads TESTSPRITE_API_KEY), or run interactively.', + 5, + ); + } + // JSON-output guard: an interactive secret prompt writes to stdout and would + // corrupt init's single-JSON-object output contract. In --output json mode + // require a non-interactive key source. Skipped under --dry-run (never prompts). + if (opts.output === 'json' && !hasKeySource && !opts.dryRun) { + throw new CLIError( + 'Interactive API-key prompt is unavailable in --output json mode (it would corrupt JSON stdout). ' + + 'Pass --api-key or --from-env.', + 5, + ); + } + + // ------------------------------------------------------------------------- + // Dry-run: zero network + zero FS writes; print preview only + // ------------------------------------------------------------------------- + if (opts.dryRun) { + stderrFn('[dry-run] no writes or network calls — preview only'); + stderrFn( + `[dry-run] would configure profile="${opts.profile}" (key source: ${ + opts.apiKey ? 'flag' : opts.fromEnv ? 'env' : 'prompt' + })`, + ); + + const agentTarget = opts.noAgent ? null : opts.agent; + + if (!opts.noAgent) { + // Delegate to runInstall's own dry-run for the file-listing preview. + // runInstall prints the would-write lines itself under dryRun. + await runInstall( + { + ...opts, + target: [agentTarget!], + force: opts.force, + dir: opts.dir, + }, + toAgentDeps(deps), + ); + } + + const summary: InitSummary = { + profile: opts.profile, + apiUrl: resolveReportedEndpoint(opts, deps), + env: 'development', + scopes: [], + agent: agentTarget ? { target: agentTarget, action: 'dry-run' } : null, + status: 'initialized', + }; + + out.print(summary, renderInitText); + return; + } + + // ------------------------------------------------------------------------- + // Step 1: Configure — validates key via GET /me before writing the profile + // ------------------------------------------------------------------------- + // --api-key takes precedence over --from-env: when an explicit key is supplied, + // force fromEnv=false so runConfigure uses the injected key (toAuthDeps wires it + // as the prompt) instead of reading TESTSPRITE_API_KEY from the environment (codex). + await runConfigure( + { ...opts, fromEnv: opts.apiKey ? false : opts.fromEnv }, + toAuthDeps(deps, opts.apiKey), + ); + + // ------------------------------------------------------------------------- + // Step 2: Whoami banner — for identity display only; not used for validation + // ------------------------------------------------------------------------- + // runWhoami resolves its key via loadConfig (`env.TESTSPRITE_API_KEY ?? profile`). + // When the user passed an explicit --api-key, that key was just written to the + // profile above — but a STALE/different TESTSPRITE_API_KEY still in the environment + // would WIN in loadConfig and make the banner read the wrong identity (a bogus key → + // 401 → misleading `production`/no-email summary). Strip it for the whoami read so it + // uses the profile we just wrote (E2E finding 2026-06-09). Only when --api-key was + // given: a bare --from-env run legitimately relies on the env key. + const whoamiDeps = toAuthDeps(deps); + if (opts.apiKey) { + const sanitizedEnv = { ...(deps.env ?? process.env) }; + delete sanitizedEnv.TESTSPRITE_API_KEY; + whoamiDeps.env = sanitizedEnv; + } + let me: MeResponse; + try { + me = await runWhoami(opts, whoamiDeps); + } catch { + // Whoami is display-only. If it fails after a successful configure, + // continue with a minimal placeholder so the summary still prints. + me = { userId: '', keyId: '', scopes: [], env: 'production' }; + } + + // ------------------------------------------------------------------------- + // Step 3: Agent skill install (unless --no-agent) + // ------------------------------------------------------------------------- + let installedTarget: string | null = null; + let installedAction: string | null = null; + + if (!opts.noAgent) { + // Run install in JSON mode internally so we can reliably parse the action + // regardless of the outer --output flag. The stdout is captured here and + // NOT forwarded — runInit owns the output surface. + let capturedInstallResult: InstallResult | null = null; + const captureStdout = (line: string) => { + try { + const parsed = JSON.parse(line) as InstallResult[]; + if (Array.isArray(parsed) && parsed.length > 0 && parsed[0]) { + capturedInstallResult = parsed[0]; + } + } catch { + // ignore non-JSON lines (shouldn't happen in json mode, but be safe) + } + }; + + try { + await runInstall( + { + ...opts, + output: 'json', // parse the result; final summary is ours to print + target: [opts.agent], + force: opts.force, + dir: opts.dir, + }, + toAgentDeps(deps, captureStdout), + ); + + installedTarget = opts.agent; + installedAction = capturedInstallResult + ? (capturedInstallResult as InstallResult).action + : 'installed'; + } catch (installErr) { + // Fix 6: credentials were already saved (Step 1+2 above succeeded). + // Emit a clear summary line BEFORE re-throwing so the user knows their + // API key was persisted — only the agent skill step failed (Fix 6). + stderrFn( + `[info] credentials saved for profile "${opts.profile}"; only the agent skill install failed — ` + + `re-run 'testsprite agent install --target ${opts.agent}' after fixing the path`, + ); + throw installErr; + } + } + + // ------------------------------------------------------------------------- + // Step 4: Summary + // ------------------------------------------------------------------------- + const agentSummary: InitSummary['agent'] = + opts.noAgent || installedTarget === null + ? null + : { target: installedTarget, action: installedAction ?? 'installed' }; + + const summary: InitSummary = { + profile: opts.profile, + // Resolved AFTER configure persists the profile → reflects the real endpoint + // (staging/dev/prod), not a flat prod default (codex). + apiUrl: resolveReportedEndpoint(opts, deps), + env: me.env, + email: me.email, + scopes: me.scopes, + agent: agentSummary, + status: 'initialized', + }; + + out.print(summary, renderInitText); +} + +// --------------------------------------------------------------------------- +// Text renderer +// --------------------------------------------------------------------------- + +function renderInitText(data: unknown): string { + const s = data as InitSummary; + const lines: string[] = []; + + lines.push('TestSprite initialized.'); + lines.push(''); + lines.push(` profile: ${s.profile}`); + lines.push(` endpoint: ${s.apiUrl}`); + lines.push(` env: ${s.env}`); + if (s.email) lines.push(` email: ${s.email}`); + if (s.scopes.length > 0) lines.push(` scopes: ${s.scopes.join(', ')}`); + lines.push(''); + if (s.agent) { + lines.push(` agent: ${s.agent.target} (${s.agent.action})`); + } else { + lines.push(' agent: skipped (--no-agent)'); + } + lines.push(''); + lines.push('Next steps:'); + lines.push(' testsprite test list # list tests in the current project'); + lines.push(' testsprite agent list # check installed agent targets'); + if (s.agent) { + lines.push( + ' testsprite agent install --target= # re-install or install additional targets', + ); + } + + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Command factory +// --------------------------------------------------------------------------- + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return undefined; + return Math.round(n * 1000); +} + +export function createInitCommand(deps: InitDeps = {}): Command { + const validTargets = Object.keys(TARGETS) as AgentTarget[]; + const defaultAgent: AgentTarget = 'claude'; + + const cmd = new Command('init') + .description('One-shot onboarding: configure an API key and install the coding-agent skill') + .option('--api-key ', 'API key to configure (skips the interactive prompt)') + .option( + '--from-env', + 'Read TESTSPRITE_API_KEY from the environment instead of prompting', + false, + ) + .option( + '--agent ', + `Coding-agent target to install: ${validTargets.join(', ')} (default: ${defaultAgent})`, + defaultAgent, + ) + .option('--no-agent', 'Skip the agent skill install (configure credentials only)') + .option('--force', 'Overwrite an existing skill file (a .bak backup is kept)') + .option('--dir ', 'Project root for the skill install (default: current directory)') + .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { + apiKey?: string; + fromEnv?: boolean; + /** + * Commander sets `agent: false` (boolean) when `--no-agent` is given, + * because `--no-agent` negates the `--agent ` option. + * Handle both string and false shapes. + */ + agent: string | false; + noAgent?: boolean; + force?: boolean; + dir?: string; + yes?: boolean; + }, + command: Command, + ) => { + const common = resolveCommonOptions(command); + + // Commander sets `agent: false` (boolean) when `--no-agent` is passed, + // because `--no-agent` is the negation of `--agent `. Guard + // against this: if agent is falsy (false or empty string) default to + // the CLI default so runInstall never receives a non-string target. + const resolvedAgent = + cmdOpts.agent && typeof cmdOpts.agent === 'string' ? cmdOpts.agent : defaultAgent; + const isNoAgent = cmdOpts.noAgent === true || cmdOpts.agent === false; + + // Fix 5: detect conflict when both --no-agent and --agent appear + // in the raw args. Thread the flag into runInit so it can emit the [warn] + // via deps.stderr (testable) instead of process.stderr directly. + // + // Commander only populates `rawArgs` on the ROOT command passed to + // parseAsync; subcommands have an empty array. Walk up to the root + // so we always inspect the full argv. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let root: any = command; + while (root.parent) root = root.parent; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const rawArgs: string[] = (root as any).rawArgs ?? process.argv; + const rawArgConflict = + rawArgs.some((a: string) => a === '--no-agent') && + rawArgs.some((a: string) => a === '--agent' || a.startsWith('--agent=')); + + const opts: InitOptions = { + ...common, + apiKey: cmdOpts.apiKey, + fromEnv: Boolean(cmdOpts.fromEnv), + agent: resolvedAgent, + noAgent: isNoAgent, + force: Boolean(cmdOpts.force), + dir: cmdOpts.dir, + yes: Boolean(cmdOpts.yes), + rawArgConflict, + }; + + // When --yes is supplied without a key source, force isTTY=false so + // runInit emits exit 5 with a clear message rather than hanging on a + // prompt in a headless CI environment where a TTY fd happens to be open. + const effectiveDeps: InitDeps = { + ...deps, + ...(opts.yes && !opts.apiKey && !opts.fromEnv ? { isTTY: false } : {}), + }; + + await runInit(opts, effectiveDeps); + }, + ); + + return cmd; +} diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts new file mode 100644 index 0000000..0037b8c --- /dev/null +++ b/src/commands/project.test.ts @@ -0,0 +1,751 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError } from '../lib/errors.js'; +import { + type CliProject, + type CliUpdateProjectResponse, + createProjectCommand, + runCreate, + runGet, + runList, + runUpdate, +} from './project.js'; + +const PROJECT_FIXTURE: CliProject = { + id: 'project_b3c91efa', + name: 'Checkout', + type: 'frontend', + createdFrom: 'portal', + createdAt: '2026-04-15T10:23:00.000Z', + updatedAt: '2026-05-05T08:12:00.000Z', +}; + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13501', +): { + credentialsPath: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'cli-p2-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +describe('createProjectCommand', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exposes list, get, create and update subcommands', () => { + const project = createProjectCommand(); + const names = project.commands.map(c => c.name()).sort(); + expect(names).toEqual(['create', 'get', 'list', 'update']); + }); + + it('list exposes the pagination flags from the design contract', () => { + const project = createProjectCommand(); + const list = project.commands.find(c => c.name() === 'list')!; + const flagNames = list.options.map(o => o.long); + expect(flagNames).toContain('--page-size'); + expect(flagNames).toContain('--starting-token'); + expect(flagNames).toContain('--max-items'); + }); +}); + +describe('runList', () => { + it('returns the first page when no flags are passed (auto-paging follows nextToken)', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch((_url, _init) => { + calls += 1; + if (calls === 1) { + return { + body: { items: [PROJECT_FIXTURE], nextToken: 'opaque-cursor-1' }, + }; + } + return { body: { items: [{ ...PROJECT_FIXTURE, id: 'project_2' }], nextToken: null } }; + }); + + const out: string[] = []; + const page = await runList( + { profile: 'default', output: 'json', debug: false }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(calls).toBe(2); + expect(page.items).toHaveLength(2); + expect(page.nextToken).toBeNull(); + expect(JSON.parse(out[0]!).items).toHaveLength(2); + }); + + it('--page-size returns one page (no auto-paging) and surfaces the nextToken', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { + body: { + items: [PROJECT_FIXTURE], + nextToken: 'opaque-cursor-A', + }, + }; + }); + + const out: string[] = []; + const page = await runList( + { profile: 'default', output: 'json', debug: false, pageSize: 1 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(seen).toHaveLength(1); + expect(seen[0]).toContain('pageSize=1'); + expect(page.items).toHaveLength(1); + expect(page.nextToken).toBe('opaque-cursor-A'); + }); + + it('--max-items caps the result count across multiple pages', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch(() => { + calls += 1; + return { + body: { + items: [ + { ...PROJECT_FIXTURE, id: `project_${calls}_a` }, + { ...PROJECT_FIXTURE, id: `project_${calls}_b` }, + ], + nextToken: calls < 3 ? `cursor-${calls}` : null, + }, + }; + }); + + const page = await runList( + { profile: 'default', output: 'json', debug: false, maxItems: 3 }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + + expect(page.items).toHaveLength(3); + // Server still has more pages; the resumable token surfaces. + expect(page.nextToken).toBe('cursor-2'); + }); + + it('--starting-token resumes pagination from the supplied cursor', async () => { + const { credentialsPath } = makeCreds(); + const seenCursors: Array = []; + const fetchImpl = makeFetch(url => { + const match = /cursor=([^&]+)/.exec(url); + seenCursors.push(match ? decodeURIComponent(match[1]!) : null); + return { body: { items: [PROJECT_FIXTURE], nextToken: null } }; + }); + + await runList( + { profile: 'default', output: 'json', debug: false, startingToken: 'resume-here' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + + expect(seenCursors[0]).toBe('resume-here'); + }); + + it('rejects pageSize=0 with a local VALIDATION_ERROR (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('network should not be hit'); + }); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 0 }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: { field: 'page-size' }, + }); + }); + + it('rejects pageSize=101 with VALIDATION_ERROR exit 5 (Fix 7 — upper-bound enforced client-side)', async () => { + // Previously silently clamped to 100; now rejected so callers get fast feedback. + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('network should not be hit'); + }); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 101 }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + }); + + it('renders text output with a column header and nextToken footer when present', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: 'next-please' }, + })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toContain('ID'); + expect(block).toContain('NAME'); + expect(block).toContain('TYPE'); + expect(block).toContain('FROM'); + expect(block).toContain('CREATED'); + expect(block).toContain('Checkout'); + expect(block).toContain('nextToken: next-please'); + }); + + it('text output reads "No projects." when items is empty and nextToken is null', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(out.join('\n')).toBe('No projects.'); + }); + + it('text output reads "No projects on this page." with nextToken when filtered out', async () => { + const { credentialsPath } = makeCreds(); + // Empty page that still has a nextToken — happens when a server-side + // filter excludes everything in the current window. + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: 'still-more' } })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toContain('No projects on this page.'); + expect(block).toContain('nextToken: still-more'); + }); + + it('--debug emits HTTP events to stderr', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); + + const stderr: string[] = []; + await runList( + { profile: 'default', output: 'json', debug: true, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => stderr.push(line) }, + ); + + // Format is now "[debug ] {...}" + expect(stderr.some(line => line.startsWith('[debug '))).toBe(true); + expect(stderr.some(line => line.includes('"kind":"request"'))).toBe(true); + }); +}); + +describe('createProjectCommand --page-size option parser', () => { + it('rejects non-numeric --page-size values via commander', async () => { + const project = createProjectCommand(); + const list = project.commands.find(c => c.name() === 'list')!; + project.exitOverride(); + list.exitOverride(); + + await expect( + project.parseAsync(['list', '--page-size', 'abc'], { from: 'user' }), + ).rejects.toThrow(); + }); + + it('rejects --page-size=0 via commander', async () => { + const project = createProjectCommand(); + const list = project.commands.find(c => c.name() === 'list')!; + project.exitOverride(); + list.exitOverride(); + + await expect( + project.parseAsync(['list', '--page-size', '0'], { from: 'user' }), + ).rejects.toThrow(); + }); + + it('forwards a server VALIDATION_ERROR envelope as ApiError exit 5', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 400, + body: { + error: { + code: 'VALIDATION_ERROR', + message: 'bad cursor', + nextAction: 'pass nextToken from a previous response', + requestId: 'req_test', + details: { field: 'cursor' }, + }, + }, + })); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, startingToken: 'bogus' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toBeInstanceOf(ApiError); + }); +}); + +describe('runGet', () => { + it('GETs /projects/{id} and prints the §6.1 fields in text mode', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: PROJECT_FIXTURE }; + }); + + const out: string[] = []; + const project = await runGet( + { profile: 'default', output: 'text', debug: false, projectId: 'project_b3c91efa' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(seen[0]).toContain('/projects/project_b3c91efa'); + expect(project.id).toBe('project_b3c91efa'); + const block = out.join('\n'); + expect(block).toContain('id: project_b3c91efa'); + expect(block).toContain('type: frontend'); + expect(block).toContain('createdFrom: portal'); + }); + + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Resource not found.', + nextAction: 'Check the id with `testsprite project list`.', + requestId: 'req_test', + details: { resource: 'project', id: 'project_missing' }, + }, + }, + })); + + await expect( + runGet( + { profile: 'default', output: 'json', debug: false, projectId: 'project_missing' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND', exitCode: 4 }); + }); + + it('URL-encodes the project id (defense against `/` or `?` in ids)', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: PROJECT_FIXTURE }; + }); + + await runGet( + { profile: 'default', output: 'json', debug: false, projectId: 'odd/id?weird' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + + expect(seen[0]).toContain('odd%2Fid%3Fweird'); + }); +}); + +// --------------------------------------------------------------------------- +// P6 — project create +// --------------------------------------------------------------------------- + +describe('runCreate', () => { + it('P6 FE happy — POSTs /projects with type=frontend + name + idempotency header', async () => { + const { credentialsPath } = makeCreds(); + const sentBodies: unknown[] = []; + const sentHeaders: Record[] = []; + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_new', + type: 'frontend', + name: 'My FE App', + }; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + const body = init.body ? (JSON.parse(init.body as string) as unknown) : undefined; + if (body) sentBodies.push(body); + const h = new Headers(init.headers); + const entry: Record = {}; + h.forEach((v, k) => { + entry[k] = v; + }); + sentHeaders.push(entry); + return new Response(JSON.stringify(createdProject), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const stderrLines: string[] = []; + const out: string[] = []; + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'My FE App', + targetUrl: 'https://example.com', + idempotencyKey: 'idem-fe-001', + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + + expect(result.id).toBe('proj_new'); + expect(result.type).toBe('frontend'); + // Verify body + const body = sentBodies[0] as Record; + expect(body.type).toBe('frontend'); + expect(body.name).toBe('My FE App'); + // Verify idempotency header + const h = sentHeaders[0]!; + expect(h['idempotency-key']).toBe('idem-fe-001'); + // User-supplied idempotency key is NOT echoed to stderr (P2-6: only + // auto-generated keys are surfaced at --verbose/--debug/json mode). + expect(stderrLines.some(l => l.includes('idem-fe-001'))).toBe(false); + }); + + it('P6 BE happy — POSTs /projects with type=backend', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_be', + type: 'backend', + name: 'My BE API', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'backend', + name: 'My BE API', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.id).toBe('proj_be'); + expect(result.type).toBe('backend'); + }); + + it('P6 — dry-run returns canned shape without hitting the network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network in dry-run'); + }); + const out: string[] = []; + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + type: 'frontend', + name: 'DryRun Project', + targetUrl: 'https://example.com', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => out.push(line), + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.type).toBe('frontend'); + expect(result.name).toBe('DryRun Project'); + }); + + it('P6 — renders text mode with §6.1 field labels', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_text', + name: 'Text Mode', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + type: 'frontend', + name: 'Text Mode', + targetUrl: 'https://example.com', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + const block = out.join('\n'); + expect(block).toContain('id:'); + expect(block).toContain('name:'); + expect(block).toContain('type:'); + }); + + it('P6 — frontend without --url rejects with VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'No URL Project', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// P7 — project update +// --------------------------------------------------------------------------- + +describe('runUpdate', () => { + it('P7 happy — PATCHes /projects/{id} with the updated fields', async () => { + const { credentialsPath } = makeCreds(); + const updateResponse: CliUpdateProjectResponse = { + id: 'proj_abc', + updatedFields: ['name'], + updatedAt: '2026-05-16T10:00:00.000Z', + }; + const sentBodies: unknown[] = []; + const sentMethods: string[] = []; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + sentMethods.push(init.method ?? 'GET'); + if (init.body) sentBodies.push(JSON.parse(init.body as string) as unknown); + return new Response(JSON.stringify(updateResponse), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const stderrLines: string[] = []; + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + name: 'New Name', + idempotencyKey: 'idem-upd-001', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + expect(result.id).toBe('proj_abc'); + expect(result.updatedFields).toEqual(['name']); + expect(sentMethods[0]).toBe('PATCH'); + const body = sentBodies[0] as Record; + expect(body.name).toBe('New Name'); + // User-supplied idempotency key is NOT echoed to stderr (P2-6). + expect(stderrLines.some(l => l.includes('idem-upd-001'))).toBe(false); + }); + + it('P7 — exits 5 VALIDATION_ERROR when no mutable flag is supplied', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + // no mutable fields + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('P7 — dry-run returns canned shape without network call', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + name: 'Dry Name', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.id).toBe('proj_dry'); + expect(result.updatedFields).toContain('name'); + }); + + it('P7 — renders text mode with updatedFields and updatedAt', async () => { + const { credentialsPath } = makeCreds(); + const updateResponse: CliUpdateProjectResponse = { + id: 'proj_text', + updatedFields: ['name', 'description'], + updatedAt: '2026-05-16T10:00:00.000Z', + }; + const fetchImpl = makeFetch(() => ({ body: updateResponse })); + const out: string[] = []; + await runUpdate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_text', + name: 'New Name', + description: 'New desc', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + const block = out.join('\n'); + expect(block).toContain('updatedFields:'); + expect(block).toContain('updatedAt:'); + }); + + it('Fix 1 — response without updatedFields renders without throwing (text mode)', async () => { + // Backend may omit updatedFields; the CLI must not crash with + // "Cannot read properties of undefined (reading 'join')". + const { credentialsPath } = makeCreds(); + const responseWithoutField: Omit & { + updatedFields?: string[]; + } = { + id: 'proj_no_fields', + updatedAt: '2026-06-07T00:00:00.000Z', + // updatedFields intentionally absent + }; + const fetchImpl = makeFetch(() => ({ body: responseWithoutField })); + + const out: string[] = []; + const result = await runUpdate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_no_fields', + name: 'Changed Name', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + + expect(result.id).toBe('proj_no_fields'); + // Must not throw; text output should contain a graceful "(none)" placeholder. + const block = out.join('\n'); + expect(block).toContain('updatedFields: (none)'); + }); + + it('Fix 1 — response without updatedFields renders gracefully in json mode', async () => { + const { credentialsPath } = makeCreds(); + const responseWithoutField = { + id: 'proj_json_no_fields', + updatedAt: '2026-06-07T00:00:00.000Z', + }; + const fetchImpl = makeFetch(() => ({ body: responseWithoutField })); + + const out: string[] = []; + // Must not throw in json mode either. + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_json_no_fields', + name: 'Changed Name', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + + expect(result.id).toBe('proj_json_no_fields'); + expect(result.updatedFields).toBeUndefined(); + }); +}); diff --git a/src/commands/project.ts b/src/commands/project.ts new file mode 100644 index 0000000..277f3ca --- /dev/null +++ b/src/commands/project.ts @@ -0,0 +1,608 @@ +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { Command } from 'commander'; +import { + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { ApiError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import type { HttpClient } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { assertNotLocal } from '../lib/target-url.js'; +import { assertIdempotencyKey } from '../lib/validate.js'; +import { + fetchSinglePage, + paginate, + validatePaginationFlags, + type Page, + type PaginationFlags, +} from '../lib/pagination.js'; + +export interface CliProject { + id: string; + name: string; + type: 'frontend' | 'backend'; + createdFrom: 'portal' | 'mcp' | 'cli'; + createdAt: string; + updatedAt: string; +} + +export interface ProjectDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; +} + +type CommonOptions = FactoryCommonOptions; + +interface ListOptions extends CommonOptions { + pageSize?: number; + startingToken?: string; + maxItems?: number; +} + +export async function runList( + opts: ListOptions, + deps: ProjectDeps = {}, +): Promise> { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + const paginationFlags: PaginationFlags = validatePaginationFlags({ + pageSize: opts.pageSize, + startingToken: opts.startingToken, + maxItems: opts.maxItems, + }); + + // When the user explicitly passed a page-size flag and did NOT ask + // for --max-items, treat that as a "give me one page and the cursor" + // request — same shape AWS CLI ships. Otherwise auto-page. + const useSinglePage = opts.pageSize !== undefined && opts.maxItems === undefined; + + let page: Page; + if (useSinglePage) { + page = await fetchSinglePage( + client, + '/projects', + paginationFlags.pageSize!, + opts.startingToken, + ); + } else { + page = await paginate( + async ({ pageSize, cursor }) => + client.get>('/projects', { + query: { pageSize, cursor }, + }), + paginationFlags, + ); + } + + out.print(page, data => { + const p = data as Page; + return renderProjectListText(p); + }); + return page; +} + +interface GetOptions extends CommonOptions { + projectId: string; +} + +export async function runGet(opts: GetOptions, deps: ProjectDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + const project = await client.get(`/projects/${encodeURIComponent(opts.projectId)}`); + out.print(project, data => renderProjectText(data as CliProject)); + return project; +} + +// --------------------------------------------------------------------------- +// project create +// --------------------------------------------------------------------------- + +export interface CliCreateProjectRequest { + type: 'frontend' | 'backend'; + name: string; + targetUrl?: string; + description?: string; + username?: string; + password?: string; + instruction?: string; +} + +export type CliCreateProjectResponse = CliProject; + +interface CreateOptions extends CommonOptions { + type: 'frontend' | 'backend'; + name: string; + targetUrl?: string; + description?: string; + username?: string; + password?: string; + passwordFile?: string; + instruction?: string; + idempotencyKey?: string; +} + +export async function runCreate( + opts: CreateOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // P1-2: validate idempotency key before sending as an HTTP header. + // Non-ASCII chars cause a ByteString TypeError at the transport layer + // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. + assertIdempotencyKey(opts.idempotencyKey); + + // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.length > 200) { + throw localValidationError('--name must be at most 200 characters'); + } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('--description must be at most 2000 characters'); + } + + // P2-7: guard --url against localhost/RFC1918/non-http(s) (same rules as + // `test create --target-url`). Applies to both FE (required) and BE (optional). + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + if (opts.type === 'frontend' && !opts.targetUrl) { + throw localValidationError('--url is required for --type frontend'); + } + + if (opts.dryRun) { + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-create-${randomUUID()}`; + // P2-6: gate idempotency-key output behind --verbose/--debug/json (matches + // test create convention). Suppress in plain text interactive mode to reduce + // noise; still available for automation and retry flows. + if ( + opts.idempotencyKey === undefined && + (opts.output === 'json' || opts.verbose || opts.debug) + ) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + const sample: CliCreateProjectResponse = { + id: 'p_dryrun_2026', + type: opts.type, + name: opts.name, + targetUrl: opts.targetUrl ?? '', + createdFrom: 'cli', + createdAt: '2026-05-16T00:00:00.000Z', + updatedAt: '2026-05-16T00:00:00.000Z', + } as unknown as CliCreateProjectResponse; + out.print(sample, data => renderProjectText(data as CliProject)); + return sample; + } + + // Resolve password: flag > file > none + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-create-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const body: CliCreateProjectRequest = { + type: opts.type, + name: opts.name, + ...(opts.targetUrl !== undefined ? { targetUrl: opts.targetUrl } : {}), + ...(opts.description !== undefined ? { description: opts.description } : {}), + ...(opts.username !== undefined ? { username: opts.username } : {}), + ...(password !== undefined ? { password } : {}), + ...(opts.instruction !== undefined ? { instruction: opts.instruction } : {}), + }; + + const client = makeClient(opts, deps); + const created = await client.post('/projects', { + body, + headers: { 'idempotency-key': idempotencyKey }, + }); + + out.print(created, data => renderProjectText(data as CliProject)); + return created; +} + +// --------------------------------------------------------------------------- +// project update +// --------------------------------------------------------------------------- + +export interface CliUpdateProjectResponse { + id: string; + /** Backend may omit this field; treat absence as no specific fields reported. */ + updatedFields?: string[]; + updatedAt: string; +} + +interface UpdateOptions extends CommonOptions { + projectId: string; + name?: string; + targetUrl?: string; + username?: string; + password?: string; + passwordFile?: string; + description?: string; + instruction?: string; + idempotencyKey?: string; +} + +export async function runUpdate( + opts: UpdateOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // P1-2: validate idempotency key before sending as an HTTP header. + assertIdempotencyKey(opts.idempotencyKey); + + // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.length > 200) { + throw localValidationError('--name must be at most 200 characters'); + } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('--description must be at most 2000 characters'); + } + + // Resolve password + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + + // P2-7: guard --url against localhost/RFC1918/non-http(s). + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + const mutableFields: Record = { + name: opts.name, + targetUrl: opts.targetUrl, + username: opts.username, + password, + description: opts.description, + instruction: opts.instruction, + }; + const presentFields = Object.entries(mutableFields).filter(([, v]) => v !== undefined); + if (presentFields.length === 0) { + throw localValidationError( + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', + ); + } + + if (opts.dryRun) { + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; + if ( + opts.idempotencyKey === undefined && + (opts.output === 'json' || opts.verbose || opts.debug) + ) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + const sample: CliUpdateProjectResponse = { + id: opts.projectId, + updatedFields: presentFields.map(([k]) => k), + updatedAt: '2026-05-16T00:00:00.000Z', + }; + out.print(sample, data => renderUpdateText(data as CliUpdateProjectResponse)); + return sample; + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const body = Object.fromEntries(presentFields) as Record; + const client = makeClient(opts, deps); + const updated = await client.patch( + `/projects/${encodeURIComponent(opts.projectId)}`, + { + body, + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + + out.print(updated, data => renderUpdateText(data as CliUpdateProjectResponse)); + return updated; +} + +export function createProjectCommand(deps: ProjectDeps = {}): Command { + const project = new Command('project').description('Manage TestSprite projects'); + + project + .command('list') + .description( + 'List projects visible to the API key\n' + + '\nExit codes:\n' + + ' 0 success\n' + + ' 3 auth error\n' + + ' 5 validation error (e.g., bad --page-size)\n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command', + ) + .option('--page-size ', 'service page-size hint (1-100, default 25)') + .option('--starting-token ', 'opaque cursor from a previous list response') + .option('--max-items ', 'stop after this many items across auto-paged pages') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ListFlagOpts, command: Command) => { + // Don't parse numeric flags via Commander — its parser throws a + // plain `Error`, which `index.ts` maps to exit code 1. Local + // validation lives in `runList → validatePaginationFlags`, which + // raises a typed `ApiError(VALIDATION_ERROR)` and surfaces with + // the contract-mandated exit code 5. + await runList( + { + ...resolveCommonOptions(command), + pageSize: parseFlag(cmdOpts.pageSize, 'page-size'), + startingToken: cmdOpts.startingToken, + maxItems: parseFlag(cmdOpts.maxItems, 'max-items'), + }, + deps, + ); + }); + + project + .command('get ') + .description('Get a project by id') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, _cmdOpts, command: Command) => { + await runGet({ ...resolveCommonOptions(command), projectId }, deps); + }); + + project + .command('create') + .description('Create a new project') + .option('--type ', 'project type (required)') + .option('--name ', 'project name (required)') + .option('--url ', 'target URL (required for frontend)') + .option('--description ', 'optional human description') + .option('--username ', 'optional auth username') + .option('--password ', 'optional auth password (use --password-file for non-interactive)') + .option('--password-file ', 'read password from file instead of inline flag') + .option('--instruction ', 'optional FE plan-gen instruction hint') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: CreateFlagOpts, command: Command) => { + if (!cmdOpts.type) throw localValidationError('--type is required (frontend|backend)'); + if (!cmdOpts.name) throw localValidationError('--name is required'); + const type = cmdOpts.type as 'frontend' | 'backend'; + if (type !== 'frontend' && type !== 'backend') { + throw localValidationError('--type must be frontend or backend'); + } + if (type === 'frontend' && !cmdOpts.url) { + throw localValidationError('--url is required for --type frontend'); + } + await runCreate( + { + ...resolveCommonOptions(command), + type, + name: cmdOpts.name, + targetUrl: cmdOpts.url, + description: cmdOpts.description, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + instruction: cmdOpts.instruction, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('update ') + .description('Update project metadata') + .option('--name ', 'new project name') + .option('--url ', 'new target URL') + .option('--username ', 'new auth username') + .option('--password ', 'new auth password') + .option('--password-file ', 'read new password from file') + .option('--description ', 'new description') + .option('--instruction ', 'new FE plan-gen instruction hint') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: UpdateFlagOpts, command: Command) => { + await runUpdate( + { + ...resolveCommonOptions(command), + projectId, + name: cmdOpts.name, + targetUrl: cmdOpts.url, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + description: cmdOpts.description, + instruction: cmdOpts.instruction, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + return project; +} + +interface ListFlagOpts { + pageSize?: string; + startingToken?: string; + maxItems?: string; +} + +interface CreateFlagOpts { + type?: string; + name?: string; + url?: string; + description?: string; + username?: string; + password?: string; + passwordFile?: string; + instruction?: string; + idempotencyKey?: string; +} + +interface UpdateFlagOpts { + name?: string; + url?: string; + username?: string; + password?: string; + passwordFile?: string; + description?: string; + instruction?: string; + idempotencyKey?: string; +} + +function parseFlag(raw: string | undefined, flagName: string): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n)) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Invalid request.', + nextAction: `Flag \`--${flagName}\` is invalid: must be an integer.`, + requestId: 'local', + details: { field: flagName, reason: 'must be an integer' }, + }, + }); + } + return n; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + // P2-8: validate --output before allowing silent fallback to 'text'. + const rawOutput = globals.output; + if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { + throw localValidationError('--output must be one of: json, text'); + } + return { + profile: globals.profile ?? 'default', + output: (globals.output as OutputMode | undefined) ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * Returns `undefined` when the flag was not supplied (factory falls back to + * the env var / default). Silently clamps out-of-range values. + */ +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return undefined; + return Math.round(n * 1000); // seconds → milliseconds +} + +function makeClient(opts: CommonOptions, deps: ProjectDeps): HttpClient { + return makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); +} + +function makeOutput(mode: OutputMode, deps: ProjectDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} + +function renderProjectListText(page: Page): string { + if (page.items.length === 0) { + return page.nextToken + ? `No projects on this page.\nnextToken: ${page.nextToken}` + : 'No projects.'; + } + // Compact, AWS-CLI-grade columnar output. Column widths are computed + // per-call so a single absurdly long project name doesn't push the + // whole table off-screen. + const idWidth = Math.max(2, ...page.items.map(p => p.id.length)); + const nameWidth = Math.max(4, ...page.items.map(p => p.name.length)); + const typeWidth = 8; + const fromWidth = 6; + + const header = + pad('ID', idWidth) + + ' ' + + pad('NAME', nameWidth) + + ' ' + + pad('TYPE', typeWidth) + + ' ' + + pad('FROM', fromWidth) + + ' ' + + 'CREATED'; + + const rows = page.items.map( + p => + pad(p.id, idWidth) + + ' ' + + pad(p.name, nameWidth) + + ' ' + + pad(p.type, typeWidth) + + ' ' + + pad(p.createdFrom, fromWidth) + + ' ' + + p.createdAt, + ); + + const lines = [header, ...rows]; + if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); + return lines.join('\n'); +} + +function renderProjectText(p: CliProject): string { + return [ + `id: ${p.id}`, + `name: ${p.name}`, + `type: ${p.type}`, + `createdFrom: ${p.createdFrom}`, + `createdAt: ${p.createdAt}`, + `updatedAt: ${p.updatedAt}`, + ].join('\n'); +} + +function pad(s: string, width: number): string { + if (s.length >= width) return s; + return s + ' '.repeat(width - s.length); +} + +function renderUpdateText(r: CliUpdateProjectResponse): string { + return [ + `id: ${r.id}`, + `updatedFields: ${r.updatedFields?.join(', ') ?? '(none)'}`, + `updatedAt: ${r.updatedAt}`, + ].join('\n'); +} + +function localValidationError(message: string): ApiError { + return ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Invalid request.', + nextAction: message, + requestId: 'local', + details: { reason: 'missing_required_flag' }, + }, + }); +} diff --git a/src/commands/test.artifact.spec.ts b/src/commands/test.artifact.spec.ts new file mode 100644 index 0000000..07cd7a5 --- /dev/null +++ b/src/commands/test.artifact.spec.ts @@ -0,0 +1,874 @@ +/** + * M3.3 piece-4 — `test artifact get ` unit tests. + * + * Covers every flag combination, 404 reason, 409 retry, meta.runId + * mismatch, dry-run, and the §7 on-disk layout. + * + * The M2 non-regression test at the bottom confirms that `writeBundle` + * called from `runFailureGet` (M2 path) still exits 0 with an M2-shaped + * context (meta.runId = null), verifying the `requireRunId` option is + * strictly opt-in. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'; +import type { ApiError } from '../lib/errors.js'; +import type { CliFailureContext, CliLatestResult, CliTestStep } from './test.js'; +import { + assertOutDirParentExists, + createTestArtifactCommand, + runArtifactGet, + runFailureGet, +} from './test.js'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; + +// --------------------------------------------------------------------------- +// Test fixtures +// --------------------------------------------------------------------------- + +const SAMPLE_RUN_ID = 'run_piece4_test'; +const SAMPLE_SNAPSHOT_ID = 'snap_2026_05_15_piece4'; + +const SAMPLE_STEPS: CliTestStep[] = [ + { + testId: 'test_fe', + stepIndex: 4, + action: 'click', + description: 'Click cart', + status: 'passed', + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/04.html', + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: '2026-05-15T10:00:00.000Z', + updatedAt: '2026-05-15T10:00:01.000Z', + outcomeContributesToFailure: false, + }, + { + testId: 'test_fe', + stepIndex: 5, + action: 'click', + description: 'Click submit', + status: 'failed', + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/05.html', + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: '2026-05-15T10:00:01.000Z', + updatedAt: '2026-05-15T10:00:02.000Z', + outcomeContributesToFailure: true, + }, + { + testId: 'test_fe', + stepIndex: 6, + action: 'expect', + description: 'Expect heading', + status: null, + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/06.html', + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-15T10:00:03.000Z', + outcomeContributesToFailure: false, + }, +]; + +function makeArtifactContext(overrides: Partial = {}): CliFailureContext { + const result: CliLatestResult = { + testId: 'test_fe', + status: 'failed', + startedAt: '2026-05-15T10:00:00.000Z', + finishedAt: '2026-05-15T10:01:00.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: SAMPLE_SNAPSHOT_ID, + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + targetUrl: 'https://staging.example.com/checkout', + failedStepIndex: 5, + failureKind: 'assertion', + summary: { passed: 4, failed: 1, skipped: 0 }, + ...overrides.result, + }; + return { + snapshotId: SAMPLE_SNAPSHOT_ID, + testId: 'test_fe', + projectId: 'project_alice', + result, + steps: SAMPLE_STEPS, + code: { + testId: 'test_fe', + language: 'typescript', + framework: 'playwright', + code: "import { test } from '@playwright/test';\ntest('checkout', () => {});\n", + codeVersion: 'v3', + etag: null, + }, + failure: { + rootCauseHypothesis: 'Submit button is disabled.', + recommendedFixTarget: { + kind: 'code', + reference: 'src/Checkout.tsx:12', + rationale: 'isFormValid', + }, + evidence: [ + { + kind: 'snapshot', + stepIndex: 5, + url: 'https://signed.example.com/ev/05.html', + summary: 'Step 5 failed. Submit button disabled.', + }, + ], + }, + ...overrides, + }; +} + +type FetchInput = Parameters[0]; + +/** + * Build a mock fetch that handles: + * - Backend API calls (returning JSON via the handler) + * - Presigned S3 URL downloads (returning a small dummy response) + */ +function makeFetch( + apiHandler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + // Presigned URL downloads (snapshot HTML files) + if (url.includes('signed.example.com')) { + return new Response('snapshot', { + status: 200, + headers: { 'content-type': 'text/html' }, + }); + } + + const { status = 200, body } = apiHandler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json', 'x-request-id': 'req_piece4_test' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:14400', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-')); + const credPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { mode: 0o600 }); + return { credentialsPath: credPath }; +} + +// --------------------------------------------------------------------------- +// assertOutDirParentExists +// --------------------------------------------------------------------------- + +describe('assertOutDirParentExists', () => { + it('resolves successfully when parent dir exists', async () => { + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-parent-')); + const target = join(tmpDir, 'bundle'); + await expect(assertOutDirParentExists(target)).resolves.toBeUndefined(); + }); + + it('throws VALIDATION_ERROR when parent does not exist', async () => { + const target = join(tmpdir(), 'does-not-exist-xyz', 'bundle'); + await expect(assertOutDirParentExists(target)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ + reason: expect.stringContaining('parent directory does not exist'), + }), + }); + }); + + it('throws VALIDATION_ERROR when parent is not a directory (is a file)', async () => { + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-parent-')); + const fileAsParent = join(tmpDir, 'file.txt'); + await writeFile(fileAsParent, 'content'); + const target = join(fileAsParent, 'bundle'); + await expect(assertOutDirParentExists(target)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ + reason: expect.stringContaining('parent path is not a directory'), + }), + }); + }); + + it('throws VALIDATION_ERROR when --out points to an existing file', async () => { + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-parent-')); + const existingFile = join(tmpDir, 'bundle.txt'); + await writeFile(existingFile, 'content'); + await expect(assertOutDirParentExists(existingFile)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ + reason: expect.stringContaining('must point to a directory'), + }), + }); + }); + + it('resolves successfully when --out points to an existing directory', async () => { + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-parent-')); + const existingDir = join(tmpDir, 'bundle'); + await mkdir(existingDir); + await expect(assertOutDirParentExists(existingDir)).resolves.toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// runArtifactGet +// --------------------------------------------------------------------------- + +describe('runArtifactGet', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + // ---- Happy path: --out specified, bundle written ---- + + it('happy path: writes bundle to --out dir and returns context + bundle', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeArtifactContext(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + const outDir = join(tmpDir, 'bundle'); + + const fetchImpl = makeFetch(url => { + expect(url).toContain(`/runs/${SAMPLE_RUN_ID}/failure`); + return { body: ctx }; + }); + + const out: string[] = []; + const result = await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: outDir, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(result.context).toEqual(ctx); + expect(result.bundle).toBeDefined(); + expect(result.bundle!.dir).toBe(outDir); + + // meta.json must exist as the atomicity signal + const meta = JSON.parse(readFileSync(join(outDir, 'meta.json'), 'utf8')); + expect(meta.runIdIfAvailable).toBe(SAMPLE_RUN_ID); + expect(meta.testId).toBe('test_fe'); + + // JSON envelope on stdout + const envelope = JSON.parse(out[0]!); + expect(envelope.out).toBe(outDir); + expect(envelope.snapshotId).toBe(SAMPLE_SNAPSHOT_ID); + expect(envelope.meta.runId).toBe(SAMPLE_RUN_ID); + }); + + // ---- Default dir: .testsprite/runs// ---- + + it('uses default dir .testsprite/runs// when --out is absent', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeArtifactContext(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-cwd-')); + + // Override process.cwd so default dir resolves under tmpDir + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(tmpDir); + try { + const fetchImpl = makeFetch(() => ({ body: ctx })); + const out: string[] = []; + + const result = await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const expectedDir = join(tmpDir, '.testsprite', 'runs', SAMPLE_RUN_ID); + expect(result.bundle!.dir).toBe(expectedDir); + expect(existsSync(join(expectedDir, 'meta.json'))).toBe(true); + } finally { + cwdSpy.mockRestore(); + } + }); + + // ---- --failed-only passed through to writeBundle ---- + + it('passes --failed-only through to writeBundle (steps filtered to failed ± 1)', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeArtifactContext(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + const outDir = join(tmpDir, 'bundle'); + + const fetchImpl = makeFetch(() => ({ body: ctx })); + + const result = await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: outDir, + failedOnly: true, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + // With failed step at index 5 and --failed-only, steps 4, 5, 6 should be written + // Verify the bundle was written + expect(existsSync(join(outDir, 'meta.json'))).toBe(true); + expect(result.bundle).toBeDefined(); + }); + + // ---- --output json without --out: writes to default dir, prints envelope ---- + + it('--output json without --out: writes to default .testsprite/runs// and prints JSON envelope', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeArtifactContext(); + const fetchImpl = makeFetch(() => ({ body: ctx })); + + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-cwd2-')); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(tmpDir); + try { + const out: string[] = []; + const result = await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(result.context).toEqual(ctx); + expect(result.bundle).toBeDefined(); + // stdout has the JSON envelope (dir + snapshotId + meta) + const parsed = JSON.parse(out[0]!); + expect(parsed).toHaveProperty('out'); + expect(parsed).toHaveProperty('snapshotId', SAMPLE_SNAPSHOT_ID); + expect(parsed).toHaveProperty('meta'); + expect(parsed.meta.runId).toBe(SAMPLE_RUN_ID); + } finally { + cwdSpy.mockRestore(); + } + }); + + // ---- --output text mode ---- + + it('text mode (no --output json, with --out) prints human summary', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeArtifactContext(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + const outDir = join(tmpDir, 'bundle'); + + const fetchImpl = makeFetch(() => ({ body: ctx })); + const out: string[] = []; + + await runArtifactGet( + { + profile: 'default', + output: 'text', + debug: false, + runId: SAMPLE_RUN_ID, + out: outDir, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const text = out.join('\n'); + expect(text).toContain('Bundle written to'); + expect(text).toContain(SAMPLE_RUN_ID); + expect(text).toContain(SAMPLE_SNAPSHOT_ID); + }); + + // ---- --dry-run: no network, no disk write ---- + + it('--dry-run: exits without making any network call or writing any file', async () => { + const { credentialsPath } = makeCreds(); + let networkCalled = false; + const fetchImpl = makeFetch(() => { + networkCalled = true; + return { body: {} }; + }); + + const out: string[] = []; + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + + await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: join(tmpDir, 'bundle'), + failedOnly: false, + dryRun: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(networkCalled).toBe(false); + expect(existsSync(join(tmpDir, 'bundle', 'meta.json'))).toBe(false); + + // JSON output matches the real success schema: { out, snapshotId, meta } + const envelope = JSON.parse(out[0]!); + expect(envelope).toHaveProperty('out'); + expect(envelope).toHaveProperty('snapshotId'); + expect(envelope).toHaveProperty('meta'); + // `out` is the resolved --out path (explicit in this test) + expect(envelope.out).toBeTruthy(); + }); + + it('--dry-run text mode: prints human-readable envelope', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: {} })); + const out: string[] = []; + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + + await runArtifactGet( + { + profile: 'default', + output: 'text', + debug: false, + runId: SAMPLE_RUN_ID, + out: join(tmpDir, 'bundle'), + failedOnly: false, + dryRun: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const text = out.join('\n'); + expect(text).toContain('[dry-run]'); + expect(text).toContain('GET'); + }); + + // ---- --out parent missing → exit 5 ---- + + it('--out with missing parent directory → VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: {} })); + + const missingParent = join(tmpdir(), 'definitely-does-not-exist-xyz', 'bundle'); + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: missingParent, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + // ---- --out is a file → exit 5 ---- + + it('--out pointing to an existing file → VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + const existingFile = join(tmpDir, 'bundle.txt'); + await writeFile(existingFile, 'content'); + + const fetchImpl = makeFetch(() => ({ body: {} })); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: existingFile, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + // ---- 404 not_found → exit 4 ---- + + it('404 not_found → propagates as NOT_FOUND ApiError (exit 4)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Run not found.', + nextAction: 'Check the id.', + requestId: 'req_404', + details: { reason: 'not_found' }, + }, + }, + })); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND' }); + }); + + // ---- 404 run_not_ready → exit 4 with nextAction ---- + + it('404 run_not_ready → NOT_FOUND with run_not_ready reason', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Run not ready.', + nextAction: `Wait for the run to finish: testsprite test wait ${SAMPLE_RUN_ID}`, + requestId: 'req_notready', + details: { reason: 'run_not_ready' }, + }, + }, + })); + + let error: ApiError | undefined; + try { + await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + } catch (e) { + error = e as ApiError; + } + + expect(error).toBeDefined(); + expect(error!.code).toBe('NOT_FOUND'); + expect(error!.nextAction).toContain('testsprite test wait'); + }); + + // ---- 404 no_failing_run → exit 4 ---- + + it('404 no_failing_run → NOT_FOUND with no_failing_run reason', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Run passed.', + nextAction: 'Run passed — there is no failure bundle to download.', + requestId: 'req_nofail', + details: { reason: 'no_failing_run' }, + }, + }, + })); + + let error: ApiError | undefined; + try { + await runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + } catch (e) { + error = e as ApiError; + } + + expect(error).toBeDefined(); + expect(error!.code).toBe('NOT_FOUND'); + expect(error!.nextAction).toContain('no failure bundle'); + }); + + // ---- 404 cancelled_no_artifacts → exit 4 ---- + + it('404 cancelled_no_artifacts → NOT_FOUND error', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Run cancelled.', + nextAction: '', + requestId: 'req_cancelled', + details: { reason: 'cancelled_no_artifacts' }, + }, + }, + })); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND' }); + }); + + // ---- meta.runId mismatch → exit 5 ---- + + it('meta.runId mismatch (backend bug) → VALIDATION_ERROR exit 5, no disk write', async () => { + const { credentialsPath } = makeCreds(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-')); + const outDir = join(tmpDir, 'bundle'); + + // Backend returns a context for a DIFFERENT runId + const wrongRunId = 'run_WRONG_123'; + const ctx = makeArtifactContext({ + result: { + ...makeArtifactContext().result, + runIdIfAvailable: wrongRunId, + snapshotId: SAMPLE_SNAPSHOT_ID, + }, + }); + + const fetchImpl = makeFetch(() => ({ body: ctx })); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + out: outDir, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + // Item-10 fix: assertContextIntegrity (requireRunId path) catches step + // runId mismatch before the caller's explicit result.runId check. + details: expect.objectContaining({ reason: 'run_id_mismatch' }), + }); + + // No bundle written + expect(existsSync(join(outDir, 'meta.json'))).toBe(false); + }); + + // ---- 409 CONFLICT: 1× retry → exit 6 on second 409 ---- + + it('409 CONFLICT retries once, then propagates CONFLICT on second 409', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl = makeFetch(() => { + callCount++; + return { + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'Snapshot in flight.', + nextAction: 'Retry in a few seconds.', + requestId: 'req_conflict', + details: { reason: 'snapshot_in_flight' }, + }, + }, + }; + }); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: SAMPLE_RUN_ID, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'CONFLICT' }); + + // HttpClient retries once on CONFLICT (total 2 attempts per the CLI error spec §7) + expect(callCount).toBe(2); + }, 10000); + + // ---- createTestArtifactCommand: exposes get subcommand ---- + + it('createTestArtifactCommand exposes artifact get ', () => { + const cmd = createTestArtifactCommand({}); + expect(cmd.name()).toBe('artifact'); + const sub = cmd.commands.find(c => c.name() === 'get'); + expect(sub).toBeDefined(); + const flagNames = sub!.options.map(o => o.long); + expect(flagNames).toContain('--out'); + expect(flagNames).toContain('--failed-only'); + // --dry-run is a global flag (not a per-subcommand option); it is + // inherited via optsWithGlobals() and must NOT appear in the local + // options list (shadowing the global flag was the Item-9 bug). + expect(flagNames).not.toContain('--dry-run'); + }); +}); + +// --------------------------------------------------------------------------- +// M2 non-regression: runFailureGet still works after requireRunId was added +// --------------------------------------------------------------------------- + +describe('runFailureGet M2 non-regression (requireRunId is opt-in)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('M2 path (meta.runId = null) succeeds without requireRunId', async () => { + const { credentialsPath } = makeCreds(); + const tmpDir = await mkdtemp(join(tmpdir(), 'cli-p4-m2reg-')); + const outDir = join(tmpDir, 'bundle'); + + // M2-shaped context: runIdIfAvailable is null + const m2Ctx: CliFailureContext = { + snapshotId: 'snap_m2_nullrunid', + testId: 'test_m2', + projectId: 'project_m2', + result: { + testId: 'test_m2', + status: 'failed', + startedAt: null, + finishedAt: '2026-05-15T09:00:00.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: 'snap_m2_nullrunid', + runIdIfAvailable: null, // M2 shape: null + codeVersion: 'v1', + targetUrl: 'https://example.com', + failedStepIndex: 1, + failureKind: 'assertion', + summary: { passed: 0, failed: 1, skipped: 0 }, + }, + steps: [ + { + testId: 'test_m2', + stepIndex: 1, + action: 'click', + description: 'click', + status: 'failed', + screenshotUrl: null, + htmlSnapshotUrl: null, + runIdIfAvailable: null, // M2 shape: null + codeVersion: 'v1', + capturedAt: null, + updatedAt: '2026-05-15T09:00:00.000Z', + outcomeContributesToFailure: true, + }, + ], + code: { + testId: 'test_m2', + language: 'typescript', + framework: 'playwright', + code: 'test("m2", () => {});', + codeVersion: 'v1', + etag: null, + }, + failure: { + rootCauseHypothesis: null, + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: null }, + evidence: [ + { + kind: 'snapshot', + stepIndex: 1, + url: 'https://m2.example.com/snap.html', + summary: 'failed step', + }, + ], + }, + }; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('m2.example.com')) { + return new Response('', { status: 200, headers: { 'content-type': 'text/html' } }); + } + return new Response(JSON.stringify(m2Ctx), { + status: 200, + headers: { 'content-type': 'application/json', 'x-request-id': 'req_m2' }, + }); + }) as typeof globalThis.fetch; + + // M2 path uses testId, not runId. Should succeed with exit 0. + const result = await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_m2', + out: outDir, + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + // Exit 0 path: bundle was written, no error thrown + expect(result.bundle).toBeDefined(); + const meta = JSON.parse(readFileSync(join(outDir, 'meta.json'), 'utf8')); + expect(meta.runIdIfAvailable).toBeNull(); + expect(meta.testId).toBe('test_m2'); + }); +}); diff --git a/src/commands/test.create-batch-run.spec.ts b/src/commands/test.create-batch-run.spec.ts new file mode 100644 index 0000000..5719543 --- /dev/null +++ b/src/commands/test.create-batch-run.spec.ts @@ -0,0 +1,2790 @@ +/** + * Unit tests for `test create-batch --run` fan-out — UC#2. + * + * Covers: + * - Happy path: 3 specs, all pass with --wait → results array, exit 0 + * - Mixed outcomes: 2 pass, 1 fail → exit 1, results array shows mixed + * - --max-concurrency 2 with 4 specs: only 2 in-flight at any time + * - --timeout per-run honored: one slow run hits timeout, others complete + * - Idempotency: each per-run key is unique, never reuses create key + * - CONFLICT on one run auto-resumed (same path as single-test run) + * - --dry-run mode works on the chain, no network calls + * - create-batch command exposes --timeout flag + * - RATE_LIMITED trigger retried (outer retry loop): first call → RATE_LIMITED, second → success + * - All RATE_LIMITED after outer retry cap exhausted → error result for that spec + * - Client-side throttle: a full 50-spec batch stays within the 50/min window + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError } from '../lib/errors.js'; +import type { RunResponse, TriggerRunResponse } from '../lib/runs.types.js'; +import { + BATCH_RUN_RATE_LIMIT, + BATCH_RUN_RATE_MAX_OUTER_RETRIES, + DEFAULT_BATCH_RUN_CONCURRENCY, + createTestCommand, + isTransientRateLimit, + runCreateBatch, + runTestRun, +} from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13502', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-batch-run-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +function writePlansJsonl(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-batch-plans-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; +} + +const FE_SPEC = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'spec-one', + planSteps: [{ type: 'action', description: 'navigate to home' }], +}; + +function makeBatchCreateResponse(testIds: string[]) { + return { + results: testIds.map((testId, i) => ({ + specIndex: i, + testId, + status: 'created' as const, + })), + summary: { total: testIds.length, created: testIds.length, failed: 0 }, + }; +} + +function makeTriggerResponse(testId: string, runId: string): TriggerRunResponse { + return { + runId, + status: 'queued', + enqueuedAt: '2026-05-26T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }; +} + +function makePassedRun(testId: string, runId: string): RunResponse { + return { + runId, + testId, + projectId: 'project_alice', + userId: 'user_1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-26T10:00:00.000Z', + startedAt: '2026-05-26T10:00:01.000Z', + finishedAt: '2026-05-26T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 3, failedCount: 0 }, + }; +} + +function makeFailedRun(testId: string, runId: string): RunResponse { + return { + ...makePassedRun(testId, runId), + status: 'failed', + failedStepIndex: 1, + failureKind: 'assertion', + }; +} + +const instantSleep = () => Promise.resolve(); + +// --------------------------------------------------------------------------- +// Surface: create-batch exposes --timeout, --run, --wait, --max-concurrency +// --------------------------------------------------------------------------- + +describe('create-batch command surface', () => { + it('exposes --timeout flag', async () => { + const test = createTestCommand(); + const batch = test.commands.find(c => c.name() === 'create-batch')!; + expect(batch).toBeDefined(); + const flagNames = batch.options.map(o => o.long); + expect(flagNames).toContain('--timeout'); + expect(flagNames).toContain('--run'); + expect(flagNames).toContain('--wait'); + expect(flagNames).toContain('--max-concurrency'); + }); + + it('--run flag description no longer says UNSUPPORTED', async () => { + const test = createTestCommand(); + const batch = test.commands.find(c => c.name() === 'create-batch')!; + const runOpt = batch.options.find(o => o.long === '--run')!; + expect(runOpt.description).not.toContain('UNSUPPORTED'); + expect(runOpt.description).not.toContain('exits 7'); + }); +}); + +// --------------------------------------------------------------------------- +// Happy path — 3 specs, all pass with --wait → results array, exit 0 +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run --wait: happy path', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('3 specs all pass — results array has 3 entries, exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_a1', 'test_a2', 'test_a3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + const fetchImpl = makeFetch(url => { + // Batch create + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + // Trigger POST /tests/{id}/runs + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const runId = `run_${testId}`; + return { body: makeTriggerResponse(testId, runId) }; + } + // Poll GET /runs/{runId} + const pollMatch = /\/runs\/(run_test_[a-z0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return { body: makePassedRun(testId, runId) }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + // JSON output is the batch-run envelope + const printed = JSON.parse(stdout.join('')) as { results: unknown[] }; + expect(printed.results).toHaveLength(3); + const results = printed.results as Array<{ testId: string; status: string; runId: string }>; + expect(results.every(r => r.status === 'passed')).toBe(true); + expect(results.map(r => r.testId).sort()).toEqual(['test_a1', 'test_a2', 'test_a3'].sort()); + // All runIds present + expect(results.every(r => r.runId.startsWith('run_test_'))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Mixed outcomes — 2 pass, 1 fail → exit 1, results array shows mixed +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run --wait: mixed outcomes', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('2 pass, 1 fail → exit 1, all 3 result entries in envelope', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_b1', 'test_b2', 'test_b3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const runId = `run_${testId}`; + return { body: makeTriggerResponse(testId, runId) }; + } + const pollMatch = /\/runs\/(run_test_[a-z0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + // test_b3 fails; others pass + if (testId === 'test_b3') return { body: makeFailedRun(testId, runId) }; + return { body: makePassedRun(testId, runId) }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + const err = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + + // Should throw CLIError exit 1 + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + + // Results envelope was printed on stdout before the error + const printed = JSON.parse(stdout.join('')) as { results: unknown[] }; + expect(printed.results).toHaveLength(3); + const results = printed.results as Array<{ testId: string; status: string }>; + const passed = results.filter(r => r.status === 'passed'); + const failed = results.filter(r => r.status === 'failed'); + expect(passed).toHaveLength(2); + expect(failed).toHaveLength(1); + expect(failed[0]?.testId).toBe('test_b3'); + }); +}); + +// --------------------------------------------------------------------------- +// --max-concurrency: verify only N in-flight at any time +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run: --max-concurrency bounds concurrency', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('--max-concurrency 2 with 4 specs: triggers at most 2 at a time', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_c1', 'test_c2', 'test_c3', 'test_c4']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC, FE_SPEC]); + + // Track concurrent in-flight triggers + let inFlightCount = 0; + let maxObservedConcurrency = 0; + + // We use a two-phase fetch: trigger returns a pending promise so we can + // observe concurrent depth, then resolves when the poll GET is called. + const triggerResolvers = new Map void>(); + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + // Batch create — instant + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + // Trigger POST — track concurrent count, resolve immediately for this test + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + inFlightCount++; + maxObservedConcurrency = Math.max(maxObservedConcurrency, inFlightCount); + // Simulate async work so concurrency can be observed + await new Promise(resolve => { + triggerResolvers.set(testId, resolve); + // Auto-resolve after a microtask so tests don't hang + Promise.resolve().then(resolve); + }); + inFlightCount--; + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + // Poll GET — return terminal immediately + const pollMatch = /\/runs\/(run_test_[a-z0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 2, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + + // Verify all 4 results are present + const printed = JSON.parse(stdout.join('')) as { results: unknown[] }; + expect(printed.results).toHaveLength(4); + // Concurrency never exceeded 2 + expect(maxObservedConcurrency).toBeLessThanOrEqual(2); + }); +}); + +// --------------------------------------------------------------------------- +// --timeout per-run honored +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run --wait: --timeout per-run', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('one slow run hits timeout while other completes; overall exit is 1 (not 7, mixed)', async () => { + // test_d1: passes quickly (poll returns terminal on first call — returns + // *before* mockNow crosses the deadline) + // test_d2: always returns non-terminal; each poll advances mockNow by + // 600 ms, so the 1 s deadline is crossed on the second iteration. + const { credentialsPath } = makeCreds(); + const testIds = ['test_d1', 'test_d2']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + + // Fetch-driven mock clock: trigger and batch-create calls don't advance + // time; only poll fetches do. This is robust against indeterminate + // Date.now() call counts in the HTTP layer. + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + try { + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const runId = `run_${testId}`; + return { body: makeTriggerResponse(testId, runId) }; + } + const pollMatch = /\/runs\/(run_test_d[12])/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + // Each poll fetch advances the mock clock by 600 ms. With 1 s + // timeout, test_d2 needs ~2 iterations to hit the deadline. + mockNow += 600; + if (testId === 'test_d1') return { body: makePassedRun(testId, runId) }; + return { body: { ...makePassedRun(testId, runId), status: 'running' as const } }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const err = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 1, // per-run timeout of 1 second + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + + // Should exit 1 (mixed: one pass, one timeout — not all timeout → exit 1) + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + + // Results: test_d1 passes, test_d2 has timeout error + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + expect(printed.results).toHaveLength(2); + const d1 = printed.results.find(r => r.testId === 'test_d1'); + const d2 = printed.results.find(r => r.testId === 'test_d2'); + expect(d1?.status).toBe('passed'); + expect(d2?.status).toBe('timeout'); + } finally { + Date.now = realDateNow; + } + }); + + it('all runs time out → exit 7', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_e1', 'test_e2']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + + // Fetch-driven mock clock — same approach as the mixed-outcome test. + // Both runs always return non-terminal, so every poll fetch advances + // mockNow until the 1 s deadline is crossed. + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + try { + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const runId = `run_${testId}`; + return { body: makeTriggerResponse(testId, runId) }; + } + // Always return running (non-terminal) to force timeout. + // Each poll fetch advances mockNow by 600 ms — with 1 s timeout + // both runs cross the deadline on the second iteration. + const pollMatch = /\/runs\/(run_test_e[12])/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + mockNow += 600; + return { body: { ...makePassedRun(testId, runId), status: 'running' as const } }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const err = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + + // All runs timed out → exit 7 + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(7); + } finally { + Date.now = realDateNow; + } + }); +}); + +// --------------------------------------------------------------------------- +// Idempotency: each per-run key is unique (regression guard) +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run: idempotency key uniqueness', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('each run trigger gets a distinct idempotency key, none matches the create key', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_f1', 'test_f2', 'test_f3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + const createIdemKey = 'cli-create-batch-test-key'; + + const seenRunIdempotencyKeys: string[] = []; + let createIdempotencyKeySeen = ''; + + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const headers = new Headers(init.headers as Record); + + if (url.includes('/tests/batch')) { + createIdempotencyKeySeen = headers.get('idempotency-key') ?? ''; + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const key = headers.get('idempotency-key') ?? ''; + seenRunIdempotencyKeys.push(key); + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_[a-z0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + idempotencyKey: createIdemKey, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + + // 3 run triggers → 3 run idempotency keys + expect(seenRunIdempotencyKeys).toHaveLength(3); + // All run keys must be unique + const uniqueKeys = new Set(seenRunIdempotencyKeys); + expect(uniqueKeys.size).toBe(3); + // None of the run keys matches the create key (regression guard) + expect(seenRunIdempotencyKeys).not.toContain(createIdemKey); + // The create key was used on the batch POST + expect(createIdempotencyKeySeen).toBe(createIdemKey); + // All run keys should start with 'cli-batch-run-' + for (const key of seenRunIdempotencyKeys) { + expect(key).toMatch(/^cli-batch-run-[0-9a-f-]{36}$/); + } + }); +}); + +// --------------------------------------------------------------------------- +// CONFLICT auto-resume on one run +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run --wait: CONFLICT auto-resume', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('one run returns CONFLICT run_in_flight with currentRunId → auto-resumes, result uses conflict runId', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_g1', 'test_g2']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + + // test_g1: normal trigger → passes + // test_g2: trigger returns 409 CONFLICT with currentRunId; poll on inflight → passes + const conflictRunId = 'run_inflight_g2'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + + // test_g1 trigger — normal + if (url.includes('/tests/test_g1/runs')) { + return { body: makeTriggerResponse('test_g1', 'run_test_g1') }; + } + + // test_g2 trigger — returns CONFLICT + if (url.includes('/tests/test_g2/runs')) { + return { + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'Test test_g2 already has a run in flight.', + nextAction: 'wait for it.', + requestId: 'req_conflict_g2', + details: { reason: 'run_in_flight', currentRunId: conflictRunId }, + }, + }, + }; + } + + // Poll for test_g1 run + if (url.includes('/runs/run_test_g1')) { + return { body: makePassedRun('test_g1', 'run_test_g1') }; + } + + // Poll for the conflict inflight run + if (url.includes(`/runs/${conflictRunId}`)) { + return { body: makePassedRun('test_g2', conflictRunId) }; + } + + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; runId: string; status: string }>; + }; + expect(printed.results).toHaveLength(2); + + const g1 = printed.results.find(r => r.testId === 'test_g1'); + const g2 = printed.results.find(r => r.testId === 'test_g2'); + + // Both should pass + expect(g1?.status).toBe('passed'); + // g2's result should use the inflight runId, not a freshly minted one + expect(g2?.runId).toBe(conflictRunId); + expect(g2?.status).toBe('passed'); + + // Advisory should mention the inflight runId + expect(stderrLines.some(l => l.includes(conflictRunId))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// --dry-run mode +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run --dry-run', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('dry-run: no real trigger calls; prints descriptor envelope with testIds and dryRun: true', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_h1', 'test_h2']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + + // fetchImpl should not be called for trigger/poll in dry-run. + // It IS called for the batch create (dry-run fetch returns sample). + // We simulate a batch create dry-run response. + let triggerCallCount = 0; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + // Any trigger or poll call is a test failure + if (url.includes('/runs')) { + triggerCallCount++; + } + return { status: 200, body: {} }; + }); + + const stdout: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + plans: plansFile, + run: true, + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + + // No real trigger/poll calls + expect(triggerCallCount).toBe(0); + + // Descriptor envelope must contain dryRun: true and testIds + const envelope = JSON.parse(stdout.join('')) as Record; + expect(envelope.dryRun).toBe(true); + expect(Array.isArray(envelope.testIds)).toBe(true); + expect((envelope.testIds as string[]).sort()).toEqual(testIds.sort()); + // results array is present + expect(Array.isArray(envelope.results)).toBe(true); + expect((envelope.results as unknown[]).length).toBe(testIds.length); + }); +}); + +// --------------------------------------------------------------------------- +// DEFAULT_BATCH_RUN_CONCURRENCY — bounded default (not Infinity) + override +// --------------------------------------------------------------------------- + +describe('DEFAULT_BATCH_RUN_CONCURRENCY', () => { + it('is 50 (bounded, not Infinity)', () => { + expect(DEFAULT_BATCH_RUN_CONCURRENCY).toBe(50); + expect(DEFAULT_BATCH_RUN_CONCURRENCY).not.toBe(Infinity); + expect(Number.isFinite(DEFAULT_BATCH_RUN_CONCURRENCY)).toBe(true); + }); + + it('--max-concurrency override is respected (not clamped to default)', async () => { + // When --max-concurrency 2 is passed, the actual concurrency cap must be 2 + // (less than DEFAULT_BATCH_RUN_CONCURRENCY=50), confirming the override path. + const { credentialsPath } = makeCreds(); + const testIds = ['test_conc1', 'test_conc2', 'test_conc3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + let maxObservedConcurrency = 0; + let inFlightCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_conc[0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + inFlightCount++; + maxObservedConcurrency = Math.max(maxObservedConcurrency, inFlightCount); + await Promise.resolve(); // yield so concurrency can be observed + inFlightCount--; + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_conc[0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 2, // explicit override — lower than default 50 + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + + // Override is respected: concurrency was bounded at 2, not at DEFAULT (10) + expect(maxObservedConcurrency).toBeLessThanOrEqual(2); + }); + + it('omitting --max-concurrency uses DEFAULT_BATCH_RUN_CONCURRENCY (50), not Infinity', async () => { + // With 50 specs (== MAX_BATCH_SPECS) and no --max-concurrency, the raised + // default (50) lets all 50 be in flight at once, bounded at + // DEFAULT_BATCH_RUN_CONCURRENCY (50) — never Infinity (unbounded all-at-once). + const { credentialsPath } = makeCreds(); + const SPEC_COUNT = 50; + const testIds = Array.from({ length: SPEC_COUNT }, (_, i) => `test_def${i}`); + const plansFile = writePlansJsonl(Array.from({ length: SPEC_COUNT }, () => FE_SPEC)); + + let maxObservedConcurrency = 0; + let inFlightCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_def[0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + inFlightCount++; + maxObservedConcurrency = Math.max(maxObservedConcurrency, inFlightCount); + await Promise.resolve(); + inFlightCount--; + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_def[0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + // maxConcurrency intentionally omitted → should use DEFAULT (10) + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + + // With 50 specs, the default concurrency 50 bounds us at or below 50 + expect(maxObservedConcurrency).toBeLessThanOrEqual(DEFAULT_BATCH_RUN_CONCURRENCY); + // And the raised default lets concurrency climb well past the old cap of 10 + expect(maxObservedConcurrency).toBeGreaterThan(10); + }); +}); + +// --------------------------------------------------------------------------- +// RATE_LIMITED outer retry: trigger returns RATE_LIMITED then succeeds +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run: RATE_LIMITED outer retry', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('trigger returns RATE_LIMITED once then succeeds — result is passed, no terminal error', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_rl1', 'test_rl2']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + + // test_rl1: trigger fails with RATE_LIMITED on first attempt, succeeds on second. + // test_rl2: triggers normally. + const triggerAttempts = new Map(); + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_rl[12])\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const attempt = (triggerAttempts.get(testId) ?? 0) + 1; + triggerAttempts.set(testId, attempt); + + // test_rl1: first outer-loop attempt returns RATE_LIMITED; second attempt succeeds. + // The batch path passes retryOnRateLimit: false to triggerRunWithMeta, so the + // HTTP layer throws immediately on the first 429 — no internal sub-retries. + // The outer retry loop in triggerOne owns backoff and re-trigger. Each outer + // attempt = exactly ONE HTTP call. `attempt <= 1` covers the first outer attempt. + if (testId === 'test_rl1' && attempt <= 1) { + // First outer attempt → HTTP layer throws RATE_LIMITED immediately + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait before retrying.', + requestId: `req_rl_${attempt}`, + details: {}, + }, + }), + { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '1' }, + }, + ); + } + + // Success + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_rl[12])/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + const stderrLines: string[] = []; + const sleepCalls: number[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, // serial to make call order deterministic + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: ms => { + sleepCalls.push(ms); + return Promise.resolve(); + }, + }, + ); + + // Both should pass — test_rl1 recovered from RATE_LIMITED via outer retry + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; runId: string }>; + }; + expect(printed.results).toHaveLength(2); + expect(printed.results.every(r => r.status === 'passed')).toBe(true); + + // The outer retry should have emitted a RATE_LIMITED retry advisory on stderr + const rateLimitedAdvisory = stderrLines.find(l => l.includes('RATE_LIMITED')); + expect(rateLimitedAdvisory).toBeDefined(); + + // Sleep was called (the outer retry back-off) + expect(sleepCalls.length).toBeGreaterThan(0); + }); + + it('trigger returns RATE_LIMITED on all outer retries → error result for that spec', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_rl3']; + const plansFile = writePlansJsonl([FE_SPEC]); + + // Every trigger call returns RATE_LIMITED so the outer retry cap is exceeded. + // retryOnRateLimit: false (batch path) means the HTTP layer throws immediately on + // the first 429 — each outer attempt = exactly ONE HTTP call. Use Retry-After: 0 + // so outer-retry back-offs are instant. + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_rl3/runs')) { + // Retry-After: 0 → HTTP-layer sleeps 0 ms per sub-attempt (instant) + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait before retrying.', + requestId: 'req_rl_always', + details: {}, + }, + }), + { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '0' }, + }, + ); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + + const err = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: () => Promise.resolve(), // instant outer-retry back-off + }, + ).catch(e => e); + + // Overall exit should be a CLIError (exit 11 for uniform RATE_LIMITED) + expect(err).toBeInstanceOf(CLIError); + // The spec should appear in results with status 'error' and code RATE_LIMITED + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ + testId: string; + status: string; + error?: { code: string; exitCode: number }; + }>; + }; + expect(printed.results).toHaveLength(1); + const result = printed.results[0]!; + expect(result.status).toBe('error'); + expect(result.error?.code).toBe('RATE_LIMITED'); + expect(result.error?.exitCode).toBe(11); + }); // retryOnRateLimit: false → 1 HTTP call per outer attempt; extended timeout not needed +}); + +// --------------------------------------------------------------------------- +// Client-side rate throttle: BATCH_RUN_RATE_LIMIT + rate window constants +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run: client-side throttle constants and behaviour', () => { + it('BATCH_RUN_RATE_LIMIT is 50 (sits under the server 60/min/key cap)', () => { + expect(BATCH_RUN_RATE_LIMIT).toBe(50); + }); + + it('BATCH_RUN_RATE_MAX_OUTER_RETRIES is > 0 (outer retry is bounded)', () => { + expect(BATCH_RUN_RATE_MAX_OUTER_RETRIES).toBeGreaterThan(0); + }); + + it('a full 50-spec batch (== rate limit) fires with no client throttle delay', async () => { + // A maxed-out create-batch holds MAX_BATCH_SPECS (50) specs, and the client + // throttle limit is also 50/window, so every trigger acquires a slot without + // waiting — even with the window frozen. We assert no positive throttle sleep. + const { credentialsPath } = makeCreds(); + const SPEC_COUNT = BATCH_RUN_RATE_LIMIT; // a full batch == the per-minute cap (== MAX_BATCH_SPECS) + const testIds = Array.from({ length: SPEC_COUNT }, (_, i) => `test_thr${i}`); + const plansFile = writePlansJsonl(Array.from({ length: SPEC_COUNT }, () => FE_SPEC)); + + // Track sleep calls. Polls resolve 'passed' immediately (no backoff), so any + // positive sleep here would be a throttle delay — which must NOT happen. + const sleepCallsMs: number[] = []; + + // Freeze Date.now so the throttle window never advances — proving all 50 + // triggers fit under the 50-slot limit within a single window, no wait. + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + try { + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_thr[0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + const runId = `run_${testId}`; + // Advance mock time by 1 ms per trigger so throttle slots are slightly + // spread (avoids exact-boundary edge cases) but well within 60 s window. + mockNow += 1; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_thr[0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r1' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + // --wait:true with all polls resolving 'passed' immediately, so the only + // possible positive sleep would come from the rate throttle. A generous + // 300s budget rules out any deadline interaction; the assertion below + // proves the throttle never delayed any of the 50 triggers. + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, // serial: one trigger at a time so throttle fires cleanly + timeoutSeconds: 300, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: (ms: number) => { + sleepCallsMs.push(ms); + // Advance mockNow so the throttle window progresses on the sleep call + // (simulates wall-clock time passing during the throttle delay). + mockNow += ms; + return Promise.resolve(); + }, + }, + ); + } finally { + Date.now = realDateNow; + } + + // No trigger should have been throttled: 50 specs all fit under the 50/window + // limit, so the throttle emits zero positive-ms sleep calls. + const throttleDelays = sleepCallsMs.filter(ms => ms > 0); + expect(throttleDelays.length).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// --max-concurrency help text accuracy +// --------------------------------------------------------------------------- + +describe('create-batch --max-concurrency help text', () => { + it('accurately states the server 60/min/key cap and the 50/min client throttle', () => { + const test = createTestCommand(); + const batch = test.commands.find(c => c.name() === 'create-batch')!; + const opt = batch.options.find(o => o.long === '--max-concurrency')!; + // Names the real server cap (raised 20 → 60 on the backend, PR #531) + expect(opt.description).toContain('60/min/key'); + // ...and the client-side throttle / default figure + expect(opt.description).toContain('50'); + // The stale 20/min figure must be gone + expect(opt.description).not.toContain('20'); + }); +}); + +// --------------------------------------------------------------------------- +// CODEX ROUND-1 FIXES +// --------------------------------------------------------------------------- + +// MAJOR 1: HTTP-layer no longer retries RATE_LIMITED for triggerRunWithMeta. +// The outer loop is the sole retrier; total trigger POSTs stay within the 50/min client throttle. +// +// Regression proof: with HTTP-layer retry enabled, a single invocation that +// receives RATE_LIMITED could fire up to 3 HTTP attempts internally, so a 20-spec +// batch at concurrency 20 could send 60 POSTs/min (3× the cap). +// With retryOnRateLimit:false on triggerRunWithMeta, each RATE_LIMITED response +// surfaces immediately as a single throw. +describe('MAJOR 1: HTTP-layer does not retry RATE_LIMITED for triggerRunWithMeta', () => { + it('a single RATE_LIMITED trigger fires exactly ONE POST before throwing', async () => { + // triggerRunWithMeta sets retryOnRateLimit: false. Verify that a single 429 + // response causes exactly one fetch call (not 1 + up to 2 retries = 3). + const { credentialsPath } = makeCreds(); + const testIds = ['test_m1a']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerCallCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + if (url.includes('/tests/test_m1a/runs')) { + triggerCallCount++; + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait 60s.', + requestId: `req_m1_${triggerCallCount}`, + details: { retryAfterSeconds: 1 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '1' } }, + ); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + // Outer retry sleeps are instant but still called; we don't care about + // them here — we only care that the HTTP layer fired exactly 1 POST per + // outer attempt. + sleep: () => Promise.resolve(), + }, + ).catch(() => {}); + + // Each outer retry attempt should have fired at most 1 POST (not 3). + // With BATCH_RUN_RATE_MAX_OUTER_RETRIES=5 outer attempts, max is 6 + // (1 initial + 5 retries), each firing exactly 1 POST = 6 total. + // Without the fix (retryOnRateLimit: true in HttpClient), each outer + // attempt would fire 3 POSTs = 18 total. + expect(triggerCallCount).toBeLessThanOrEqual(BATCH_RUN_RATE_MAX_OUTER_RETRIES + 1); + // Each outer attempt fired exactly 1 POST (not 3 HTTP-layer sub-retries). + expect(triggerCallCount).toBeGreaterThan(0); + }); +}); + +// MAJOR 2: Sleeps clamped to --wait deadline; expiring deadline returns timeout +// and does NOT start a fresh poll. +describe('MAJOR 2: sleeps clamped to --wait deadline; no fresh poll after deadline', () => { + it('throttle-wait deadline clamp: a RATE_LIMITED retry that overflows the window times out (no poll)', async () => { + // With BATCH_RUN_RATE_LIMIT (50) == MAX_BATCH_SPECS (50), a single batch's 50 + // initial triggers exactly fill the window — they never wait. The throttle- + // wait path is still reachable on a RATE_LIMITED *retry*: the retry's + // re-acquire is the 51st slot in the (frozen) window and must wait. Here that + // wait is clamped to the --wait deadline, so the spec returns 'timeout' + // (exit 7) at the throttle-acquire loop without ever polling. + const { credentialsPath } = makeCreds(); + const SPEC_COUNT = BATCH_RUN_RATE_LIMIT; // 50: a full batch == the window cap + const testIds = Array.from({ length: SPEC_COUNT }, (_, i) => `test_dl${i}`); + const plansFile = writePlansJsonl(Array.from({ length: SPEC_COUNT }, () => FE_SPEC)); + const lastId = `test_dl${SPEC_COUNT - 1}`; + + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + try { + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const triggerMatch = /\/tests\/(test_dl[0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + // The last spec always returns a transient RATE_LIMITED (Retry-After 1s), + // forcing a retry whose re-acquire is the 51st slot in the frozen window. + if (testId === lastId) { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait and retry.', + requestId: 'req_dl', + details: { retryAfterSeconds: 1 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '1' } }, + ); + } + const runId = `run_${testId}`; + return new Response(JSON.stringify(makeTriggerResponse(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + const pollMatch = /\/runs\/(run_test_dl[0-9]+)/.exec(url); + if (pollMatch?.[1]) { + const runId = pollMatch[1]; + const testId = runId.replace('run_', ''); + return new Response(JSON.stringify(makePassedRun(testId, runId)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + const sleepCallsMs: number[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, // serial so the 50 initial acquires precede the retry + timeoutSeconds: 2, // deadline > Retry-After(1s) so the throttle wait (not the backoff) exhausts it + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: (ms: number) => { + // Realistic clock: advance by exactly `ms` (no jump-past) so the + // deadline is consumed *during* the throttle-wait sleep, exercising the + // clamp at the throttle-acquire loop, not the pre-trigger deadline guard. + sleepCallsMs.push(ms); + mockNow += ms; + return Promise.resolve(); + }, + }, + ).catch(() => {}); + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; error?: { message?: string } }>; + }; + const last = printed.results.find(r => r.testId === lastId); + expect(last?.status).toBe('timeout'); + // Branch-specific: the throttle-wait deadline clamp (not the pre-trigger + // deadline guard, nor the RATE_LIMITED backoff) — its message names the slot. + expect(last?.error?.message ?? '').toContain('waiting to acquire throttle slot'); + // Corroborating: both the RATE_LIMITED backoff and the throttle-wait slept + // (a single backoff sleep alone would trip the pre-trigger guard instead). + expect(sleepCallsMs.filter(ms => ms > 0).length).toBeGreaterThanOrEqual(2); + } finally { + Date.now = realDateNow; + } + }); + + it('RATE_LIMITED retry sleep clamped: deadline expires during backoff → timeout (no poll started)', async () => { + // Trigger always returns RATE_LIMITED so the outer loop sleeps 60s. + // With --timeout 1s the sleep should be clamped to ~1s and then timeout. + // The test verifies poll is NOT called after the deadline expires. + const { credentialsPath } = makeCreds(); + const testIds = ['test_dl_rl']; + const plansFile = writePlansJsonl([FE_SPEC]); + + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + let pollCallCount = 0; + + try { + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_dl_rl/runs')) { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait.', + requestId: 'req_dl_rl', + details: { retryAfterSeconds: 60 }, + }, + }), + { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '60' }, + }, + ); + } + if (url.includes('/runs/')) { + pollCallCount++; + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 1, // 1 second budget + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: (ms: number) => { + // Advance time past deadline on any sleep so the clamping logic kicks in. + mockNow += ms + 2000; + return Promise.resolve(); + }, + }, + ).catch(() => {}); + + // Poll was never called — the RATE_LIMITED backoff consumed the deadline. + expect(pollCallCount).toBe(0); + + // Result for the spec should be timeout, not error or passed. + if (stdout.length > 0) { + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + const result = printed.results.find(r => r.testId === 'test_dl_rl'); + if (result !== undefined) { + expect(result.status).toBe('timeout'); + } + } + } finally { + Date.now = realDateNow; + } + }); +}); + +// MAJOR 3: Retry-After header preserved on thrown ApiError; honored by outer loop. +describe('MAJOR 3: Retry-After header honored by outer retry loop', () => { + it('trigger 429 with Retry-After:5 causes exactly 5s sleep (not hardcoded 60s)', async () => { + // Test that the outer loop uses the Retry-After header value, not the fallback. + const { credentialsPath } = makeCreds(); + const testIds = ['test_ra1']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerAttempt = 0; + const sleepCallsMs: number[] = []; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_ra1/runs')) { + triggerAttempt++; + if (triggerAttempt === 1) { + // Return 429 with Retry-After: 5 (should be honored, not the 60s default) + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait 5s.', + requestId: 'req_ra1', + details: { retryAfterSeconds: 5 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '5' } }, + ); + } + // Second attempt succeeds + return new Response(JSON.stringify(makeTriggerResponse('test_ra1', 'run_ra1')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/run_ra1')) { + return new Response(JSON.stringify(makePassedRun('test_ra1', 'run_ra1')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 120, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: (ms: number) => { + sleepCallsMs.push(ms); + return Promise.resolve(); + }, + }, + ); + + // The outer retry sleep should be exactly 5000ms (from Retry-After: 5), + // NOT 60000ms (the hardcoded fallback). Filter out tiny throttle sleeps (< 100ms). + const backoffSleeps = sleepCallsMs.filter(ms => ms >= 1000); + expect(backoffSleeps.length).toBeGreaterThan(0); + // The sleep should be ~5s (from the header), not ~60s + expect(backoffSleeps[0]).toBe(5000); + // Definitely not the 60s hardcoded fallback + expect(backoffSleeps[0]).not.toBe(60_000); + }); + + it('trigger 429 with Retry-After:400 is clamped to 300s ([1s,300s] range)', async () => { + // Server sends an unreasonably long Retry-After; should be clamped. + const { credentialsPath } = makeCreds(); + const testIds = ['test_ra2']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let attempt = 0; + const sleepCallsMs: number[] = []; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_ra2/runs')) { + attempt++; + if (attempt === 1) { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait.', + requestId: 'req_ra2', + details: { retryAfterSeconds: 400 }, + }, + }), + // Retry-After: 400 — well above the 300s cap + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '400' } }, + ); + } + return new Response(JSON.stringify(makeTriggerResponse('test_ra2', 'run_ra2')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/run_ra2')) { + return new Response(JSON.stringify(makePassedRun('test_ra2', 'run_ra2')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 600, // big budget so clamp is the only thing that fires + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: (ms: number) => { + sleepCallsMs.push(ms); + return Promise.resolve(); + }, + }, + ); + + // HttpClient clamps Retry-After to [1s, 300s] before setting retryAfterMs. + const backoffSleeps = sleepCallsMs.filter(ms => ms >= 1000); + expect(backoffSleeps.length).toBeGreaterThan(0); + // Must be ≤ 300s (300000ms), never the raw 400s from the header + expect(backoffSleeps[0]).toBeLessThanOrEqual(300_000); + // Must be > 0 + expect(backoffSleeps[0]).toBeGreaterThan(0); + }); +}); + +// Credit-depletion vs transient rate-limit: predicate and retry behavior. +describe('isTransientRateLimit + credit-depletion not retried', () => { + // Unit tests for the predicate + describe('isTransientRateLimit predicate', () => { + function makeRateLimitedError(opts: { + message: string; + retryAfterMs?: number; + details?: Record; + }): ApiError { + return new ApiError( + { + code: 'RATE_LIMITED', + message: opts.message, + nextAction: '', + requestId: 'req_pred', + details: opts.details ?? {}, + }, + 429, + opts.retryAfterMs, + ); + } + + it('returns true when retryAfterMs is set (Retry-After header present)', () => { + const err = makeRateLimitedError({ + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + retryAfterMs: 5000, + }); + expect(isTransientRateLimit(err)).toBe(true); + }); + + it('returns true when message matches per-minute rate limit wording', () => { + const err = makeRateLimitedError({ + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + retryAfterMs: undefined, + }); + expect(isTransientRateLimit(err)).toBe(true); + }); + + it('returns true when details.retryAfterSeconds is present', () => { + const err = makeRateLimitedError({ + message: 'Some other rate limit message.', + retryAfterMs: undefined, + details: { retryAfterSeconds: 30 }, + }); + expect(isTransientRateLimit(err)).toBe(true); + }); + + it('returns false for credit-depletion message (no Retry-After, no per-min wording)', () => { + const err = makeRateLimitedError({ + message: + 'Insufficient credits: 1 credit(s) required. Top up at https://www.testsprite.com/settings/billing.', + retryAfterMs: undefined, + details: { required: 1, userId: 'user_123' }, + }); + expect(isTransientRateLimit(err)).toBe(false); + }); + + it('returns false for unknown RATE_LIMITED with no header and no per-min wording', () => { + const err = makeRateLimitedError({ + message: 'Rate limit exceeded.', + retryAfterMs: undefined, + }); + expect(isTransientRateLimit(err)).toBe(false); + }); + }); + + // Integration: credit-depletion RATE_LIMITED is NOT retried in the outer loop. + it('credit-depletion RATE_LIMITED is NOT retried — surfaces immediately as terminal error', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_cred']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerCallCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_cred/runs')) { + triggerCallCount++; + // Credit-depletion error: no Retry-After header, message differs from per-min wording + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: + 'Insufficient credits: 1 credit(s) required. Top up at https://www.testsprite.com/settings/billing.', + nextAction: + 'Top up your credit balance at https://www.testsprite.com/settings/billing, then retry.', + requestId: 'req_cred', + details: { required: 1, userId: 'user_x' }, + }, + }), + { + // No Retry-After header — key distinguishing signal + status: 429, + headers: { 'content-type': 'application/json' }, + }, + ); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + const sleepCalls: number[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: ms => { + sleepCalls.push(ms); + return Promise.resolve(); + }, + }, + ).catch(() => {}); + + // Trigger was called exactly ONCE — no retries for credit depletion. + expect(triggerCallCount).toBe(1); + // No sleep calls for retry backoff (only throttle-acquire sleeps possible, but + // with one spec in an empty window those are 0). + const backoffSleeps = sleepCalls.filter(ms => ms > 0); + expect(backoffSleeps).toHaveLength(0); + + // Result should be status='error' with code=RATE_LIMITED + if (stdout.length > 0) { + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; error?: { code: string } }>; + }; + const result = printed.results.find(r => r.testId === 'test_cred'); + expect(result?.status).toBe('error'); + // Credits depletion is now re-mapped to INSUFFICIENT_CREDITS (exit 12) + expect(result?.error?.code).toBe('INSUFFICIENT_CREDITS'); + } + }); + + it('per-minute RATE_LIMITED IS retried and eventually succeeds', async () => { + // The transient case (per-minute cap) should still retry. + // First trigger → RATE_LIMITED with Retry-After: 1; second → success. + const { credentialsPath } = makeCreds(); + const testIds = ['test_transient']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let attempt = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_transient/runs')) { + attempt++; + if (attempt === 1) { + // Transient per-minute rate limit WITH Retry-After header + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait 1s.', + requestId: 'req_transient', + details: { retryAfterSeconds: 1 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '1' } }, + ); + } + return new Response( + JSON.stringify(makeTriggerResponse('test_transient', 'run_transient')), + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, + ); + } + if (url.includes('/runs/run_transient')) { + return new Response(JSON.stringify(makePassedRun('test_transient', 'run_transient')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 120, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: () => Promise.resolve(), // instant backoff + }, + ); + + // Trigger was retried (attempt = 2 total) and eventually passed. + expect(attempt).toBe(2); + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + const result = printed.results.find(r => r.testId === 'test_transient'); + expect(result?.status).toBe('passed'); + }); +}); + +// --------------------------------------------------------------------------- +// CODEX ROUND-2 FIXES +// --------------------------------------------------------------------------- + +/** + * (a) Single `test run` STILL retries a transient 429 (HTTP-layer retry intact). + * + * `triggerRunWithMeta` now defaults `retryOnRateLimit: true`, so the HTTP layer + * retries 429 for non-batch callers. Only the batch path passes `false`. + */ +describe('ROUND-2 (a): single test run retries transient 429 via HTTP-layer', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('runTestRun retries a transient 429 at the HTTP layer and eventually succeeds', async () => { + const { credentialsPath } = makeCreds(); + let fetchCallCount = 0; + + const triggerResp: TriggerRunResponse = { + runId: 'run_r2a', + status: 'queued', + enqueuedAt: '2026-05-30T10:00:00Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }; + const passedRun: RunResponse = { + runId: 'run_r2a', + testId: 'test_r2a', + projectId: 'project_p1', + userId: 'user_1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-30T10:00:00Z', + startedAt: '2026-05-30T10:00:01Z', + finishedAt: '2026-05-30T10:00:10Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/test_r2a/runs')) { + fetchCallCount++; + if (fetchCallCount === 1) { + // First call → transient 429 with Retry-After + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait.', + requestId: 'req_r2a', + details: {}, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '1' } }, + ); + } + // Second call → success (HTTP-layer retry) + return new Response(JSON.stringify(triggerResp), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/run_r2a')) { + return new Response(JSON.stringify(passedRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + await runTestRun( + { + testId: 'test_r2a', + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + sleep: () => Promise.resolve(), // instant HTTP-layer retry sleep + stderr: () => {}, + }, + ); + + // HTTP layer made 2 fetch calls to /runs (1 RATE_LIMITED + 1 success). + // If retryOnRateLimit were false (old behavior), it would throw after call 1. + expect(fetchCallCount).toBe(2); + }); +}); + +/** + * (b) Batch path does NOT double-retry — the outer loop is the sole retrier. + * + * With `retryOnRateLimit: false` on the batch call, each 429 produces exactly + * ONE HTTP call before the outer loop re-acquires a throttle slot and retries. + */ +describe('ROUND-2 (b): batch path does NOT double-retry 429 (no HTTP-layer sub-retries)', () => { + it('each outer attempt fires exactly 1 HTTP call for the trigger POST', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_r2b']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerCallCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_r2b/runs')) { + triggerCallCount++; + if (triggerCallCount <= 1) { + // First outer attempt → RATE_LIMITED, no HTTP-layer retries with retryOnRateLimit:false + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait.', + requestId: `req_r2b_${triggerCallCount}`, + details: { retryAfterSeconds: 1 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '1' } }, + ); + } + // Second outer attempt → success + return new Response(JSON.stringify(makeTriggerResponse('test_r2b', 'run_r2b')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/run_r2b')) { + return new Response(JSON.stringify(makePassedRun('test_r2b', 'run_r2b')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: () => Promise.resolve(), + }, + ); + + // With retryOnRateLimit: false on the batch path, each outer attempt fires + // exactly 1 HTTP call (not 3). So: attempt 1 = 1 call (RATE_LIMITED), + // attempt 2 = 1 call (success) → total = 2. + // If HTTP-layer sub-retries were happening, this would be 4 (1×3 + 1). + expect(triggerCallCount).toBe(2); + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + expect(printed.results[0]?.status).toBe('passed'); + }); +}); + +/** + * (c) No trigger fires after the `--wait` deadline expired. + * + * The deadline check at the TOP of the outer loop (before rateThrottle.acquire) + * ensures that after a retry sleep that pushes past the deadline, no new POST + * is sent on the next iteration. + */ +describe('ROUND-2 (c): no trigger fires after --wait deadline expires', () => { + it('after a RATE_LIMITED sleep that expires the deadline, no further trigger POST is sent', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_r2c']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerCallCount = 0; + + const realDateNow = Date.now; + let mockNow = realDateNow(); + Date.now = () => mockNow; + + try { + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_r2c/runs')) { + triggerCallCount++; + // Always return RATE_LIMITED with Retry-After: 60 + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.', + nextAction: 'Wait.', + requestId: `req_r2c_${triggerCallCount}`, + details: { retryAfterSeconds: 60 }, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '60' } }, + ); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: 1, // 1s deadline + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: (ms: number) => { + // Advance time past the 1s deadline during the first backoff sleep. + mockNow += ms + 2000; + return Promise.resolve(); + }, + }, + ).catch(() => {}); + + // The deadline check at the top of the outer loop fires after the first + // RATE_LIMITED response causes a sleep that pushes mockNow past deadline. + // No second trigger POST should have been sent. + expect(triggerCallCount).toBe(1); + + if (stdout.length > 0) { + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + expect(printed.results[0]?.status).toBe('timeout'); + } + } finally { + Date.now = realDateNow; + } + }); +}); + +/** + * (d) 0 remaining ms → timeout result without polling. + * + * Fix 3 short-circuits before `pollRunUntilTerminal` when `remainingMs() <= 0`, + * rather than converting 0 → 1s via `Math.max(1, Math.floor(0/1000))`. + */ +describe('ROUND-2 (d): 0 remaining ms before poll yields timeout without polling', () => { + it('trigger succeeds but deadline is 0 ms when polling would start → timeout, not 1s poll', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_r2d']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let pollCallCount = 0; + + const realDateNow = Date.now; + let mockNow = realDateNow(); + // Start with mockNow such that deadline = now + 1s. The trigger POST advances + // mockNow past the deadline, so remainingMs() = 0 when polling would start. + const DEADLINE_MS = 1000; // 1s + + try { + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_r2d/runs')) { + // Trigger succeeds — but advance time past the deadline so the poll + // check sees remainingMs() <= 0. + mockNow += DEADLINE_MS + 500; // jump past the 1s deadline + return new Response(JSON.stringify(makeTriggerResponse('test_r2d', 'run_r2d')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + pollCallCount++; + return new Response(JSON.stringify(makePassedRun('test_r2d', 'run_r2d')), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + Date.now = () => mockNow; + + const stdout: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: true, + maxConcurrency: 1, + timeoutSeconds: DEADLINE_MS / 1000, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: () => Promise.resolve(), + }, + ).catch(() => {}); + + // Poll should NOT have been called — deadline was 0 ms when the check ran. + // Old code: Math.max(1, Math.floor(0/1000)) = 1 → would start a 1s poll. + // Fixed code: remainingMs() <= 0 → returns timeout immediately. + expect(pollCallCount).toBe(0); + + if (stdout.length > 0) { + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + }; + expect(printed.results[0]?.status).toBe('timeout'); + } + } finally { + Date.now = realDateNow; + } + }); +}); + +/** + * (e) Insufficient-credits 429 carrying a Retry-After header is NOT retried. + * + * Fix 4: `isTransientRateLimit` now checks "Insufficient credits" FIRST and + * short-circuits to terminal, regardless of whether a Retry-After header was + * present. This closes a gap where a credits-depletion 429 with a stray + * Retry-After header would be incorrectly classified as transient. + */ +describe('ROUND-2 (e): insufficient-credits 429 with Retry-After header is terminal (not retried)', () => { + it('isTransientRateLimit returns false for credits error even with retryAfterMs set', () => { + const err = new ApiError( + { + code: 'RATE_LIMITED', + message: + 'Insufficient credits: 5 credit(s) required. Top up at https://www.testsprite.com/settings/billing.', + nextAction: 'Top up your balance.', + requestId: 'req_e', + details: { required: 5 }, + }, + 429, + // retryAfterMs is set (as if Retry-After header was present on the response) + 30_000, + ); + // Must be false despite retryAfterMs being set — credits check short-circuits. + expect(isTransientRateLimit(err)).toBe(false); + }); + + it('batch path does not retry credits-depletion 429 even when Retry-After header is present', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_r2e']; + const plansFile = writePlansJsonl([FE_SPEC]); + + let triggerCallCount = 0; + + const fetchImpl = (async (input: FetchInput, _init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/tests/batch')) { + return new Response(JSON.stringify(makeBatchCreateResponse(testIds)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/tests/test_r2e/runs')) { + triggerCallCount++; + // Credits-depletion error WITH a Retry-After header (the bad case the fix closes). + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: + 'Insufficient credits: 3 credit(s) required. Top up at https://www.testsprite.com/settings/billing.', + nextAction: 'Top up.', + requestId: 'req_r2e', + details: { required: 3 }, + }, + }), + { + status: 429, + // Stray Retry-After header — should be ignored for credits errors + headers: { 'content-type': 'application/json', 'retry-after': '30' }, + }, + ); + } + return new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof globalThis.fetch; + + const stdout: string[] = []; + const sleepCalls: number[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: ms => { + sleepCalls.push(ms); + return Promise.resolve(); + }, + }, + ).catch(() => {}); + + // Credits-depletion is terminal — trigger fired exactly once, no retries. + expect(triggerCallCount).toBe(1); + + // No backoff sleep for a terminal error. + const backoffSleeps = sleepCalls.filter(ms => ms > 0); + expect(backoffSleeps).toHaveLength(0); + + if (stdout.length > 0) { + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; error?: { code: string } }>; + }; + const result = printed.results.find(r => r.testId === 'test_r2e'); + expect(result?.status).toBe('error'); + // Credits depletion is now re-mapped to INSUFFICIENT_CREDITS (exit 12) + expect(result?.error?.code).toBe('INSUFFICIENT_CREDITS'); + } + }); +}); diff --git a/src/commands/test.quickwins.spec.ts b/src/commands/test.quickwins.spec.ts new file mode 100644 index 0000000..c781300 --- /dev/null +++ b/src/commands/test.quickwins.spec.ts @@ -0,0 +1,939 @@ +/** + * Unit tests for dogfood L1796 quick-wins: + * Piece 3: `test create-batch --plan-from-dir ` + * Piece 4: `test delete-batch` + `test delete-batch --all` + * + * All HTTP is mocked; no real credentials required. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { runCreateBatch } from './test.js'; +import { runDeleteBatch } from './test.js'; +import type { CliBulkDeleteSummary } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers shared across pieces +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-qw-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +/** Minimal valid plan spec (FE). */ +const PLAN_SPEC = { + projectId: 'project_abc', + type: 'frontend', + name: 'Plan test', + planSteps: [ + { type: 'action', description: 'Click button' }, + { type: 'assertion', description: 'Check result' }, + ], +}; + +/** Canned batch-create server response. */ +function batchCreateResp(count: number) { + return { + results: Array.from({ length: count }, (_, i) => ({ + specIndex: i, + testId: `test_batch_${i}`, + status: 'created', + })), + summary: { total: count, created: count, failed: 0 }, + }; +} + +/** Canned delete server response. */ +function deleteResp(testId: string) { + return { + testId, + deletedAt: '2026-06-03T00:00:00.000Z', + }; +} + +// --------------------------------------------------------------------------- +// Piece 3: `test create-batch --plan-from-dir` +// --------------------------------------------------------------------------- + +describe('runCreateBatch --plan-from-dir (dogfood L1796)', () => { + it('reads *.json files from the dir, assembles specs, and posts to /tests/batch', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-')); + + // Write 3 plan files. + for (let i = 0; i < 3; i++) { + writeFileSync( + join(dir, `plan_${String(i).padStart(2, '0')}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Plan ${i}` }), + 'utf8', + ); + } + + let capturedBody: unknown; + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch') && init.method === 'POST') { + capturedBody = JSON.parse(init.body as string); + return { body: batchCreateResp(3) }; + } + return { body: {} }; + }); + + const stderr: string[] = []; + const result = await runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: (l: string) => stderr.push(l) }, + ); + + expect(result.summary.total).toBe(3); + expect(result.summary.created).toBe(3); + + // The captured request body should have 3 specs in name-sorted order. + const body = capturedBody as { tests: Array<{ name: string }> }; + expect(body.tests).toHaveLength(3); + expect(body.tests[0]!.name).toBe('Plan 0'); + expect(body.tests[1]!.name).toBe('Plan 1'); + expect(body.tests[2]!.name).toBe('Plan 2'); + + // Stderr should mention reading N plan files. + expect(stderr.some(l => l.includes('Reading 3 plan files'))).toBe(true); + }); + + it('rejects when dir does not exist', async () => { + const creds = makeCreds(); + await expect( + runCreateBatch( + { + plans: '', + planFromDir: '/tmp/definitely-does-not-exist-qw-test', + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects when dir has no *.json files', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-empty-')); + // Write a non-json file. + writeFileSync(join(dir, 'plan.txt'), 'not json'); + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects when a file contains invalid JSON', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-bad-')); + writeFileSync(join(dir, 'plan_00.json'), '{ not json }'); + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects when a file fails schema validation (missing planSteps)', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-bad-schema-')); + writeFileSync( + join(dir, 'plan_00.json'), + JSON.stringify({ projectId: 'p1', type: 'frontend', name: 'No steps' }), + ); + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects when both --plans and --plan-from-dir are supplied', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-mutual-')); + await expect( + runCreateBatch( + { + plans: '/tmp/some.jsonl', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('create-batch command exposes --plan-from-dir flag', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const batch = test.commands.find(c => c.name() === 'create-batch')!; + const flagNames = batch.options.map(o => o.long); + expect(flagNames).toContain('--plan-from-dir'); + }); + + it('Fix 2 — dir with suite-index.json + N valid plans ingests exactly N plans and emits a warn', async () => { + // suite-index.json is not a valid plan (missing planSteps, wrong shape). + // The old behavior aborted the whole batch; now it skips with a [warn]. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fix2-')); + + // 2 valid plan files + for (let i = 0; i < 2; i++) { + writeFileSync( + join(dir, `plan_0${i}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Fix2 Plan ${i}` }), + 'utf8', + ); + } + // suite-index.json — the problematic non-plan JSON file + writeFileSync( + join(dir, 'suite-index.json'), + JSON.stringify({ suiteId: 'wc-v1', tests: [] }), + 'utf8', + ); + + let capturedBody: unknown; + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch') && init.method === 'POST') { + capturedBody = JSON.parse(init.body as string); + return { body: batchCreateResp(2) }; + } + return { body: {} }; + }); + + const stderr: string[] = []; + const result = await runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: (l: string) => stderr.push(l) }, + ); + + // Only 2 valid plans should have been submitted. + expect(result.summary.total).toBe(2); + const body = capturedBody as { tests: unknown[] }; + expect(body.tests).toHaveLength(2); + + // A [warn] advisory should have been emitted for the skipped file. + const warnLine = stderr.find(l => l.includes('[warn]') && l.includes('suite-index.json')); + expect(warnLine).toBeDefined(); + // No fatal error — function returned normally. + }); + + it('Fix 2 — dir with ONLY invalid files still throws a fatal VALIDATION_ERROR', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fix2-all-bad-')); + writeFileSync( + join(dir, 'suite-index.json'), + JSON.stringify({ suiteId: 'x', tests: [] }), + 'utf8', + ); + writeFileSync(join(dir, 'meta.json'), JSON.stringify({ version: '1.0' }), 'utf8'); + + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + // --------------------------------------------------------------------------- + // Finding 2 — batch limit enforced on VALID specs AFTER skipping invalid files + // --------------------------------------------------------------------------- + + it('[finding-2] 50 valid plans + 1 invalid (suite-index.json) succeeds (not over-limit)', async () => { + // Before the fix, entries.length > MAX_BATCH_SPECS fired BEFORE the skip + // loop, so 50 valid + 1 invalid = 51 entries was rejected even though + // only 50 valid specs would be submitted. After the fix the check is on + // specs.length (valid only) after filtering. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-f2-51-')); + + // 50 valid plan files + for (let i = 0; i < 50; i++) { + writeFileSync( + join(dir, `plan_${String(i).padStart(2, '0')}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Batch50 Plan ${i}` }), + 'utf8', + ); + } + // 1 invalid (non-plan) file that should be skipped + writeFileSync( + join(dir, 'suite-index.json'), + JSON.stringify({ suiteId: 'wc-v1', tests: [] }), + 'utf8', + ); + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch') && init.method === 'POST') { + return { body: batchCreateResp(50) }; + } + return { body: {} }; + }); + + const stderr: string[] = []; + const result = await runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: (l: string) => stderr.push(l) }, + ); + + // Should have submitted exactly 50 valid specs + expect(result.summary.total).toBe(50); + // The suite-index.json should have been skipped with a [warn] + expect(stderr.some(l => l.includes('[warn]') && l.includes('suite-index.json'))).toBe(true); + }); + + it('[finding-2] 51 valid plans (no invalid files) still triggers the over-limit error', async () => { + // 51 genuine plan files should still be rejected — the limit on valid specs. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-f2-overlimit-')); + + for (let i = 0; i < 51; i++) { + writeFileSync( + join(dir, `plan_${String(i).padStart(2, '0')}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `OverLimit Plan ${i}` }), + 'utf8', + ); + } + + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + // --------------------------------------------------------------------------- + // Finding A (codex round-2) — malformed/truncated JSON must remain FATAL; + // only skip valid-JSON files that clearly lack plan identity. + // --------------------------------------------------------------------------- + + it('[finding-A] truncated/invalid JSON in plan file is FATAL (not silently skipped)', async () => { + // A truncated file (partial write, disk error) looks like an intended plan + // that got corrupted. Silently skipping it would let automation create an + // incomplete suite. Must abort the whole batch with VALIDATION_ERROR. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fa-truncated-')); + + // Write one good plan and one truncated "plan" file + writeFileSync( + join(dir, 'plan_00.json'), + JSON.stringify({ ...PLAN_SPEC, name: 'Good Plan' }), + 'utf8', + ); + writeFileSync(join(dir, 'plan_01.json'), '{"projectId": "p1", "planSteps": [', 'utf8'); + + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('[finding-A] a file with projectId but botched planSteps is FATAL (not skipped)', async () => { + // A file that has projectId (looks like a plan) but invalid planSteps is a + // botched/partial plan — must be FATAL so the user knows to fix it. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fa-botched-')); + + writeFileSync( + join(dir, 'plan_00.json'), + JSON.stringify({ + projectId: 'p1', + type: 'frontend', + name: 'Botched', + planSteps: 'not-array', + }), + 'utf8', + ); + + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('[finding-A] clearly-non-plan JSON (no projectId/planSteps) is skipped not fatal', async () => { + // A file with no plan-identity fields (e.g. a config/lock file accidentally + // placed in the plan directory) should be skipped with a [warn], not abort. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fa-nonplan-')); + + // Non-plan file: no projectId, no planSteps + writeFileSync( + join(dir, 'suite-index.json'), + JSON.stringify({ suiteId: 'v1', generatedAt: '2026-01-01', tests: [] }), + 'utf8', + ); + // One valid plan file to ensure the batch doesn't fail on "no valid plans" + writeFileSync( + join(dir, 'plan_00.json'), + JSON.stringify({ ...PLAN_SPEC, name: 'Real Plan' }), + 'utf8', + ); + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch') && init.method === 'POST') { + return { body: batchCreateResp(1) }; + } + return { body: {} }; + }); + const stderr: string[] = []; + const result = await runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: (l: string) => stderr.push(l) }, + ); + + // Only 1 valid plan submitted + expect(result.summary.total).toBe(1); + // A [warn] advisory for the skipped non-plan file + expect(stderr.some(l => l.includes('[warn]') && l.includes('suite-index.json'))).toBe(true); + }); + + it('[finding-A] a file with only planSteps (no projectId) is still skipped (clearly non-plan)', async () => { + // A file with planSteps but no projectId could be a step-template file, + // not a full plan spec. Heuristic: must have EITHER projectId OR planSteps + // to be treated as an intended plan. Since this has planSteps but no projectId, + // looksLikePlan is still true — it will attempt assertPlanShape and fail FATAL. + // Document that intent here so we don't accidentally weaken the gate. + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-pfd-fa-steps-only-')); + + writeFileSync( + join(dir, 'steps-template.json'), + JSON.stringify({ + planSteps: [{ type: 'action', description: 'Click' }], + // Missing projectId, type, name + }), + 'utf8', + ); + + await expect( + runCreateBatch( + { + plans: '', + planFromDir: dir, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); +}); + +// --------------------------------------------------------------------------- +// Piece 4: `test delete-batch` and `test delete-batch --all` +// --------------------------------------------------------------------------- + +describe('runDeleteBatch (dogfood L1796)', () => { + it('exit 5 when --confirm is not set', async () => { + const creds = makeCreds(); + await expect( + runDeleteBatch( + { + testIds: ['test_a', 'test_b'], + all: false, + confirm: false, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 when no testIds and --all is false', async () => { + const creds = makeCreds(); + await expect( + runDeleteBatch( + { + testIds: [], + all: false, + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 when --all is set but --project is missing', async () => { + const creds = makeCreds(); + await expect( + runDeleteBatch( + { + testIds: [], + all: true, + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('deletes explicit testIds sequentially, returns summary', async () => { + const creds = makeCreds(); + const deleted: string[] = []; + const fetchImpl = makeFetch((url, init) => { + if (init.method === 'DELETE' && url.includes('/tests/')) { + const testId = url.split('/tests/')[1]!.split('?')[0]!; + deleted.push(testId); + return { body: deleteResp(decodeURIComponent(testId)) }; + } + return { body: {} }; + }); + + const stderr: string[] = []; + const result = await runDeleteBatch( + { + testIds: ['test_a', 'test_b'], + all: false, + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: (l: string) => stderr.push(l) }, + ); + + expect(result.summary).toEqual({ total: 2, deleted: 2, skipped: 0, failed: 0 }); + expect(deleted.sort()).toEqual(['test_a', 'test_b'].sort()); + // stderr summary + expect(stderr.some(l => l.includes('Deleted 2'))).toBe(true); + }); + + it('counts 404 as skipped, not error, and still returns 0 exit', async () => { + const creds = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + if (init.method === 'DELETE') { + if (url.includes('test_gone')) { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'not found', + nextAction: '', + requestId: 'r1', + details: {}, + }, + }, + }; + } + const testId = url.split('/tests/')[1]!.split('?')[0]!; + return { body: deleteResp(testId) }; + } + return { body: {} }; + }); + + const result = await runDeleteBatch( + { + testIds: ['test_ok', 'test_gone'], + all: false, + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: () => {} }, + ); + + expect(result.summary).toEqual({ total: 2, deleted: 1, skipped: 1, failed: 0 }); + const skipped = result.results.find(r => r.testId === 'test_gone'); + expect(skipped?.status).toBe('skipped'); + }); + + it('--dry-run skips all HTTP calls and returns full synthetic summary', async () => { + const creds = makeCreds(); + const callCount = { n: 0 }; + const fetchImpl = makeFetch(() => { + callCount.n++; + return { body: {} }; + }); + + const result = await runDeleteBatch( + { + testIds: ['test_a', 'test_b', 'test_c'], + all: false, + confirm: false, // --dry-run skips confirm check + output: 'json', + profile: 'default', + dryRun: true, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: () => {} }, + ); + + // No HTTP calls should have been made. + expect(callCount.n).toBe(0); + expect(result.summary).toEqual({ total: 3, deleted: 3, skipped: 0, failed: 0 }); + }); + + it('--all resolves project tests and deletes all of them', async () => { + const creds = makeCreds(); + const deletedIds: string[] = []; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && (!init.method || init.method === 'GET')) { + return { + body: { + items: [ + { id: 'test_a', status: 'passed' }, + { id: 'test_b', status: 'failed' }, + ], + nextToken: null, + }, + }; + } + if (init.method === 'DELETE') { + const testId = url.split('/tests/')[1]!.split('?')[0]!; + deletedIds.push(decodeURIComponent(testId)); + return { body: deleteResp(decodeURIComponent(testId)) }; + } + return { body: {} }; + }); + + const result: CliBulkDeleteSummary = await runDeleteBatch( + { + testIds: [], + all: true, + projectId: 'project_abc', + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: () => {} }, + ); + + expect(deletedIds.sort()).toEqual(['test_a', 'test_b'].sort()); + expect(result.summary).toEqual({ total: 2, deleted: 2, skipped: 0, failed: 0 }); + }); + + it('--all with --status filter only deletes matching tests', async () => { + const creds = makeCreds(); + const deletedIds: string[] = []; + + const allTests = [ + { id: 'test_passed', status: 'passed' }, + { id: 'test_failed', status: 'failed' }, + { id: 'test_running', status: 'running' }, + ]; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && (!init.method || init.method === 'GET')) { + return { body: { items: allTests, nextToken: null } }; + } + if (init.method === 'DELETE') { + const testId = url.split('/tests/')[1]!.split('?')[0]!; + deletedIds.push(decodeURIComponent(testId)); + return { body: deleteResp(decodeURIComponent(testId)) }; + } + return { body: {} }; + }); + + await runDeleteBatch( + { + testIds: [], + all: true, + projectId: 'project_abc', + statusFilter: 'failed', + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl, stderr: () => {} }, + ); + + // Only the failed test should be deleted. + expect(deletedIds).toEqual(['test_failed']); + }); + + it('exit 5 when --status contains an invalid token', async () => { + const creds = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && (!init.method || init.method === 'GET')) { + return { body: { items: [], nextToken: null } }; + } + return { body: {} }; + }); + await expect( + runDeleteBatch( + { + testIds: [], + all: true, + projectId: 'project_abc', + statusFilter: 'notastatus', + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + // ---- correctness-fix tests (code-review 2026-06-04) ---------------------- + + it('exit 5 when explicit IDs and --all are both supplied (data-loss guard)', async () => { + const creds = makeCreds(); + // Passing both explicit test IDs and --all is ambiguous; the CLI must + // reject early with VALIDATION_ERROR rather than silently discarding the + // explicit IDs and wiping the whole project. + await expect( + runDeleteBatch( + { + testIds: ['test_a', 'test_b'], + all: true, + projectId: 'project_abc', + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + // localValidationError always sets message: 'Invalid request.'; the + // readable explanation lives in nextAction. + nextAction: expect.stringContaining('--all'), + }); + }); + + it('exit 5 when --status is set without --all (would be silently ignored)', async () => { + const creds = makeCreds(); + // --status only has meaning in the --all path (it filters the project-wide + // listing). Without --all the flag would be silently discarded; reject instead. + await expect( + runDeleteBatch( + { + testIds: ['test_a'], + all: false, + statusFilter: 'failed', + confirm: true, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + creds, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + // localValidationError always sets message: 'Invalid request.'; the + // readable explanation lives in nextAction. + nextAction: expect.stringContaining('--status'), + }); + }); + + it('--dry-run --all emits a warning that the preview uses sample data', async () => { + const creds = makeCreds(); + const stderrLines: string[] = []; + + // The dry-run client uses canned responses regardless of the real project, + // so the "would delete" list is misleading. A clear warning must be emitted. + await runDeleteBatch( + { + testIds: [], + all: true, + projectId: 'project_abc', + confirm: false, // --dry-run skips the confirm gate + output: 'json', + profile: 'default', + dryRun: true, + debug: false, + verbose: false, + }, + { ...creds, stderr: (l: string) => stderrLines.push(l) }, + ); + + const warnLine = stderrLines.find(l => l.includes('sample data')); + expect(warnLine).toBeDefined(); + expect(warnLine).toMatch(/does NOT reflect/); + }); + + it('delete-batch command is registered in the test command', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const names = test.commands.map(c => c.name()); + expect(names).toContain('delete-batch'); + }); + + it('delete-batch command exposes expected flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const batch = test.commands.find(c => c.name() === 'delete-batch')!; + const flagNames = batch.options.map(o => o.long); + expect(flagNames).toContain('--confirm'); + expect(flagNames).toContain('--all'); + expect(flagNames).toContain('--project'); + expect(flagNames).toContain('--status'); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts new file mode 100644 index 0000000..e33634b --- /dev/null +++ b/src/commands/test.rerun.spec.ts @@ -0,0 +1,4477 @@ +/** + * Unit tests for `test rerun` — M3.4 piece-3. + * + * All HTTP is mocked via `makeFetch`. The polling loop's sleep injection is + * wired through `TestDeps.sleep` to avoid real delays. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; +import type { FetchImpl } from '../lib/http.js'; +import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-m34-rerun-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +/** Instant sleep — avoids real delays in tests. */ +const instantSleep = () => Promise.resolve(); + +/** Canned FE test record (frontend type). */ +const FE_TEST = { + id: 'test_fe_01', + projectId: 'project_abc', + name: 'FE checkout test', + type: 'frontend' as const, + createdFrom: 'portal' as const, + status: 'passed' as const, + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', +}; + +/** Canned BE test record (backend type). */ +const BE_TEST = { + id: 'test_be_consumer_01', + projectId: 'project_abc', + name: 'BE consumer test', + type: 'backend' as const, + createdFrom: 'portal' as const, + status: 'passed' as const, + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', +}; + +/** Build a FE RerunResponse (no closure). */ +function makeFeRerunResp(overrides?: Partial): RerunResponse { + return { + runId: 'run_rerun_fe_001', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + ...overrides, + }; +} + +/** Build a BE RerunResponse with closure. */ +function makeBeRerunResp(overrides?: Partial): RerunResponse { + return { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [ + { testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }, + { testId: 'test_be_producer_01', runId: 'run_rerun_be_producer', role: 'producer' }, + ], + addedProducers: ['test_be_producer_01'], + addedTeardowns: [], + clearedCaptured: 0, + }, + ...overrides, + }; +} + +/** Build a terminal RunResponse. */ +function makeTerminalRun( + runId: string, + status: 'passed' | 'failed' | 'blocked' = 'passed', +): RunResponse { + return { + runId, + testId: 'test_fe_01', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'rerun:prior_run_01', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { + total: 5, + completed: 5, + passedCount: status === 'passed' ? 5 : 4, + failedCount: status === 'passed' ? 0 : 1, + }, + }; +} + +function errorBody( + code: string, + details: Record = {}, +): { status: number; body: unknown } { + const statusMap: Record = { + AUTH_REQUIRED: 401, + NOT_FOUND: 404, + VALIDATION_ERROR: 400, + CONFLICT: 409, + RATE_LIMITED: 429, + INTERNAL: 500, + UNAVAILABLE: 503, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +// --------------------------------------------------------------------------- +// Surface tests (command registration) +// --------------------------------------------------------------------------- + +describe('createTestCommand — rerun subcommand exposed', () => { + it('exposes rerun as a top-level test subcommand', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const names = test.commands.map(c => c.name()).sort(); + expect(names).toContain('rerun'); + }); + + it('rerun has expected flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const rerun = test.commands.find(c => c.name() === 'rerun')!; + const flagNames = rerun.options.map(o => o.long); + expect(flagNames).toContain('--all'); + expect(flagNames).toContain('--project'); + expect(flagNames).toContain('--wait'); + expect(flagNames).toContain('--timeout'); + expect(flagNames).toContain('--no-auto-heal'); + expect(flagNames).toContain('--skip-dependencies'); + expect(flagNames).toContain('--max-concurrency'); + expect(flagNames).toContain('--idempotency-key'); + }); +}); + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +describe('runTestRerun — validation', () => { + it('exit 5 when no testIds and no --all', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: [], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toThrow(ApiError); + + try { + await runTestRerun( + { + testIds: [], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ); + } catch (err) { + expect(err instanceof ApiError).toBe(true); + if (err instanceof ApiError) { + expect(err.code).toBe('VALIDATION_ERROR'); + } + } + }); + + it('exit 5 (VALIDATION_ERROR) when --filter is passed WITHOUT --all', async () => { + // --filter is an --all-only narrowing filter. With explicit ids it would be + // silently ignored while the named test still reran (codex finding). The + // guard throws BEFORE any network/dispatch, so no rerun is triggered. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + nameFilter: 'checkout', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 when --all without --project', async () => { + const creds = makeCreds(); + try { + await runTestRerun( + { + testIds: [], + all: true, + projectId: undefined, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ); + expect.fail('should have thrown'); + } catch (err) { + expect(err instanceof ApiError).toBe(true); + if (err instanceof ApiError) { + expect(err.code).toBe('VALIDATION_ERROR'); + } + } + }); + + it('rejects --max-concurrency > 100 with VALIDATION_ERROR (exit 5)', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 101, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'max-concurrency' }), + }); + }); + + it('accepts --max-concurrency = 100 (no validation error at boundary)', async () => { + const creds = makeCreds(); + const rerunResp = { + runId: 'run_mc100', + status: 'queued', + enqueuedAt: '2026-06-09T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: true, + closure: null, + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/runs/rerun')) return { body: rerunResp }; + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: '', nextAction: '', requestId: 'r', details: {} }, + }, + }; + }); + // Should not throw a VALIDATION_ERROR + await expect( + runTestRerun( + { + testIds: ['test_mc100'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 100, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + }, + ), + ).resolves.toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// R-FE1: Single FE rerun happy path (no --wait) +// --------------------------------------------------------------------------- + +describe('R-FE1: FE rerun — queued (no --wait)', () => { + it('returns the trigger response with status=queued (no credit language)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + const result = printed[0] as RerunResponse; + expect(result.runId).toBe('run_rerun_fe_001'); + expect(result.status).toBe('queued'); + // No credit language: autoHeal false, no closure + expect(result.closure).toBeUndefined(); + expect(result.autoHeal).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// R-FE1: Single FE rerun with --wait +// --------------------------------------------------------------------------- + +describe('R-FE1: FE rerun -- wait (replay, exit 0 on passed)', () => { + it('polls runId to terminal and exits 0 on passed', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const terminalRun = makeTerminalRun('run_rerun_fe_001', 'passed'); + const printed: unknown[] = []; + let pollCount = 0; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/runs/run_rerun_fe_001')) { + pollCount++; + return { body: terminalRun }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect(pollCount).toBeGreaterThan(0); + const result = printed[0] as RunResponse; + expect(result.status).toBe('passed'); + }); + + it('--wait exits 1 on failed, suggests artifact get', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const failedRun = makeTerminalRun('run_rerun_fe_001', 'failed'); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/runs/run_rerun_fe_001')) { + return { body: failedRun }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + expect.fail('should have thrown CLIError'); + } catch (err: unknown) { + // CLIError with code 1 + expect((err as { exitCode?: number }).exitCode ?? (err as ApiError).httpStatus).toBeTruthy(); + } + // nextAction should mention artifact get + const artifactHint = stderrLines.find(l => l.includes('artifact get')); + expect(artifactHint).toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// R-FE2/R-FE4: --auto-heal flag +// --------------------------------------------------------------------------- + +describe('R-FE2: auto-heal forwarded for FE paid', () => { + it('sends autoHeal=true in request body when paid (server echoes effective value)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: true }); // server echo + const printed: unknown[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(true); + const result = printed[0] as RerunResponse; + expect(result.autoHeal).toBe(true); + }); +}); + +// R-FE0: default-on — no --no-auto-heal flag → body sends autoHeal:true; advisory emitted +describe('R-FE0: auto-heal default-on (no --no-auto-heal flag)', () => { + it('sends autoHeal:true by default; emits 0.2-credit advisory when server echoes autoHeal:true', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: true }); // server confirms auto-heal + const stderrLines: string[] = []; + const printed: unknown[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, // default value — user did NOT pass --no-auto-heal + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + stderr: line => stderrLines.push(line), + }, + ); + + // Body must include autoHeal:true + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(true); + + // Advisory must mention 0.2 credit and --no-auto-heal + const advisory = stderrLines.find( + l => l.includes('[advisory]') && l.includes('0.2') && l.includes('--no-auto-heal'), + ); + expect(advisory).toBeDefined(); + }); +}); + +describe('R-FE4: server unexpectedly echoes autoHeal:false — prints "not applied" advisory; no "Pro plan" language', () => { + it('server echoes autoHeal:false; CLI prints defensive advisory without Pro-plan claim', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: false }); // server did not apply auto-heal + const stderrLines: string[] = []; + const terminalRun = makeTerminalRun('run_rerun_fe_001', 'passed'); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/runs/run_rerun_fe_001')) { + return { body: terminalRun }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: true, // default-on (user did not pass --no-auto-heal) + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // CLI prints the defensive advisory since server echoed autoHeal:false. + // Must NOT say "requires Pro plan" (CLI path has no paid gate). + const advisory = stderrLines.find(l => l.includes('[advisory]') && l.includes('not applied')); + expect(advisory).toBeDefined(); + + // Must not use Pro-plan language on the CLI path + const proLine = stderrLines.find(l => l.toLowerCase().includes('pro plan')); + expect(proLine).toBeUndefined(); + }); +}); + +// R-FE5: --no-auto-heal explicit opt-out → body sends no autoHeal field +describe('R-FE5: --no-auto-heal opt-out', () => { + it('does NOT send autoHeal in body when autoHeal:false; no advisory emitted', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: false }); // verbatim replay + const stderrLines: string[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, // user passed --no-auto-heal + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // autoHeal must NOT be sent to server (effectiveAutoHeal is false) + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + + // No advisory for a verbatim replay (server echoes false, opts.autoHeal is false) + const advisory = stderrLines.find(l => l.includes('[advisory]')); + expect(advisory).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// R-BE1: BE rerun with closure +// --------------------------------------------------------------------------- + +describe('R-BE1: BE rerun — prints closure + polls all members + exits on named test', () => { + it('happy path: all members pass; exit 0', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRun = makeTerminalRun('run_rerun_be_named', 'passed'); + const producerRun = makeTerminalRun('run_rerun_be_producer', 'passed'); + namedRun.testId = 'test_be_consumer_01'; + producerRun.testId = 'test_be_producer_01'; + const stderrLines: string[] = []; + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + if (url.includes('/runs/run_rerun_be_named')) { + return { body: namedRun }; + } + if (url.includes('/runs/run_rerun_be_producer')) { + return { body: producerRun }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + // Should print closure summary + const closureLine = stderrLines.find(l => l.includes('Reran') && l.includes('producer')); + expect(closureLine).toBeDefined(); + }); + + it('C2 stale-pass guard: producer already passed before rerun — waits for NEW runId', async () => { + // A BE producer that has a pre-existing `passed` status should NOT short-circuit + // the --wait poll — the poll anchors on the specific runId from the response + // (not `GET /tests/{id}/result`), so a stale prior verdict cannot be accepted. + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + // Named test: queued then passed. Producer: queued then passed. + let namedPollCount = 0; + let producerPollCount = 0; + const namedRun = makeTerminalRun('run_rerun_be_named', 'passed'); + namedRun.testId = 'test_be_consumer_01'; + const producerRun = makeTerminalRun('run_rerun_be_producer', 'passed'); + producerRun.testId = 'test_be_producer_01'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + if (url.includes('/runs/run_rerun_be_named')) { + namedPollCount++; + // First tick: still queued (not the stale prior result) + if (namedPollCount === 1) return { body: { ...namedRun, status: 'queued' } }; + return { body: namedRun }; + } + if (url.includes('/runs/run_rerun_be_producer')) { + producerPollCount++; + if (producerPollCount === 1) return { body: { ...producerRun, status: 'queued' } }; + return { body: producerRun }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + + // Both runIds were polled (not the stale test result) + expect(namedPollCount).toBeGreaterThan(1); + expect(producerPollCount).toBeGreaterThan(1); + }); + + it('closure member fails: closureFailures[] surfaced; exit keys on named test (passed)', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRun = makeTerminalRun('run_rerun_be_named', 'passed'); + namedRun.testId = 'test_be_consumer_01'; + const producerRun = makeTerminalRun('run_rerun_be_producer', 'failed'); // member fails + producerRun.testId = 'test_be_producer_01'; + const stderrLines: string[] = []; + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + if (url.includes('/runs/run_rerun_be_named')) { + return { body: namedRun }; + } + if (url.includes('/runs/run_rerun_be_producer')) { + return { body: producerRun }; + } + return errorBody('NOT_FOUND'); + }); + + // Named test passes even though producer fails → exit 0 + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + // Closure failure warning emitted + const warningLine = stderrLines.find(l => l.includes('closure member') && l.includes('failed')); + expect(warningLine).toBeDefined(); + + // closureFailures[] in JSON output + const jsonOutput = printed[0] as { closureFailures?: unknown[] }; + expect(Array.isArray(jsonOutput.closureFailures)).toBe(true); + expect((jsonOutput.closureFailures as unknown[]).length).toBeGreaterThan(0); + }); + + it('named test fails: exits 1 + suggests artifact get', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRun = makeTerminalRun('run_rerun_be_named', 'failed'); + namedRun.testId = 'test_be_consumer_01'; + const producerRun = makeTerminalRun('run_rerun_be_producer', 'passed'); + producerRun.testId = 'test_be_producer_01'; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + if (url.includes('/runs/run_rerun_be_named')) { + return { body: namedRun }; + } + if (url.includes('/runs/run_rerun_be_producer')) { + return { body: producerRun }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as { exitCode?: number }).exitCode).toBe(1); + } + }); +}); + +// --------------------------------------------------------------------------- +// R-BE2: --skip-dependencies +// --------------------------------------------------------------------------- + +describe('R-BE2: --skip-dependencies', () => { + it('sends skipDependencies:true in request body', async () => { + const creds = makeCreds(); + // BE rerun with no closure (skipDependencies=true → server returns minimal response) + const rerunResp = makeFeRerunResp({ runId: 'run_rerun_be_nodeps' }); // FE-shape (no closure) + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: true, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + + expect((sentBody as { skipDependencies?: boolean }).skipDependencies).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// R-BE3: auto-heal on BE test — suppressed or warned depending on explicit flag +// --------------------------------------------------------------------------- + +describe('R-BE3: auto-heal on BE test — default-on suppresses warning', () => { + it('auto-heal defaults true; BE type suppresses warning (autoHealExplicit:false); autoHeal NOT sent', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + // autoHeal:true is the default (user did NOT pass --no-auto-heal) + autoHeal: true, + // autoHealExplicit:false means we suppress the BE "ignoring" warning + // to avoid noise on every default BE rerun. + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // With autoHealExplicit:false, the "ignoring auto-heal" advisory must NOT be emitted + const warning = stderrLines.find( + l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + ); + expect(warning).toBeUndefined(); + + // autoHeal must NOT be sent to server (effectiveAutoHeal is false for BE) + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + }); + + it('auto-heal explicitly requested (autoHealExplicit:true); BE type emits warning', async () => { + // This covers a hypothetical future scenario where a user can explicitly + // pass --auto-heal. Since there's no such flag currently, autoHealExplicit + // can be set to true only by callers that inject it directly (e.g. tests or + // future flag additions). + const creds = makeCreds(); + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, + autoHealExplicit: true, // user explicitly requested --auto-heal + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // With autoHealExplicit:true, the warning IS emitted + const warning = stderrLines.find( + l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + ); + expect(warning).toBeDefined(); + + // autoHeal must NOT be sent to server (effectiveAutoHeal is false for BE) + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// FIX 2 — BE rerun: no "not applied" advisory (effectiveAutoHeal fix) +// --------------------------------------------------------------------------- +// D2-FIX2: Before the fix, `opts.autoHeal && !rerunResp.autoHeal` fired on +// every BE rerun because opts.autoHeal is default-on true but the server always +// echoes autoHeal:false for BE tests. The fix changes the guard to +// `effectiveAutoHeal && !rerunResp.autoHeal` so BE reruns (effectiveAutoHeal=false) +// never trigger the advisory. +describe('[fix-2] BE rerun: spurious "not applied" advisory is suppressed', () => { + it('BE rerun with default-on autoHeal does NOT print "not applied" advisory', async () => { + const creds = makeCreds(); + // Server echoes autoHeal:false for BE (expected) — before the fix, this + // would trigger the defensive advisory every time. + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, // default-on (user did NOT pass --no-auto-heal) + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // The "not applied" advisory MUST NOT fire for BE reruns (FIX 2). + // effectiveAutoHeal is false for BE, so the guard is: false && !false → false. + const notAppliedLine = stderrLines.find( + l => l.includes('not applied') || l.includes('was not applied'), + ); + expect(notAppliedLine).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// R-BAT: Batch rerun +// --------------------------------------------------------------------------- + +describe('R-BAT: batch rerun (multi-id, no --wait)', () => { + it('sends testIds to POST /tests/batch/rerun and prints accepted', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const printed: unknown[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { status: 202, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect((sentBody as { testIds: string[] }).testIds).toEqual(['test_1', 'test_2']); + const result = printed[0] as BatchRerunResponse; + expect(result.accepted).toHaveLength(2); + expect(result.accepted[0]!.runId).toBe('run_b1'); + expect(result.accepted[1]!.runId).toBe('run_b2'); + }); +}); + +describe('R-BAT: batch rerun + --wait fan-out poll by runId (C2)', () => { + it('polls each accepted runId individually; aggregate exit 0 on all passed', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const pollCounts: Record = {}; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + const run2 = makeTerminalRun('run_b2', 'passed'); + run2.testId = 'test_2'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/run_b1')) { + pollCounts['run_b1'] = (pollCounts['run_b1'] ?? 0) + 1; + return { body: run1 }; + } + if (url.includes('/runs/run_b2')) { + pollCounts['run_b2'] = (pollCounts['run_b2'] ?? 0) + 1; + return { body: run2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + + // Both runIds were polled individually (run-scoped, not testId-latest) + expect(pollCounts['run_b1']).toBeGreaterThan(0); + expect(pollCounts['run_b2']).toBeGreaterThan(0); + }); + + it('partial conflict: accepted proceed, conflicts reported', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [], + conflicts: [{ testId: 'test_conflict', currentRunId: 'run_inflight_01' }], + closure: { byProject: [] }, + }; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/run_b1')) { + return { body: run1 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_1', 'test_conflict'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // Conflict should be mentioned in summary + const conflictMention = stderrLines.find( + l => l.includes('in flight') || l.includes('conflict'), + ); + expect(conflictMention).toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// C1: deferred[] → exit 7 + retry nextAction +// --------------------------------------------------------------------------- + +describe('C1: deferred[] → exit 7 + retry nextAction', () => { + it('no --wait: exits 7 when deferred[] non-empty', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_1', 'test_deferred'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + expect.fail('should have thrown'); + } catch (err: unknown) { + // Exit 7 for incomplete run + const exitCode = (err as { exitCode?: number }).exitCode; + expect(exitCode).toBe(7); + } + + // Retry nextAction should be printed + const retryLine = stderrLines.find(l => l.includes('test_deferred')); + expect(retryLine).toBeDefined(); + }); + + it('with --wait: exits 7 when deferred[] non-empty even if accepted all passed', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/run_b1')) { + return { body: run1 }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_1', 'test_deferred'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + expect.fail('should have thrown'); + } catch (err: unknown) { + const exitCode = (err as { exitCode?: number }).exitCode; + // exit 7 because deferred makes run incomplete + expect(exitCode).toBe(7); + } + }); +}); + +// --------------------------------------------------------------------------- +// --all flag +// --------------------------------------------------------------------------- + +describe('--all: resolves project tests and batch reruns', () => { + it('lists tests then calls batch/rerun with all testIds', async () => { + const creds = makeCreds(); + const projectTests = [ + { ...FE_TEST, id: 'test_p1' }, + { ...FE_TEST, id: 'test_p2' }, + ]; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_p1', runId: 'run_p1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_p2', runId: 'run_p2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let sentBatchBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && !url.includes('batch') && !url.includes('/runs')) { + return { body: { items: projectTests, nextToken: null } }; + } + if (url.includes('/tests/batch/rerun')) { + sentBatchBody = init.body ? JSON.parse(init.body as string) : null; + return { status: 202, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + + expect((sentBatchBody as { testIds: string[] }).testIds).toEqual(['test_p1', 'test_p2']); + }); +}); + +// --------------------------------------------------------------------------- +// 409 CONFLICT (single run in flight) +// --------------------------------------------------------------------------- + +describe('409 CONFLICT → exit 6 with nextAction', () => { + it('in-flight test → CONFLICT error with nextAction: test wait ', async () => { + const creds = makeCreds(); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return errorBody('CONFLICT', { reason: 'run_in_flight', currentRunId: 'run_in_flight_01' }); + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown'); + } catch (err) { + expect(err instanceof ApiError).toBe(true); + if (err instanceof ApiError) { + expect(err.code).toBe('CONFLICT'); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// --idempotency-key passthrough +// --------------------------------------------------------------------------- + +describe('--idempotency-key passthrough', () => { + it('sends the supplied idempotency key as Idempotency-Key header', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + let receivedKey: string | null = null; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + const headers = init.headers as Record; + receivedKey = headers['idempotency-key'] ?? null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + idempotencyKey: 'my-custom-key-abc', + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + + expect(receivedKey).toBe('my-custom-key-abc'); + }); +}); + +// --------------------------------------------------------------------------- +// --timeout → exit 7 with resume nextAction +// --------------------------------------------------------------------------- + +describe('--wait --timeout exceeded → exit 7 + nextAction', () => { + it('single FE: exits 7 when run does not reach terminal within timeout', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const nonTerminalRun = makeTerminalRun('run_rerun_fe_001', 'passed'); + nonTerminalRun.status = 'running' as unknown as 'passed'; // cast to simulate non-terminal + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/runs/run_rerun_fe_001')) { + // Never becomes terminal → will cause timeout + return { + body: { + ...nonTerminalRun, + status: 'queued', + retryAfterSeconds: 0, + }, + }; + } + if (url.includes('/tests/test_fe_01')) { + return { body: FE_TEST }; // FE type so fallback doesn't fire + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 1, // Very short timeout + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown'); + } catch (err) { + expect(err instanceof ApiError).toBe(true); + if (err instanceof ApiError) { + expect(err.code).toBe('UNSUPPORTED'); // exit 7 per errors.md + expect(err.message).toContain('Timed out'); + // nextAction should suggest test wait + const nextAction = err.getDetail( + 'runId', + (v): v is string => typeof v === 'string', + ); + expect(nextAction ?? err.message).toContain('run_rerun_fe_001'); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// --output json +// --------------------------------------------------------------------------- + +describe('--output json', () => { + it('single FE no --wait: prints parseable JSON', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + let rawLine: string | undefined; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => { + rawLine = line; + }, + }, + ); + + expect(rawLine).toBeDefined(); + const parsed = JSON.parse(rawLine!); + expect(parsed.runId).toBe('run_rerun_fe_001'); + expect(parsed.status).toBe('queued'); + }); +}); + +// --------------------------------------------------------------------------- +// --dry-run +// --------------------------------------------------------------------------- + +describe('--dry-run', () => { + it('single FE: emits canned sample (no network call)', async () => { + const creds = makeCreds(); + const printed: unknown[] = []; + let networkCalled = false; + + const fetchImpl = makeFetch(() => { + networkCalled = true; + return errorBody('INTERNAL'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: true, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect(networkCalled).toBe(false); + expect(printed.length).toBeGreaterThan(0); + }); + + it('batch: emits canned sample', async () => { + const creds = makeCreds(); + const printed: unknown[] = []; + + await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: true, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect(printed.length).toBeGreaterThan(0); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 1 — --all auto-pages through nextToken +// --------------------------------------------------------------------------- + +describe('[fix-1] --all follows nextToken to collect ALL project tests', () => { + it('fetches page 2 when page 1 has a nextToken', async () => { + const creds = makeCreds(); + const page1Tests = [ + { ...FE_TEST, id: 'test_page1_a' }, + { ...FE_TEST, id: 'test_page1_b' }, + ]; + const page2Tests = [ + { ...FE_TEST, id: 'test_page2_a' }, + { ...FE_TEST, id: 'test_page2_b' }, + ]; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_page1_a', runId: 'run_pa', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_page1_b', runId: 'run_pb', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_page2_a', runId: 'run_pc', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_page2_b', runId: 'run_pd', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let listCallCount = 0; + let sentBatchBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && !url.includes('batch') && !url.includes('/runs')) { + listCallCount++; + // First page returns a nextToken; second page returns null. + if (url.includes('cursor=page2-cursor') || listCallCount >= 2) { + return { body: { items: page2Tests, nextToken: null } }; + } + return { body: { items: page1Tests, nextToken: 'page2-cursor' } }; + } + if (url.includes('/tests/batch/rerun')) { + sentBatchBody = init.body ? JSON.parse(init.body as string) : null; + return { status: 202, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + + // Both pages were fetched + expect(listCallCount).toBeGreaterThanOrEqual(2); + // Batch rerun received all 4 test ids (from both pages) + const sentIds = (sentBatchBody as { testIds: string[] }).testIds; + expect(sentIds).toContain('test_page1_a'); + expect(sentIds).toContain('test_page1_b'); + expect(sentIds).toContain('test_page2_a'); + expect(sentIds).toContain('test_page2_b'); + expect(sentIds).toHaveLength(4); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 2 — cancelled closure rerun treated as failure +// --------------------------------------------------------------------------- + +describe('[fix-2] cancelled named BE closure run treated as failure (not success)', () => { + it('named test finishing cancelled → exits 1', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + // named run lands as cancelled + const namedRun = { + ...makeTerminalRun('run_rerun_be_named'), + status: 'cancelled' as 'passed' | 'failed' | 'blocked', + testId: 'test_be_consumer_01', + }; + const producerRun = makeTerminalRun('run_rerun_be_producer', 'passed'); + producerRun.testId = 'test_be_producer_01'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + if (url.includes('/runs/run_rerun_be_named')) { + return { body: namedRun }; + } + if (url.includes('/runs/run_rerun_be_producer')) { + return { body: producerRun }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown on cancelled status'); + } catch (err: unknown) { + expect((err as { exitCode?: number }).exitCode).toBe(1); + } + }); +}); + +// --------------------------------------------------------------------------- +// Fix 3 — max-concurrency 0 / negative rejected with exit 5 +// --------------------------------------------------------------------------- + +describe('[fix-3] --max-concurrency 0 / negative rejected before polling', () => { + it('maxConcurrency 0 → VALIDATION_ERROR (exit 5)', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 0, // invalid + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('maxConcurrency -5 → VALIDATION_ERROR (exit 5)', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: -5, // invalid + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('maxConcurrency 1 → accepted (minimum valid)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await expect( + runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, // minimum valid + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ), + ).resolves.toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 4 — one `test wait ` per timed-out run in nextAction +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Fix B — timed-out closure members treated as incomplete reruns (exit 7) +// --------------------------------------------------------------------------- + +describe('[fix-B] timed-out BE closure member → exit 7, not silent success', () => { + it('non-named closure member timeout → closureFailures["timeout"] + exit 7', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRunId = rerunResp.runId; // 'run_rerun_be_named' + const producerRunId = rerunResp.closure!.members.find(m => m.role === 'producer')!.runId; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { status: 202, body: rerunResp }; + } + if (url.includes('/tests/test_be_consumer_01')) { + return { body: BE_TEST }; + } + // Named run reaches terminal (passed) immediately + if (url.includes(`/runs/${namedRunId}`)) { + return { body: makeTerminalRun(namedRunId, 'passed') }; + } + // Producer run stays non-terminal → will time out + if (url.includes(`/runs/${producerRunId}`)) { + return { + body: { + ...makeTerminalRun(producerRunId, 'passed'), + status: 'executing', // non-terminal, causes timeout + }, + }; + } + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 1, // short → producer times out + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ).catch(e => e); + + // Must exit 7 (UNSUPPORTED = timeout) + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('UNSUPPORTED'); + expect((err as ApiError).exitCode).toBe(7); + // nextAction should reference the timed-out producer run + expect((err as ApiError).nextAction).toContain(producerRunId); + }); +}); + +// --------------------------------------------------------------------------- +// Fix C — all-conflict batch rerun → exit 6 CONFLICT (not exit 0) +// --------------------------------------------------------------------------- + +describe('[fix-C] batch rerun: every test in-flight → CONFLICT exit 6', () => { + it('--wait: accepted=[], conflicts non-empty → exits 6', async () => { + const creds = makeCreds(); + const allConflictResp: BatchRerunResponse = { + accepted: [], + deferred: [], + conflicts: [ + { testId: 'test_1', currentRunId: 'run_inflight_1' }, + { testId: 'test_2', currentRunId: 'run_inflight_2' }, + ], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: allConflictResp }; + } + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + expect((err as ApiError).message).toMatch(/in flight/i); + }); + + it('no --wait: accepted=[], conflicts non-empty → exits 6', async () => { + const creds = makeCreds(); + // Need ≥2 testIds to force the batch path (single-id takes the single path). + const allConflictResp: BatchRerunResponse = { + accepted: [], + deferred: [], + conflicts: [ + { testId: 'test_1', currentRunId: 'run_inflight_1' }, + { testId: 'test_2', currentRunId: 'run_inflight_2' }, + ], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: allConflictResp }; + } + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + }); + + it('[codex-P2] no --wait: accepted=[], conflicts + notFound → CONFLICT names both causes (not "all in flight")', async () => { + const creds = makeCreds(); + const mixedResp: BatchRerunResponse = { + accepted: [], + deferred: [], + conflicts: [{ testId: 'test_1', currentRunId: 'run_inflight_1' }], + closure: { byProject: [] }, + notFound: ['test_bad_x', 'test_bad_y'], + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: mixedResp }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_bad_x', 'test_bad_y'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + // codex P2: a mixed conflicts+notFound response (no accepted runs) must name + // BOTH causes — it must not be misreported as "all ... already in flight". + expect((err as ApiError).message).toContain('not found'); + expect((err as ApiError).message).toContain('already in flight'); + }); + + it('partial conflict (some accepted + some conflicts) still exits 0 on all-passed', async () => { + const creds = makeCreds(); + const partialConflictResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_new_1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [], + conflicts: [{ testId: 'test_2', currentRunId: 'run_inflight_2' }], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: partialConflictResp }; + } + if (url.includes('/runs/run_new_1')) { + return { body: makeTerminalRun('run_new_1', 'passed') }; + } + if (url.includes('/tests/test_1')) { + return { body: FE_TEST }; + } + return errorBody('NOT_FOUND'); + }); + + // Must NOT throw — partial conflict with accepted runs is not an error + const result = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect(result).toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Fix D — --all with >50 tests: chunk into ≤50-id requests, aggregate +// --------------------------------------------------------------------------- + +describe('[fix-D] --all resolves >50 tests: chunked batch requests, aggregated result', () => { + it('60 tests → 2 batch requests (chunk 50 + chunk 10), result aggregated', async () => { + const creds = makeCreds(); + + // Build 60 test stubs + const allTests = Array.from({ length: 60 }, (_, i) => ({ + ...FE_TEST, + id: `test_bulk_${String(i).padStart(3, '0')}`, + })); + + let batchCallCount = 0; + const batchBodies: { testIds: string[] }[] = []; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && !url.includes('batch') && !url.includes('/runs')) { + // Return all 60 tests in a single page (no nextToken) + return { body: { items: allTests, nextToken: null } }; + } + if (url.includes('/tests/batch/rerun')) { + batchCallCount++; + const body = JSON.parse(init.body as string) as { testIds: string[] }; + batchBodies.push(body); + // Each chunk's accepted = one entry per testId in the chunk + const accepted = body.testIds.map(tid => ({ + testId: tid, + runId: `run_${tid}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })); + return { + status: 202, + body: { + accepted, + deferred: [], + conflicts: [], + closure: { byProject: [] }, + } satisfies BatchRerunResponse, + }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + + // Must have made exactly 2 batch calls (50 + 10) + expect(batchCallCount).toBe(2); + // First chunk: 50 ids + expect(batchBodies[0]!.testIds).toHaveLength(50); + // Second chunk: remaining 10 + expect(batchBodies[1]!.testIds).toHaveLength(10); + // No testId appears in both chunks + const allSent = [...batchBodies[0]!.testIds, ...batchBodies[1]!.testIds]; + expect(new Set(allSent).size).toBe(60); + }); + + it('exactly 50 tests → single batch request (no chunking)', async () => { + const creds = makeCreds(); + const allTests = Array.from({ length: 50 }, (_, i) => ({ + ...FE_TEST, + id: `test_exact50_${i}`, + })); + + let batchCallCount = 0; + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests') && !url.includes('batch') && !url.includes('/runs')) { + return { body: { items: allTests, nextToken: null } }; + } + if (url.includes('/tests/batch/rerun')) { + batchCallCount++; + const body = JSON.parse(init.body as string) as { testIds: string[] }; + const accepted = body.testIds.map(tid => ({ + testId: tid, + runId: `run_${tid}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })); + return { + status: 202, + body: { + accepted, + deferred: [], + conflicts: [], + closure: { byProject: [] }, + } satisfies BatchRerunResponse, + }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + + // Exactly 50 → 1 request only + expect(batchCallCount).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// FIX 1 — D2-CLI: batch rerun surfaces notFound[] ids as [warn] stderr +// --------------------------------------------------------------------------- +describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', () => { + it('single chunk: notFound ids from server response are collected and warned', async () => { + const creds = makeCreds(); + const stderrLines: string[] = []; + const stdoutLines: string[] = []; + + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_known_1', runId: 'run_k1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + notFound: ['test_bad_1', 'test_bad_2'], + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 200, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_known_1', 'test_bad_1', 'test_bad_2'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ); + + // notFound ids must be warned on stderr + const warnLine = stderrLines.find(l => l.includes('[warn]') && l.includes('2')); + expect(warnLine).toBeDefined(); + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('test_bad_1'); + expect(stderrBlock).toContain('test_bad_2'); + // Summary must mention "not found" + const summaryLine = stderrLines.find(l => l.includes('Reran')); + expect(summaryLine).toContain('not found'); + + // JSON output must include notFound array + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toContain('test_bad_1'); + expect(stdoutBlock).toContain('test_bad_2'); + }); + + it('multi-chunk: notFound ids aggregated across both chunks', async () => { + const creds = makeCreds(); + const stderrLines: string[] = []; + + // Two test ids, each chunk returns one notFound + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const body = JSON.parse(init.body as string) as { testIds: string[] }; + // First testId in the chunk is "known", rest are notFound + const [first, ...rest] = body.testIds; + return { + status: 200, + body: { + accepted: + first !== undefined + ? [{ testId: first, runId: `run_${first}`, enqueuedAt: '2026-06-03T10:00:00Z' }] + : [], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + notFound: rest, + } satisfies BatchRerunResponse, + }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + // 3 ids: first accepted, second/third notFound (both in same chunk) + testIds: ['test_a', 'test_bad_x', 'test_bad_y'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('test_bad_x'); + expect(stderrBlock).toContain('test_bad_y'); + // No warn about test_a (it was accepted) + const warnBlock = stderrLines.filter(l => l.includes('[warn]')).join('\n'); + expect(warnBlock).not.toContain('test_a'); + }); + + it('no notFound field from old backend → no warn emitted (batch path needs ≥2 ids)', async () => { + const creds = makeCreds(); + const stderrLines: string[] = []; + + // Need ≥2 testIds to force the batch path (single-id takes the single rerun path) + const batchRespNoNotFound: BatchRerunResponse = { + accepted: [ + { testId: 'test_x', runId: 'run_x', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_y', runId: 'run_y', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + // notFound absent (old backend — back-compat) + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 200, body: batchRespNoNotFound }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_x', 'test_y'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + const warnLine = stderrLines.find(l => l.includes('[warn]') && l.includes('not found')); + expect(warnLine).toBeUndefined(); + }); +}); + +describe('[fix-4] batch timeout emits one `test wait ` per timed-out run', () => { + it('two timed-out runs → two separate test wait hints', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + // Both runs stay non-terminal → timeout + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/run_b1') || url.includes('/runs/run_b2')) { + return { + body: { + ...makeTerminalRun('run_bX', 'passed'), + status: 'queued', + runId: url.includes('run_b1') ? 'run_b1' : 'run_b2', + }, + }; + } + if (url.includes('/tests/test_1') || url.includes('/tests/test_2')) { + return { body: FE_TEST }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 1, // very short → both will time out + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown'); + } catch (err) { + expect(err instanceof ApiError).toBe(true); + if (err instanceof ApiError) { + expect(err.code).toBe('UNSUPPORTED'); + // nextAction must contain a SEPARATE `test wait` for each run id. + // `test wait` accepts only ONE run id — emitting a joined string would + // produce an invalid command. + const nextAction = err.nextAction; + const b1Count = (nextAction.match(/testsprite test wait run_b1/g) ?? []).length; + const b2Count = (nextAction.match(/testsprite test wait run_b2/g) ?? []).length; + expect(b1Count).toBe(1); + expect(b2Count).toBe(1); + // Neither should appear as "test wait run_b1 run_b2" (joined multi-id form) + expect(nextAction).not.toContain('run_b1 run_b2'); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// dogfood L1796 — `test rerun --all --skip-terminal` and `--status ` +// --------------------------------------------------------------------------- + +/** Mix of terminal and non-terminal test records. */ +const MIXED_TESTS = [ + { ...FE_TEST, id: 'test_passed_1', status: 'passed' as const }, + { ...FE_TEST, id: 'test_failed_1', status: 'failed' as const }, + { ...FE_TEST, id: 'test_running_1', status: 'running' as const }, + { ...FE_TEST, id: 'test_queued_1', status: 'queued' as const }, + { ...FE_TEST, id: 'test_blocked_1', status: 'blocked' as const }, + { ...FE_TEST, id: 'test_cancelled_1', status: 'cancelled' as const }, +]; + +/** + * Build a fetch mock that serves the MIXED_TESTS page on GET /tests and + * accepts batch/rerun, recording which testIds were dispatched. + */ +function makeFilterFetch(dispatched: string[]): typeof globalThis.fetch { + return makeFetch((url, init) => { + if (url.includes('/tests') && (!init.method || init.method === 'GET')) { + return { + body: { items: MIXED_TESTS, nextToken: null }, + }; + } + if (url.includes('/tests/batch/rerun') && init.method === 'POST') { + const body = JSON.parse(init.body as string) as { testIds: string[] }; + dispatched.push(...body.testIds); + const accepted = body.testIds.map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })); + return { + body: { + accepted, + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }, + }; + } + return { body: {} }; + }); +} + +describe('runTestRerun --all --skip-terminal (dogfood L1796)', () => { + it('--skip-terminal excludes passed|failed|blocked|cancelled from dispatch', async () => { + const creds = makeCreds(); + const dispatched: string[] = []; + + const stderr: string[] = []; + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + skipTerminal: true, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: makeFilterFetch(dispatched), + stderr: (line: string) => stderr.push(line), + }, + ); + + // Only 'running' + 'queued' are non-terminal → should be dispatched. + expect(dispatched.sort()).toEqual(['test_queued_1', 'test_running_1'].sort()); + // A skip message should mention how many were skipped. + const skipMsg = stderr.find(l => l.includes('--skip-terminal')); + expect(skipMsg).toBeDefined(); + expect(skipMsg).toContain('4'); // 4 terminal tests skipped + }); + + it('--status only dispatches tests whose status matches', async () => { + const creds = makeCreds(); + const dispatched: string[] = []; + const stderr: string[] = []; + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + statusFilter: 'failed,blocked', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: makeFilterFetch(dispatched), + stderr: (line: string) => stderr.push(line), + }, + ); + + expect(dispatched.sort()).toEqual(['test_blocked_1', 'test_failed_1'].sort()); + const filterMsg = stderr.find(l => l.includes('--status filter')); + expect(filterMsg).toBeDefined(); + expect(filterMsg).toContain('4'); // 4 tests filtered out + }); + + it('--skip-terminal combined with --status only dispatches matching non-terminal tests', async () => { + const creds = makeCreds(); + const dispatched: string[] = []; + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + skipTerminal: true, + statusFilter: 'running,failed', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: makeFilterFetch(dispatched), + stderr: () => {}, + }, + ); + + // --skip-terminal removes failed+blocked+cancelled+passed, leaving running+queued + // --status=running,failed further restricts to running only (failed was removed by skip-terminal) + expect(dispatched).toEqual(['test_running_1']); + }); + + it('exit 5 when --status contains an unknown token', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + statusFilter: 'notastatus', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl: makeFilterFetch([]) }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rerun command exposes --skip-terminal and --status flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const rerun = test.commands.find(c => c.name() === 'rerun')!; + const flagNames = rerun.options.map(o => o.long); + expect(flagNames).toContain('--skip-terminal'); + expect(flagNames).toContain('--status'); + }); +}); + +// --------------------------------------------------------------------------- +// --filter (client-side name filter for --all) +// --------------------------------------------------------------------------- + +/** Extended test list including tests with different names. */ +const NAMED_TESTS = [ + { ...FE_TEST, id: 'test_checkout_1', name: 'Checkout flow', status: 'passed' as const }, + { ...FE_TEST, id: 'test_login_1', name: 'Login page', status: 'failed' as const }, + { ...FE_TEST, id: 'test_checkout_2', name: 'CHECKOUT confirmation', status: 'failed' as const }, + { ...FE_TEST, id: 'test_signup_1', name: 'Sign up form', status: 'running' as const }, +]; + +/** + * Build a fetch mock that serves NAMED_TESTS on GET /tests and records + * dispatched testIds from POST /tests/batch/rerun. + */ +function makeNamedFilterFetch(dispatched: string[]): typeof globalThis.fetch { + return makeFetch((url, init) => { + if (url.includes('/tests') && (!init.method || init.method === 'GET')) { + return { body: { items: NAMED_TESTS, nextToken: null } }; + } + if (url.includes('/tests/batch/rerun') && init.method === 'POST') { + const body = JSON.parse(init.body as string) as { testIds: string[] }; + dispatched.push(...body.testIds); + const accepted = body.testIds.map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })); + return { + body: { accepted, deferred: [], conflicts: [], closure: { byProject: [] } }, + }; + } + return { body: {} }; + }); +} + +describe('runTestRerun --all --filter (client-side name filter)', () => { + it('--filter matches a case-insensitive substring and dispatches only matching tests', async () => { + const creds = makeCreds(); + const dispatched: string[] = []; + const stderr: string[] = []; + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + nameFilter: 'checkout', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: makeNamedFilterFetch(dispatched), + stderr: (line: string) => stderr.push(line), + }, + ); + + // "Checkout flow" and "CHECKOUT confirmation" both match (case-insensitive) + expect(dispatched.sort()).toEqual(['test_checkout_1', 'test_checkout_2'].sort()); + + // Expect a skip advisory mentioning the 2 non-matching tests + const filterMsg = stderr.find(l => l.includes('--filter')); + expect(filterMsg).toBeDefined(); + expect(filterMsg).toContain('2'); // 2 tests skipped + expect(filterMsg).toContain('"checkout"'); + }); + + it('--filter with no match produces empty selection and emits the info message', async () => { + const creds = makeCreds(); + const dispatched: string[] = []; + const stderr: string[] = []; + const stdout: string[] = []; + + await runTestRerun( + { + testIds: [], + all: true, + projectId: 'project_abc', + nameFilter: 'nomatchwhatsoever', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: makeNamedFilterFetch(dispatched), + stderr: (line: string) => stderr.push(line), + stdout: (line: string) => stdout.push(line), + }, + ); + + // Nothing dispatched + expect(dispatched).toHaveLength(0); + + // The "nothing to rerun" message should appear + const noTestsMsg = stderr.find( + l => l.includes('No tests found') || l.includes('nothing to rerun'), + ); + expect(noTestsMsg).toBeDefined(); + + // stdout should contain the empty batch envelope + const body = JSON.parse(stdout.join('\n')); + expect(body).toMatchObject({ accepted: [], deferred: [], conflicts: [] }); + }); + + it('rerun command exposes --filter flag', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const rerun = test.commands.find(c => c.name() === 'rerun')!; + const flagNames = rerun.options.map(o => o.long); + expect(flagNames).toContain('--filter'); + }); +}); + +// --------------------------------------------------------------------------- +// D4 (dogfood CoderCup): under --wait the per-request timeout covers --timeout +// --------------------------------------------------------------------------- + +describe('D4: resolveWaitRequestTimeoutMs', () => { + it('returns the configured value unchanged when not waiting', () => { + expect(resolveWaitRequestTimeoutMs({ wait: false, timeoutSeconds: 600 })).toBeUndefined(); + expect( + resolveWaitRequestTimeoutMs({ wait: false, timeoutSeconds: 600, requestTimeoutMs: 30_000 }), + ).toBe(30_000); + }); + + it('raises to cover --timeout under --wait (with cushion, capped at the 600s max)', () => { + // timeout 300s → 305s per-request (300s + 5s cushion) + expect(resolveWaitRequestTimeoutMs({ wait: true, timeoutSeconds: 300 })).toBe(305_000); + // timeout 600s → clamped at the 600s max + expect(resolveWaitRequestTimeoutMs({ wait: true, timeoutSeconds: 600 })).toBe(600_000); + // timeout 3600s → still capped at 600s (a single request never needs more) + expect(resolveWaitRequestTimeoutMs({ wait: true, timeoutSeconds: 3600 })).toBe(600_000); + }); + + it('floors at the 120s default for tiny --timeout and never lowers an explicit larger value', () => { + // timeout 30s → stays at the 120s default (35s cover < 120s default) + expect(resolveWaitRequestTimeoutMs({ wait: true, timeoutSeconds: 30 })).toBe(120_000); + // explicit request-timeout above the cover value is preserved + expect( + resolveWaitRequestTimeoutMs({ wait: true, timeoutSeconds: 60, requestTimeoutMs: 500_000 }), + ).toBe(500_000); + }); +}); + +// --------------------------------------------------------------------------- +// D3 (dogfood CoderCup): batch --wait summary surfaces deferred + conflicts +// --------------------------------------------------------------------------- + +describe('D3: batch rerun summary surfaces deferred + conflicts', () => { + it('--wait JSON summary includes deferred/conflicts counts (no silent undercount)', async () => { + const creds = makeCreds(); + // Initial dispatch: 1 accepted, 1 deferred, 1 conflict. + // D3 retry loop will fire (opts.wait=true). All 3 retry attempts return the + // same deferred entry so `deferred` never drains, and each retry's `accepted` + // entry (same test_1 / run_b1) is merged in. After the loop, accepted has + // 4 entries (1 original + 3 retries) and deferred still has 1 entry. + const initialBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [{ testId: 'test_conf', currentRunId: 'run_conf' }], + closure: { byProject: [] }, + }; + // Retry responses: keep returning 1 deferred + 1 accepted (same run) so + // the loop exhausts MAX_DEFERRED_RETRIES and falls through. + const retryBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + let batchCallCount = 0; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + const printed: Array<{ summary?: Record }> = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialBatchResp : retryBatchResp }; + } + if (url.includes('/runs/run_b1')) return { body: run1 }; + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRerun( + { + testIds: ['test_1', 'test_deferred', 'test_conf'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + } catch { + // exit 7 expected (deferred still present after retries) — printed first. + } + + // D3 loop ran all 3 retries: 1 initial + 3 retries = 4 total batch calls. + expect(batchCallCount).toBe(4); + + const withSummary = printed.find(p => p.summary); + expect(withSummary).toBeDefined(); + // After D3 retries: accepted = 4 entries (same run_b1 merged 4 times); + // all 4 poll as passed. deferred = 1 (still undrained). conflicts = 1 (from initial). + expect(withSummary!.summary).toMatchObject({ + passed: 4, + failed: 0, + timedOut: 0, + deferred: 1, + conflicts: 1, + total: 4, + }); + }); +}); + +// --------------------------------------------------------------------------- +// codex P2 (D2-CLI): batch --wait JSON output must carry notFound[] too — the +// --wait path builds its own jsonPayload, so a partial batch with ≥1 accepted +// run would otherwise drop the skipped ids and read as fully successful. +// --------------------------------------------------------------------------- + +describe('[codex-P2] batch rerun --wait JSON surfaces notFound[]', () => { + it('--wait JSON output includes notFound[] + summary.notFound for a partial batch', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-08T10:00:00.000Z' }], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + notFound: ['test_bad_1', 'test_bad_2'], + }; + const run1 = makeTerminalRun('run_ok', 'passed'); + run1.testId = 'test_ok'; + const printed: Array<{ notFound?: string[]; summary?: Record }> = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_ok')) return { body: run1 }; + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_ok', 'test_bad_1', 'test_bad_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + const payload = printed.find(p => p.notFound !== undefined || p.summary); + expect(payload).toBeDefined(); + expect(payload!.notFound).toEqual(['test_bad_1', 'test_bad_2']); + expect(payload!.summary).toMatchObject({ + passed: 1, + failed: 0, + timedOut: 0, + notFound: 2, + total: 1, // dispatched (accepted) only + }); + }); +}); + +// --------------------------------------------------------------------------- +// B4 (dogfood CoderCup): BE rerun warns history can't distinguish reruns +// --------------------------------------------------------------------------- + +describe('B4: backend rerun history advisory', () => { + it('single BE rerun emits the "does not distinguish reruns" advisory', async () => { + const creds = makeCreds(); + const beResp = makeBeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/runs/rerun')) return { status: 202, body: beResp }; + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + expect(stderrLines.some(l => l.includes('does not distinguish reruns'))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// D2 (dogfood CoderCup): rerun NOT_FOUND points at a fresh `test run` +// --------------------------------------------------------------------------- + +describe('D2: rerun NOT_FOUND fallback hints', () => { + it('single rerun NOT_FOUND hints at `test run` and exits 4', async () => { + const creds = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND')); + + try { + await runTestRerun( + { + testIds: ['test_missing'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown'); + } catch (err: unknown) { + const e = err as ApiError; + expect(e.exitCode).toBe(4); + expect(e.code).toBe('NOT_FOUND'); + expect(e.message).toContain('no replayable run'); + expect(e.nextAction).toContain('test run test_missing'); + } + }); + + it('batch rerun NOT_FOUND gives an actionable hint (one bad id aborts the batch)', async () => { + const creds = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND')); + + try { + await runTestRerun( + { + testIds: ['test_1', 'test_missing'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + expect.fail('should have thrown'); + } catch (err: unknown) { + const e = err as ApiError; + expect(e.exitCode).toBe(4); + expect(e.code).toBe('NOT_FOUND'); + expect(e.message).toContain('Batch rerun aborted'); + expect(e.nextAction).toContain('test run'); + } + }); +}); + +// --------------------------------------------------------------------------- +// Finding 3 — BE closure fan-out: RequestTimeoutError emits partial stdout +// --------------------------------------------------------------------------- + +describe('[finding-3] BE closure fan-out: RequestTimeoutError emits partial stdout + exit 7', () => { + it('RequestTimeoutError in a closure member poll → partial stdout with all runIds + exit 7', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRunId = rerunResp.runId; // 'run_rerun_be_named' + const producerRunId = rerunResp.closure!.members.find(m => m.role === 'producer')!.runId; + + let callCount = 0; + // fetchImpl: rerun trigger OK, BE test GET OK, then throw RequestTimeoutError on any /runs/ poll + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + callCount++; + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if ( + url.includes('/tests/test_be_consumer_01') || + url.includes('/tests/test_be_producer_01') + ) { + return new Response(JSON.stringify(BE_TEST), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // Any /runs/ poll throws RequestTimeoutError + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, `req_timeout_${callCount}`); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ).catch(e => e); + + // Must exit 7 (RequestTimeoutError) + expect(err).toBeInstanceOf(RequestTimeoutError); + expect((err as RequestTimeoutError).exitCode).toBe(7); + + // Stdout must be non-empty and contain at least the namedRunId + expect(stdoutLines.length).toBeGreaterThan(0); + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toContain(namedRunId); + + // Stderr must include re-attach hints for the closure member runIds + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain(namedRunId); + expect(stderrBlock).toContain(producerRunId); + expect(stderrBlock).toContain('test wait'); + }); +}); + +// --------------------------------------------------------------------------- +// G1d — split teardowns from producers in the rerun stderr summary +// --------------------------------------------------------------------------- + +describe('G1d: BE rerun stderr summary splits producers and teardowns', () => { + /** Helper to trigger a BE single rerun (no --wait) and collect stderr lines. */ + async function runBeRerunNoWait(rerunResp: RerunResponse): Promise { + const creds = makeCreds(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: () => Promise.resolve(), + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + return stderrLines; + } + + it('both producers AND teardowns: "1 selected + N producers + N teardowns"', async () => { + const rerunResp: RerunResponse = { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [ + { testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }, + { testId: 'test_be_producer_01', runId: 'run_p1', role: 'producer' }, + { testId: 'test_be_producer_02', runId: 'run_p2', role: 'producer' }, + { testId: 'test_be_teardown_01', runId: 'run_t1', role: 'teardown' }, + { testId: 'test_be_teardown_02', runId: 'run_t2', role: 'teardown' }, + ], + addedProducers: ['test_be_producer_01', 'test_be_producer_02'], + addedTeardowns: ['test_be_teardown_01', 'test_be_teardown_02'], + clearedCaptured: 0, + }, + }; + + const stderrLines = await runBeRerunNoWait(rerunResp); + const reranLine = stderrLines.find(l => l.startsWith('Reran ')); + expect(reranLine).toBeDefined(); + expect(reranLine).toContain('5 tests'); + expect(reranLine).toContain('1 selected'); + expect(reranLine).toContain('2 producers'); + expect(reranLine).toContain('2 teardowns'); + // Must NOT say "upstream producer(s)" — the old mislabelled wording + expect(reranLine).not.toContain('upstream producer'); + }); + + it('producers only: "1 selected + N producer(s)", no teardown mention', async () => { + const rerunResp: RerunResponse = { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [ + { testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }, + { testId: 'test_be_producer_01', runId: 'run_p1', role: 'producer' }, + ], + addedProducers: ['test_be_producer_01'], + addedTeardowns: [], + clearedCaptured: 0, + }, + }; + + const stderrLines = await runBeRerunNoWait(rerunResp); + const reranLine = stderrLines.find(l => l.startsWith('Reran ')); + expect(reranLine).toBeDefined(); + expect(reranLine).toContain('2 tests'); + expect(reranLine).toContain('1 selected'); + expect(reranLine).toContain('1 producer'); + expect(reranLine).not.toContain('teardown'); + }); + + it('teardowns only: "1 selected + N teardown(s)", no producer mention', async () => { + const rerunResp: RerunResponse = { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [ + { testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }, + { testId: 'test_be_teardown_01', runId: 'run_t1', role: 'teardown' }, + ], + addedProducers: [], + addedTeardowns: ['test_be_teardown_01'], + clearedCaptured: 0, + }, + }; + + const stderrLines = await runBeRerunNoWait(rerunResp); + const reranLine = stderrLines.find(l => l.startsWith('Reran ')); + expect(reranLine).toBeDefined(); + expect(reranLine).toContain('2 tests'); + expect(reranLine).toContain('1 selected'); + expect(reranLine).toContain('1 teardown'); + expect(reranLine).not.toContain('producer'); + }); + + it('lone test: "1 test: 1 selected" (no added producers or teardowns)', async () => { + const rerunResp: RerunResponse = { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [{ testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }], + addedProducers: [], + addedTeardowns: [], + clearedCaptured: 0, + }, + }; + + const stderrLines = await runBeRerunNoWait(rerunResp); + const reranLine = stderrLines.find(l => l.startsWith('Reran ')); + expect(reranLine).toBeDefined(); + // Singular "test" (not "tests") because totalCount === 1 + expect(reranLine).toContain('1 test:'); + expect(reranLine).toContain('1 selected'); + expect(reranLine).not.toContain('producer'); + expect(reranLine).not.toContain('teardown'); + }); +}); + +// --------------------------------------------------------------------------- +// [P1] rerun --all D3 deferred-retry: conflicts discovered on retry must be +// merged into the running conflicts collection and appear in the final JSON. +// --------------------------------------------------------------------------- + +describe('[codex-P1] rerun --all deferred-retry: retry-conflicts merged into final accounting', () => { + // Note: D3 retry loop is in the BATCH path. The single-testId path does not + // have deferred/conflict semantics (it calls POST /tests/{id}/runs/rerun, not + // the batch endpoint). Use two testIds (or --all) to exercise the batch path. + + it('deferred→conflict on retry: JSON conflicts includes retry-discovered entry; exits 6 when all paths resolve to conflict', async () => { + const creds = makeCreds(); + + // Initial batch: 2 IDs submitted → 1 deferred, 0 accepted, 0 initial conflicts. + const initialBatchResp: BatchRerunResponse = { + accepted: [], + deferred: [{ testId: 'test_id_a', reason: 'rate_limited' }], + conflicts: [{ testId: 'test_id_b', currentRunId: 'run_b_inflight' }], + closure: { byProject: [] }, + }; + // Retry: the deferred test is now in-flight (conflict). + const retryBatchResp: BatchRerunResponse = { + accepted: [], + deferred: [], + conflicts: [{ testId: 'test_id_a', currentRunId: 'run_a_inflight' }], + closure: { byProject: [] }, + }; + + let batchCallCount = 0; + const printed: Array> = []; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialBatchResp : retryBatchResp }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'not found', + nextAction: '', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + try { + await runTestRerun( + { + // Two IDs → batch path (D3 retry loop active) + testIds: ['test_id_a', 'test_id_b'], + all: false, + wait: true, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: line => stderrLines.push(line), + }, + ); + throw new Error('Expected runTestRerun to throw exit 6'); + } catch (err) { + // After D3 retry drains deferred→conflict: accepted=0, conflicts≥2 → exit 6 + expect((err as ApiError).exitCode).toBe(6); + } + + // The retry must have fired (at least 2 batch calls). + expect(batchCallCount).toBeGreaterThanOrEqual(2); + + // The JSON output must carry BOTH the initial conflict AND the retry-discovered conflict + // (before fix the retry conflict was logged but not merged into the running conflicts var). + const withConflicts = printed.find( + p => Array.isArray(p.conflicts) && (p.conflicts as unknown[]).length > 0, + ); + expect(withConflicts).toBeDefined(); + const conflictIds = (withConflicts?.conflicts as Array<{ testId: string }>).map(c => c.testId); + // Both the initial conflict and the retry-discovered one must appear + expect(conflictIds).toContain('test_id_b'); // initial + expect(conflictIds).toContain('test_id_a'); // discovered on retry + + // The stderr must log the conflict from the retry attempt + expect(stderrLines.some(l => l.includes('[deferred-retry]') && l.includes('conflict'))).toBe( + true, + ); + }); + + it('partial deferred→conflict + accepted: retry-conflict appears in JSON summary.conflicts; exits 7 due to still-deferred', async () => { + const creds = makeCreds(); + + // Initial: 1 accepted, 2 deferred, 0 initial conflicts. + const initialBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + deferred: [ + { testId: 'test_deferred_a', reason: 'rate_limited' }, + { testId: 'test_deferred_b', reason: 'rate_limited' }, + ], + conflicts: [], + closure: { byProject: [] }, + }; + // Retry: one deferred accepted, one becomes conflict, one remains deferred (keeps looping). + const retryBatchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_deferred_a', runId: 'run_da', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + deferred: [{ testId: 'test_deferred_b', reason: 'rate_limited' }], + conflicts: [{ testId: 'test_deferred_newly_conflicted', currentRunId: 'run_inflight_c' }], + closure: { byProject: [] }, + }; + + let batchCallCount = 0; + const printed: Array> = []; + const terminalRun = makeTerminalRun('run_ok', 'passed'); + terminalRun.testId = 'test_ok'; + const terminalRunDa = makeTerminalRun('run_da', 'passed'); + terminalRunDa.testId = 'test_deferred_a'; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialBatchResp : retryBatchResp }; + } + if (url.includes('/runs/run_ok')) return { body: terminalRun }; + if (url.includes('/runs/run_da')) return { body: terminalRunDa }; + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'not found', + nextAction: '', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + try { + await runTestRerun( + { + testIds: ['test_ok', 'test_deferred_a', 'test_deferred_b'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: () => undefined, + }, + ); + throw new Error('Expected runTestRerun to throw exit 7 (still-deferred)'); + } catch (err) { + // Still deferred after all retries → exit 7 + expect((err as ApiError).exitCode).toBe(7); + } + + const withSummary = printed.find(p => p.summary); + expect(withSummary).toBeDefined(); + // Retry-discovered conflict must appear in summary.conflicts + expect((withSummary?.summary as Record).conflicts).toBeGreaterThanOrEqual(1); + // Retry-discovered conflict must appear in the JSON conflicts array + const conflictIds = (withSummary?.conflicts as Array<{ testId: string }>).map(c => c.testId); + expect(conflictIds).toContain('test_deferred_newly_conflicted'); + }); +}); + +// --------------------------------------------------------------------------- +// [P2] rerun --all D3: idempotency key truncation stays ≤ 256 chars +// --------------------------------------------------------------------------- + +describe('[codex-P2] rerun --all deferred-retry: idempotency key truncation', () => { + // Note: idempotency-key truncation only applies on the BATCH path (two+ IDs). + it('retry key does not exceed 256 chars when base key is at the 256-char limit', async () => { + const creds = makeCreds(); + + const longKey = 'k'.repeat(256); + + const initialBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + const retryBatchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_deferred', runId: 'run_d', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const capturedKeys: string[] = []; + let batchCallCount = 0; + const terminalRunOk = makeTerminalRun('run_ok', 'passed'); + terminalRunOk.testId = 'test_ok'; + const terminalRunD = makeTerminalRun('run_d', 'passed'); + terminalRunD.testId = 'test_deferred'; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + const key = (init.headers as Record)?.['idempotency-key'] ?? ''; + capturedKeys.push(key); + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialBatchResp : retryBatchResp }; + } + if (url.includes('/runs/run_ok')) return { body: terminalRunOk }; + if (url.includes('/runs/run_d')) return { body: terminalRunD }; + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'not found', + nextAction: '', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + // Two testIds → batch path (D3 retry loop active) + await runTestRerun( + { + testIds: ['test_ok', 'test_deferred'], + all: false, + wait: true, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: longKey, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + // All keys must be ≤ 256 chars + expect(capturedKeys.length).toBeGreaterThanOrEqual(2); + for (const key of capturedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + // The retry key must differ from the base (suffix appended) but still fit + expect(capturedKeys[1]).not.toBe(longKey); + expect(capturedKeys[1]).toContain('deferred-retry'); + }); +}); + +// --------------------------------------------------------------------------- +// [codex-P2] Finding 2 — rerun batch: return value === printed state +// +// `runTestRerun` (batch path) must return an object whose accepted/deferred/ +// conflicts/notFound reflect the POST-RETRY mutable state, not the stale +// initial batchResp. Programmatic callers reading the return value must see +// the same data that was printed to stdout. +// --------------------------------------------------------------------------- + +describe('[codex-P2] batch rerun return value == printed state (non-wait)', () => { + it('return value accepted/deferred/conflicts matches stdout JSON after D3 retry drains deferred', async () => { + const creds = makeCreds(); + + // Initial: 1 deferred, 1 accepted + const initialResp: BatchRerunResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + // Retry: the deferred test is now accepted + const retryResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_deferred', runId: 'run_dr', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + let batchCallCount = 0; + const printed: Array> = []; + + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialResp : retryResp }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'nf', nextAction: '', requestId: 'r', details: {} }, + }, + }; + }); + + // Non-wait (no --wait): D3 loop is skipped; deferred stays. Return value + // must reflect the initial state (accepted=1, deferred=1). + const result = await runTestRerun( + { + testIds: ['test_ok', 'test_deferred'], + all: false, + wait: false, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: () => undefined, + }, + ).catch(err => { + // non-wait with deferred exits 7; catch to inspect return + if ((err as { exitCode?: number }).exitCode === 7) return null; + throw err; + }); + + // The printed JSON must carry the initial accepted and deferred + const payload = printed[0]!; + expect(Array.isArray(payload.accepted)).toBe(true); + // Either 1 or 2 accepted depending on retry; deferred counts must be consistent + // Between the printed JSON and the exit (we are asserting consistency here not + // exact counts since the non-wait path doesn't do D3). + + // The return value may be null (threw exit 7) or a BatchRerunResponse; either + // way the test validates that printed state and returned state are consistent + // when the return is not null. + if ( + result !== null && + result !== undefined && + typeof result === 'object' && + 'deferred' in result + ) { + const batchResult = result as BatchRerunResponse; + // Return value deferred[] count must match what was printed in stdout + const printedDeferred = Array.isArray(payload.deferred) + ? (payload.deferred as unknown[]).length + : 0; + expect(batchResult.deferred.length).toBe(printedDeferred); + // Return value accepted[] count must match printed + const printedAccepted = Array.isArray(payload.accepted) + ? (payload.accepted as unknown[]).length + : 0; + expect(batchResult.accepted.length).toBe(printedAccepted); + } + }); +}); + +describe('[codex-P2] batch rerun --wait return value == printed state after D3 retry', () => { + it('return value conflicts includes retry-discovered conflict (mirrors printed JSON)', async () => { + const creds = makeCreds(); + + // Initial: 2 deferred, 0 accepted, 0 conflicts + const initialResp: BatchRerunResponse = { + accepted: [], + deferred: [ + { testId: 'test_a', reason: 'rate_limited' }, + { testId: 'test_b', reason: 'rate_limited' }, + ], + conflicts: [], + closure: { byProject: [] }, + }; + // Retry: one accepted, one becomes conflict + const retryResp: BatchRerunResponse = { + accepted: [{ testId: 'test_a', runId: 'run_a', enqueuedAt: '2026-06-09T10:00:01.000Z' }], + deferred: [], + conflicts: [{ testId: 'test_b', currentRunId: 'run_b_inflight' }], + closure: { byProject: [] }, + }; + + let batchCallCount = 0; + const terminalA = makeTerminalRun('run_a', 'passed'); + terminalA.testId = 'test_a'; + const printed: Array> = []; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialResp : retryResp }; + } + if (url.includes('/runs/run_a')) return { body: terminalA }; + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'nf', nextAction: '', requestId: 'r', details: {} }, + }, + }; + }); + + const result = (await runTestRerun( + { + testIds: ['test_a', 'test_b'], + all: false, + wait: true, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: () => undefined, + }, + )) as BatchRerunResponse | undefined; + + // The printed JSON (summary payload) must include the retry-discovered conflict + const withSummary = printed.find(p => p.summary); + expect(withSummary).toBeDefined(); + expect((withSummary!.summary as Record).conflicts).toBeGreaterThanOrEqual(1); + + const conflictIds = (withSummary!.conflicts as Array<{ testId: string }>).map(c => c.testId); + expect(conflictIds).toContain('test_b'); + + // The RETURN VALUE must also carry the retry-discovered conflict + if (result !== undefined && result !== null && 'conflicts' in result) { + const retConflicts = (result as BatchRerunResponse).conflicts.map(c => c.testId); + expect(retConflicts).toContain('test_b'); + } + }); +}); + +// --------------------------------------------------------------------------- +// [codex-P2] Finding 3 — rerun batch retry loop: notFound[] from retry merged +// +// When a deferred test becomes un-replayable during the retry window (the +// server returns it in notFound[]), the CLI must: +// 1. Merge it into the running notFound set used for stderr + JSON output. +// 2. Not report it as "resolved" (i.e. not leave it in deferred or silently drop it). +// 3. Surface it in the final JSON payload and summary.notFound count. +// --------------------------------------------------------------------------- + +describe('[codex-P2] batch rerun deferred→notFound on retry surfaces in JSON + stderr', () => { + it('deferred test that becomes notFound on retry appears in JSON notFound[] and stderr [warn]', async () => { + const creds = makeCreds(); + + // Initial: 1 accepted, 1 deferred + const initialResp: BatchRerunResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + deferred: [{ testId: 'test_deleted', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + // Retry: the deferred test vanished (notFound), nothing newly accepted + const retryResp: BatchRerunResponse = { + accepted: [], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + notFound: ['test_deleted'], + }; + + let batchCallCount = 0; + const terminalOk = makeTerminalRun('run_ok', 'passed'); + terminalOk.testId = 'test_ok'; + const printed: Array> = []; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialResp : retryResp }; + } + if (url.includes('/runs/run_ok')) return { body: terminalOk }; + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'nf', nextAction: '', requestId: 'r', details: {} }, + }, + }; + }); + + await runTestRerun( + { + testIds: ['test_ok', 'test_deleted'], + all: false, + wait: true, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: line => stderrLines.push(line), + }, + ); + + // D3 retry must have been attempted (batchCallCount >= 2) + expect(batchCallCount).toBeGreaterThanOrEqual(2); + + // The final JSON payload (with summary) must carry notFound + const withSummary = printed.find(p => p.summary); + expect(withSummary).toBeDefined(); + const notFoundArr = withSummary!.notFound; + expect(Array.isArray(notFoundArr)).toBe(true); + expect(notFoundArr as string[]).toContain('test_deleted'); + + // summary.notFound must be non-zero + expect((withSummary!.summary as Record).notFound).toBeGreaterThanOrEqual(1); + + // stderr must contain a [warn] or [deferred-retry] message about the deleted test + const hasNotFoundWarn = stderrLines.some( + l => l.includes('test_deleted') && (l.includes('not found') || l.includes('warn')), + ); + expect(hasNotFoundWarn).toBe(true); + }); + + it('deferred test that becomes notFound is removed from deferred (not double-counted)', async () => { + const creds = makeCreds(); + + // Initial: 2 deferred, 0 accepted → D3 retry fires under --wait. + const initialResp: BatchRerunResponse = { + accepted: [], + deferred: [ + { testId: 'test_stays_deferred', reason: 'rate_limited' }, + { testId: 'test_vanishes', reason: 'rate_limited' }, + ], + conflicts: [], + closure: { byProject: [] }, + }; + // Retry: one stays deferred, one becomes notFound. + const retryResp: BatchRerunResponse = { + accepted: [], + deferred: [{ testId: 'test_stays_deferred', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + notFound: ['test_vanishes'], + }; + + let batchCallCount = 0; + const printed: Array> = []; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { status: 202, body: batchCallCount === 1 ? initialResp : retryResp }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'nf', nextAction: '', requestId: 'r', details: {} }, + }, + }; + }); + + try { + await runTestRerun( + { + testIds: ['test_stays_deferred', 'test_vanishes'], + all: false, + wait: true, + timeoutSeconds: 300, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: line => stderrLines.push(line), + }, + ); + throw new Error('Expected exit 7 (still deferred)'); + } catch (err) { + expect((err as { exitCode?: number }).exitCode).toBe(7); + } + + // D3 retry must have been attempted. + expect(batchCallCount).toBeGreaterThanOrEqual(2); + + // With accepted=0 the function prints { ...batchResp, accepted, deferred, conflicts, notFound } + // via `out.print(...)` and then throws exit 7 — no `summary` key on this path. + // Look for the printed payload directly. + expect(printed.length).toBeGreaterThanOrEqual(1); + const payload = printed[printed.length - 1]!; + + // The printed payload must have notFound containing test_vanishes + expect(Array.isArray(payload.notFound)).toBe(true); + expect(payload.notFound as string[]).toContain('test_vanishes'); + + // test_vanishes must NOT appear in the deferred array (it moved to notFound) + const deferredIds = (payload.deferred as Array<{ testId: string }>).map(d => d.testId); + expect(deferredIds).not.toContain('test_vanishes'); + + // test_stays_deferred must still be in deferred + expect(deferredIds).toContain('test_stays_deferred'); + + // stderr must mention test_vanishes in a [deferred-retry] notFound warning + const hasWarn = stderrLines.some(l => l.includes('test_vanishes')); + expect(hasWarn).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// [B-E2E-02] Fix 2 regression — blocked status must exit 1 in single rerun paths +// --------------------------------------------------------------------------- + +describe('[B-E2E-02] single FE rerun --wait: blocked → exit 1 (regression)', () => { + // Previously the weak assertion `exitCode ?? httpStatus` was truthy for ANY + // error. These tests pin specifically that `blocked` resolves to exit 1. + + it('FE rerun --wait: blocked status → exit 1 (not 0)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const blockedRun = makeTerminalRun('run_rerun_fe_001', 'blocked'); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/runs/run_rerun_fe_001')) return { body: blockedRun }; + return errorBody('NOT_FOUND'); + }); + + let caughtErr: unknown; + try { + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + } catch (err) { + caughtErr = err; + } + + expect(caughtErr).toBeDefined(); + expect((caughtErr as { exitCode?: number }).exitCode).toBe(1); + }); + + it('FE rerun --wait: failed status → exit 1 (existing coverage hardened)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const failedRun = makeTerminalRun('run_rerun_fe_001', 'failed'); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/runs/run_rerun_fe_001')) return { body: failedRun }; + return errorBody('NOT_FOUND'); + }); + + let caughtErr: unknown; + try { + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + } catch (err) { + caughtErr = err; + } + + expect(caughtErr).toBeDefined(); + expect((caughtErr as { exitCode?: number }).exitCode).toBe(1); + }); + + it('FE rerun --wait: passed status → resolves (exit 0)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const passedRun = makeTerminalRun('run_rerun_fe_001', 'passed'); + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/runs/run_rerun_fe_001')) return { body: passedRun }; + return errorBody('NOT_FOUND'); + }); + + // Should NOT throw — passed run must resolve without error + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + const result = printed[0] as RunResponse; + expect(result.status).toBe('passed'); + }); +}); + +describe('[B-E2E-02] skip-dependencies rerun --wait: blocked → exit 1 (regression)', () => { + it('--skip-dependencies + --wait: blocked → exit 1', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ runId: 'run_rerun_be_nodeps' }); + const blockedRun: RunResponse = { + runId: 'run_rerun_be_nodeps', + testId: 'test_be_consumer_01', + projectId: 'project_abc', + userId: 'user_1', + status: 'blocked', + source: 'cli', + createdAt: '2026-06-09T11:00:00.000Z', + startedAt: '2026-06-09T11:00:01.000Z', + finishedAt: '2026-06-09T11:00:10.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 0, failedCount: 0 }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/runs/run_rerun_be_nodeps')) return { body: blockedRun }; + return errorBody('NOT_FOUND'); + }); + + let caughtErr: unknown; + try { + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: true, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl }, + ); + } catch (err) { + caughtErr = err; + } + + expect(caughtErr).toBeDefined(); + expect((caughtErr as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// dashboardUrl on rerun --wait terminal output (colleague feedback 2026-06-10) +// --------------------------------------------------------------------------- + +describe('rerun --wait — dashboardUrl on terminal output', () => { + it('JSON mode (prod endpoint): terminal envelope includes dashboardUrl from the run row', async () => { + const creds = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const rerunResp = makeFeRerunResp(); + const terminalRun = makeTerminalRun('run_rerun_fe_001', 'passed'); + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + if (url.includes('/runs/run_rerun_fe_001')) { + return { body: terminalRun }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + stderr: () => undefined, + }, + ); + + const result = printed[0] as RunResponse & { dashboardUrl?: string }; + expect(result.status).toBe('passed'); + expect(result.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_abc/test/test_fe_01', + ); + }); +}); diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts new file mode 100644 index 0000000..dea16b6 --- /dev/null +++ b/src/commands/test.result.history.spec.ts @@ -0,0 +1,1293 @@ +/** + * Unit tests for `test result --history` — M3.4 piece-5. + * + * Tests the `runResultHistory` function and the `parseDuration` helper. + * All HTTP is mocked via injectable `TestDeps.fetchImpl`. + * + * Coverage targets: + * - history table rendering (text mode) + * - JSON output shape: `{ runs, nextCursor }` + * - pagination cursor round-trip + * - `--source` / `--since` filter forwarding + * - empty / pre-cutover note rendering + * - short-page hint when filtered page < pageSize but nextCursor non-null + * - `isRerun` column rendering + * - back-compat: bare `test result ` (no --history) returns M2 latest UNCHANGED + * - 404 cross-tenant/unknown test → exit 4 + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from '../lib/errors.js'; +import type { ListRunsResponse, RunHistoryItem } from '../lib/runs.types.js'; +import type { CliLatestResult } from './test.js'; +import { runResultHistory, runResult, parseDuration } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { + credentialsPath: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'cli-m34-hist-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +/** Build a canned RunHistoryItem. */ +function makeHistoryItem(overrides?: Partial): RunHistoryItem { + return { + runId: 'run_hist_001', + status: 'passed', + source: 'cli', + isRerun: false, + createdFrom: null, + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:05.000Z', + finishedAt: '2026-06-03T10:02:00.000Z', + codeVersion: 'v1', + failureKind: null, + ...overrides, + }; +} + +/** Build a canned ListRunsResponse. */ +function makeHistoryResp( + items: RunHistoryItem[] = [makeHistoryItem()], + nextCursor: string | null = null, + meta: ListRunsResponse['meta'] = { testKind: 'frontend' }, +): ListRunsResponse { + return { runs: items, nextCursor, meta }; +} + +/** Canned M2 CliLatestResult. */ +const CANNED_LATEST_RESULT: CliLatestResult = { + testId: 'test_abc', + status: 'passed', + startedAt: '2026-06-03T10:00:00.000Z', + finishedAt: '2026-06-03T10:02:00.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: 'snap_001', + runIdIfAvailable: 'run_001', + codeVersion: 'v1', + targetUrl: 'https://example.com', + failedStepIndex: null, + failureKind: null, + summary: { passed: 5, failed: 0, skipped: 0 }, +}; + +function errorEnvelope(code: string): unknown { + return { + error: { + code, + message: `Error: ${code}`, + nextAction: 'retry', + requestId: 'req_test', + details: {}, + }, + }; +} + +// --------------------------------------------------------------------------- +// parseDuration +// --------------------------------------------------------------------------- + +describe('parseDuration', () => { + const NOW = new Date('2026-06-03T12:00:00.000Z'); + + it('24h → 24 hours before now', () => { + const result = parseDuration('24h', NOW); + expect(result).toBe('2026-06-02T12:00:00.000Z'); + }); + + it('7d → 7 days before now', () => { + const result = parseDuration('7d', NOW); + expect(result).toBe('2026-05-27T12:00:00.000Z'); + }); + + it('1h → 1 hour before now', () => { + const result = parseDuration('1h', NOW); + expect(result).toBe('2026-06-03T11:00:00.000Z'); + }); + + it('30d → 30 days before now', () => { + const result = parseDuration('30d', NOW); + // 30 days before 2026-06-03 = 2026-05-04 + expect(result).toBe('2026-05-04T12:00:00.000Z'); + }); + + it('ISO timestamp returned verbatim', () => { + const iso = '2026-05-14T00:00:00.000Z'; + expect(parseDuration(iso, NOW)).toBe(iso); + }); + + it('epoch string returned verbatim', () => { + expect(parseDuration('1748822400000', NOW)).toBe('1748822400000'); + }); + + it('unknown string returned verbatim', () => { + expect(parseDuration('yesterday', NOW)).toBe('yesterday'); + }); + + it('case-insensitive hour suffix', () => { + expect(parseDuration('24H', NOW)).toBe('2026-06-02T12:00:00.000Z'); + }); + + it('case-insensitive day suffix', () => { + expect(parseDuration('7D', NOW)).toBe('2026-05-27T12:00:00.000Z'); + }); +}); + +// --------------------------------------------------------------------------- +// runResultHistory — text mode +// --------------------------------------------------------------------------- + +describe('runResultHistory — text mode', () => { + it('renders a table with RUN ID, STATUS, SOURCE, RERUN?, WHEN, DURATION columns', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/RUN ID/); + expect(output).toMatch(/STATUS/); + expect(output).toMatch(/SOURCE/); + expect(output).toMatch(/RERUN\?/); + expect(output).toMatch(/WHEN/); + expect(output).toMatch(/DURATION/); + }); + + it('renders run_hist_001 row with status passed and source cli', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/run_hist_001/); + expect(output).toMatch(/passed/); + expect(output).toMatch(/cli/); + // isRerun: false → "no" in the RERUN? column + expect(output).toMatch(/\bno\b/); + }); + + it('RERUN? column shows "yes" when isRerun is true', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_rerun_001', + isRerun: true, + createdFrom: 'rerun:run_hist_001', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/\byes\b/); + }); + + it('renders DURATION as "NmNs" when startedAt and finishedAt are present', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + // 2 minutes 30 seconds duration + return { + body: makeHistoryResp([ + makeHistoryItem({ + startedAt: '2026-06-03T10:00:00.000Z', + finishedAt: '2026-06-03T10:02:30.000Z', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/2m 30s/); + }); + + it('renders DURATION as "Ns" for sub-minute runs', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + startedAt: '2026-06-03T10:00:00.000Z', + finishedAt: '2026-06-03T10:00:45.000Z', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/45s/); + }); + + it('renders DURATION as "—" when startedAt is null', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([makeHistoryItem({ startedAt: null, finishedAt: null })]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/—/); + }); + + it('shows per-run detail footer pointing at test wait and test artifact get', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/test wait/); + expect(output).toMatch(/test artifact get/); + }); +}); + +// --------------------------------------------------------------------------- +// runResultHistory — JSON mode +// --------------------------------------------------------------------------- + +describe('runResultHistory — JSON mode', () => { + it('emits { runs, nextCursor } envelope in JSON mode', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const resp = makeHistoryResp([makeHistoryItem()], 'cursor_xyz'); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: resp }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as { runs: unknown[]; nextCursor: string | null }; + expect(Array.isArray(parsed.runs)).toBe(true); + expect(parsed.runs).toHaveLength(1); + expect(parsed.nextCursor).toBe('cursor_xyz'); + }); + + it('JSON mode does not include meta in output envelope', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const resp = makeHistoryResp([makeHistoryItem()], null, { testKind: 'frontend' }); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: resp }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as Record; + // The JSON output shape is { runs, nextCursor } — not the full wire envelope. + expect('meta' in parsed).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Pagination cursor +// --------------------------------------------------------------------------- + +describe('runResultHistory — pagination', () => { + it('forwards --cursor to the request URL', async () => { + const { credentialsPath } = makeCreds(); + let capturedUrl = ''; + const fetchImpl = makeFetch(url => { + capturedUrl = url; + return { body: makeHistoryResp() }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + cursor: 'cursor_abc123', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(capturedUrl).toContain('cursor=cursor_abc123'); + }); + + it('shows Next page hint when nextCursor is non-null', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp([makeHistoryItem()], 'cursor_next_page') }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/--cursor cursor_next_page/); + }); + + it('forwards --page-size to the request URL', async () => { + const { credentialsPath } = makeCreds(); + let capturedUrl = ''; + const fetchImpl = makeFetch(url => { + capturedUrl = url; + return { body: makeHistoryResp() }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + pageSize: 5, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(capturedUrl).toContain('pageSize=5'); + }); +}); + +// --------------------------------------------------------------------------- +// --source / --since filter forwarding +// --------------------------------------------------------------------------- + +describe('runResultHistory — --source / --since filters', () => { + it('forwards --source to the request URL', async () => { + const { credentialsPath } = makeCreds(); + let capturedUrl = ''; + const fetchImpl = makeFetch(url => { + capturedUrl = url; + return { body: makeHistoryResp() }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + source: 'portal', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(capturedUrl).toContain('source=portal'); + }); + + it('forwards --since as ISO timestamp after parseDuration', async () => { + const { credentialsPath } = makeCreds(); + let capturedUrl = ''; + const fetchImpl = makeFetch(url => { + capturedUrl = url; + return { body: makeHistoryResp() }; + }); + + // We use an ISO timestamp directly (bypassing clock dependency). + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + since: '2026-05-14T00:00:00.000Z', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(capturedUrl).toContain('since=2026-05-14T00'); + }); + + it('does NOT append source/since to URL when not provided', async () => { + const { credentialsPath } = makeCreds(); + let capturedUrl = ''; + const fetchImpl = makeFetch(url => { + capturedUrl = url; + return { body: makeHistoryResp() }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(capturedUrl).not.toContain('source='); + expect(capturedUrl).not.toContain('since='); + }); +}); + +// --------------------------------------------------------------------------- +// Empty / pre-cutover note +// --------------------------------------------------------------------------- + +describe('runResultHistory — empty / pre-cutover', () => { + it('prints meta.note when runs array is empty', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: { + runs: [], + nextCursor: null, + meta: { + historyStartsAt: '2026-05-14', + note: 'No CLI-tracked history before 2026-05-14; older runs are in the Portal.', + portalUrl: 'https://app.testsprite.com/tests/test_abc', + }, + } satisfies ListRunsResponse, + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/No CLI-tracked history before 2026-05-14/); + }); + + it('prints default note when runs array is empty and meta.note is absent', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: { + runs: [], + nextCursor: null, + meta: {}, + } satisfies ListRunsResponse, + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/No CLI-tracked history/); + }); + + it('does NOT render a table when runs is empty (pre-cutover)', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([], null, { + note: 'empty pre-cutover', + }), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + // No table headers when empty + expect(output).not.toMatch(/RUN ID.*STATUS/); + }); +}); + +// --------------------------------------------------------------------------- +// Fix A — empty filtered page with non-null nextCursor must NOT report +// "no CLI-tracked history" — it must surface the pagination cursor instead. +// --------------------------------------------------------------------------- + +describe('[fix-A] empty filtered page + non-null nextCursor → paginatable (not pre-cutover)', () => { + it('shows pagination prompt (not pre-cutover note) when runs=[] but nextCursor is non-null', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: { + runs: [], + nextCursor: 'cursor_next_page_abc', + meta: { testKind: 'frontend' }, + } satisfies ListRunsResponse, + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + source: 'cli', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + // Must surface the cursor so the user can continue + expect(output).toMatch(/--cursor cursor_next_page_abc/); + // Must NOT claim there is no history + expect(output).not.toMatch(/No CLI-tracked history/); + }); + + it('shows pre-cutover note when runs=[] AND nextCursor is null', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: { + runs: [], + nextCursor: null, + meta: { testKind: 'frontend' }, + } satisfies ListRunsResponse, + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + source: 'cli', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + // Truly empty: should show the no-history note + expect(output).toMatch(/No CLI-tracked history/); + expect(output).not.toMatch(/--cursor/); + }); +}); + +// --------------------------------------------------------------------------- +// Short-page hint +// --------------------------------------------------------------------------- + +describe('runResultHistory — short filtered page hint', () => { + it('emits stderr hint when filtered page < pageSize but nextCursor is non-null', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + // Return only 1 row but with a nextCursor → short-page scenario + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([makeHistoryItem()], 'cursor_more_pages'), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + pageSize: 20, + source: 'portal', // filter that would cause short page + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + const stderrOutput = stderrLines.join('\n'); + expect(stderrOutput).toMatch(/more may exist/); + expect(stderrOutput).toMatch(/--cursor cursor_more_pages/); + }); + + it('does NOT emit hint when page is full even with nextCursor', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + // Return exactly pageSize rows (1) with nextCursor — NOT a "short" page + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([makeHistoryItem()], 'cursor_full'), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + pageSize: 1, // pageSize = 1, returns exactly 1 → not short + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + const stderrOutput = stderrLines.join('\n'); + expect(stderrOutput).not.toMatch(/more may exist/); + }); + + it('does NOT emit hint when nextCursor is null', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp([makeHistoryItem()], null) }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + pageSize: 20, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + expect(stderrLines.join('\n')).not.toMatch(/more may exist/); + }); +}); + +// --------------------------------------------------------------------------- +// 404 handling +// --------------------------------------------------------------------------- + +describe('runResultHistory — 404 (cross-tenant / unknown)', () => { + it('propagates NOT_FOUND (404) → exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: errorEnvelope('NOT_FOUND'), + })); + + const err = await runResultHistory( + { + output: 'text', + testId: 'test_unknown', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + expect((err as ApiError).exitCode).toBe(4); + }); +}); + +// --------------------------------------------------------------------------- +// Back-compat: bare `test result ` (no --history) is UNCHANGED (M2) +// --------------------------------------------------------------------------- + +describe('runResult — back-compat (M2 latest result)', () => { + it('calls /tests/{testId}/result (NOT /tests/{testId}/runs)', async () => { + const { credentialsPath } = makeCreds(); + let calledUrl = ''; + const fetchImpl = makeFetch(url => { + calledUrl = url; + return { body: CANNED_LATEST_RESULT }; + }); + + await runResult( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + expect(calledUrl).toContain('/tests/test_abc/result'); + expect(calledUrl).not.toContain('/runs'); + }); + + it('returns the CliLatestResult shape (not ListRunsResponse)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: CANNED_LATEST_RESULT })); + + const result = await runResult( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + + // runResult must return the M2 CliLatestResult shape + expect(result.testId).toBe('test_abc'); + expect(result.snapshotId).toBeDefined(); + expect(result.summary).toBeDefined(); + // Must NOT have 'runs' or 'nextCursor' (those belong to ListRunsResponse) + expect((result as unknown as { runs?: unknown }).runs).toBeUndefined(); + expect((result as unknown as { nextCursor?: unknown }).nextCursor).toBeUndefined(); + }); + + it('renders status and snapshotId in text mode (not a history table)', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(() => ({ body: CANNED_LATEST_RESULT })); + + await runResult( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/status:\s+passed/); + expect(output).toMatch(/snapshotId:\s+snap_001/); + // Must NOT look like a history table (no RUN ID header) + expect(output).not.toMatch(/RUN ID\s+STATUS/); + }); +}); + +// --------------------------------------------------------------------------- +// Multiple runs in history table +// --------------------------------------------------------------------------- + +describe('runResultHistory — multiple rows', () => { + it('renders multiple run rows', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ runId: 'run_001', status: 'passed', isRerun: false }), + makeHistoryItem({ + runId: 'run_002', + status: 'failed', + isRerun: true, + createdFrom: 'rerun:run_001', + }), + makeHistoryItem({ + runId: 'run_003', + status: 'blocked', + source: 'portal', + isRerun: false, + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/run_001/); + expect(output).toMatch(/run_002/); + expect(output).toMatch(/run_003/); + expect(output).toMatch(/failed/); + expect(output).toMatch(/blocked/); + expect(output).toMatch(/portal/); + }); +}); + +// --------------------------------------------------------------------------- +// G1b — per-run targetUrl in history table +// --------------------------------------------------------------------------- + +describe('runResultHistory — G1b targetUrl in history table (text mode)', () => { + it('renders targetUrl sub-line when targetUrl is present and targetUrlSource is "run"', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/targetUrl: https:\/\/staging\.example\.com\/checkout/); + }); + + it('renders targetUrl: — when targetUrlSource is "unresolved"', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_002', + targetUrl: null, + targetUrlSource: 'unresolved', + }), + ]), + }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).toMatch(/targetUrl: —/); + }); + + it('omits targetUrl sub-line when targetUrl is absent (pre-G1b backend)', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + // makeHistoryItem() has no targetUrl field by default + return { body: makeHistoryResp([makeHistoryItem()]) }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).not.toMatch(/targetUrl:/); + }); + + it('truncates a very long targetUrl to HISTORY_TARGET_URL_MAX chars', async () => { + const longUrl = 'https://example.com/' + 'a'.repeat(100); + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([makeHistoryItem({ targetUrl: longUrl, targetUrlSource: 'run' })]), + }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + // Should contain truncation marker + expect(output).toMatch(/targetUrl:.*…/); + // The full URL should not appear verbatim + expect(output).not.toContain(longUrl); + }); + + it('G1b: targetUrl passes through in --output json (RunHistoryItem fields)', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + targetUrl: 'https://example.com', + targetUrlSource: 'run', + }), + ]), + }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'nf', + nextAction: 'retry', + requestId: 'r', + details: {}, + }, + }, + }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as { runs: RunHistoryItem[] }; + const firstRun = parsed.runs[0]; + expect(firstRun).toBeDefined(); + expect(firstRun?.targetUrl).toBe('https://example.com'); + expect(firstRun?.targetUrlSource).toBe('run'); + }); +}); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts new file mode 100644 index 0000000..8de3c7c --- /dev/null +++ b/src/commands/test.run.spec.ts @@ -0,0 +1,3528 @@ +/** + * Unit tests for `test run ` — M3.3 piece-3. + * + * All HTTP is mocked via `makeFetch` / `makeCreds`. The polling loop's + * sleep injection is wired through `TestDeps.sleep` to avoid real delays. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Command } from 'commander'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import type { RunResponse, TriggerRunResponse, BatchRunFreshResponse } from '../lib/runs.types.js'; +import { runTestRun, runTestRunAll } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13502', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-m33-run-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +const TRIGGER_RESP: TriggerRunResponse = { + runId: 'run_abc', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', +}; + +function makePassedRun(): RunResponse { + return { + runId: 'run_abc', + testId: 'test_xyz', + projectId: 'project_1', + userId: 'user_1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }; +} + +function makeFailedRun(): RunResponse { + return { ...makePassedRun(), status: 'failed', failedStepIndex: 2, failureKind: 'assertion' }; +} + +function errorBody( + code: string, + details: Record = {}, +): { + status: number; + body: unknown; +} { + const statusMap: Record = { + AUTH_REQUIRED: 401, + AUTH_FORBIDDEN: 403, + NOT_FOUND: 404, + VALIDATION_ERROR: 400, + CONFLICT: 409, + RATE_LIMITED: 429, + INTERNAL: 500, + UNAVAILABLE: 503, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +const instantSleep = () => Promise.resolve(); + +function disableExits(cmd: Command): void { + cmd.exitOverride(); + cmd.commands.forEach(disableExits); +} + +// --------------------------------------------------------------------------- +// Surface test +// --------------------------------------------------------------------------- + +describe('createTestCommand — run + wait subcommands exposed', () => { + it('exposes run and wait as top-level subcommands', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const names = test.commands.map(c => c.name()).sort(); + expect(names).toContain('run'); + expect(names).toContain('wait'); + }); + + it('run subcommand has --wait, --timeout, --target-url, --idempotency-key flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const run = test.commands.find(c => c.name() === 'run')!; + const flagNames = run.options.map(o => o.long); + expect(flagNames).toContain('--wait'); + expect(flagNames).toContain('--timeout'); + expect(flagNames).toContain('--target-url'); + expect(flagNames).toContain('--idempotency-key'); + }); + + it('wait subcommand has --timeout flag', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const wait = test.commands.find(c => c.name() === 'wait')!; + const flagNames = wait.options.map(o => o.long); + expect(flagNames).toContain('--timeout'); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — no-wait path (fire and return) +// --------------------------------------------------------------------------- + +describe('runTestRun — no-wait (fire and return)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('happy path — fires trigger, returns TriggerRunResponse, exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/runs')) return { body: TRIGGER_RESP }; + return { status: 404, body: {} }; + }); + const stdout: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + // Should return TriggerRunResponse (not RunResponse) + expect(result).toMatchObject({ runId: 'run_abc', status: 'queued' }); + expect(stdout.join('')).toContain('run_abc'); + }); + + it('no-wait path sends POST to /tests/{testId}/runs', async () => { + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: TRIGGER_RESP }; + }); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + sleep: instantSleep, + }, + ); + expect(seenUrls.some(u => u.includes('/tests/test_xyz/runs'))).toBe(true); + }); + + it('sends correct idempotency key header when provided', async () => { + const { credentialsPath } = makeCreds(); + const seenHeaders: Record[] = []; + const fetchImpl = (async (_input: FetchInput, init: RequestInit = {}) => { + const h = new Headers(init.headers); + const entry: Record = {}; + h.forEach((v, k) => { + entry[k] = v; + }); + seenHeaders.push(entry); + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + idempotencyKey: 'test-idem-key-001', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + sleep: instantSleep, + }, + ); + const triggerHeaders = seenHeaders.find(h => h['idempotency-key']); + expect(triggerHeaders?.['idempotency-key']).toBe('test-idem-key-001'); + }); + + it('--output json prints JSON envelope on stdout', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TRIGGER_RESP })); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const parsed = JSON.parse(stdout.join('')) as Record; + expect(parsed.runId).toBe('run_abc'); + expect(parsed.status).toBe('queued'); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — --wait path +// --------------------------------------------------------------------------- + +describe('runTestRun — with --wait', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('polls until passed and returns RunResponse, exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + let getCount = 0; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + getCount++; + if (getCount < 2) return { body: { ...makePassedRun(), status: 'running' as const } }; + return { body: makePassedRun() }; + }); + const stdout: string[] = []; + const stderrLines: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(result).toMatchObject({ runId: 'run_abc', status: 'passed' }); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.status).toBe('passed'); + }); + + it('wait path — failed status → throws CLIError with exitCode 1', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: makeFailedRun() }; + }); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeDefined(); + expect(err.exitCode).toBe(1); + expect(err.name).toBe('CLIError'); + }); + + it('wait timeout → UNSUPPORTED error (exit 7) with nextAction containing run-id', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + // Always return non-terminal to force timeout + return { body: { ...makePassedRun(), status: 'running' as const } }; + }); + + // Mock Date.now to force timeout after first check + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('UNSUPPORTED'); + expect((err as ApiError).exitCode).toBe(7); + expect((err as ApiError).nextAction).toContain('test wait'); + } finally { + Date.now = realDateNow; + } + }); + + it('wait path — --output json disables ticker (stderr stays clean)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: makePassedRun() }; + }); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', // disables ticker + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + // With --output json, no ticker noise goes to stderr + // (artifact hint still goes to stderr if non-passed, but passed = no hint) + const hasAnsiEscape = stderrLines.some(l => l.includes('\x1b[')); + expect(hasAnsiEscape).toBe(false); + }); + + it('blocked status → CLIError exit 1 with artifact hint on stderr', async () => { + const { credentialsPath } = makeCreds(); + const blockedRun = { ...makePassedRun(), status: 'blocked' as const }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: blockedRun }; + }); + const stderrLines: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + expect(stderrLines.some(l => l.includes('artifact'))).toBe(true); + }); + + // B3 — cancelled must NOT emit the artifact hint (no artifacts to download) + it('B3 — cancelled status → CLIError exit 1 with NO artifact hint on stderr', async () => { + const { credentialsPath } = makeCreds(); + const cancelledRun = { ...makePassedRun(), status: 'cancelled' as const }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: cancelledRun }; + }); + const stderrLines: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + // Must still exit 1 + expect(err.exitCode).toBe(1); + expect(err.name).toBe('CLIError'); + // Must NOT emit artifact hint for cancelled (no artifacts were captured) + expect(stderrLines.some(l => l.includes('artifact'))).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — error scenarios +// --------------------------------------------------------------------------- + +describe('runTestRun — error scenarios', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('409 CONFLICT → throws ApiError exit 6 (already running)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('CONFLICT', { currentRunId: 'run_existing' })); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + }); + + it('403 AUTH_FORBIDDEN → throws ApiError exit 3', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('AUTH_FORBIDDEN')); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(3); + }); + + it('400 VALIDATION_ERROR from server → throws ApiError exit 5', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('VALIDATION_ERROR')); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + }); + + it('--target-url localhost rejected client-side (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('network should not be hit'); + }); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'http://localhost:3000', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('--target-url 10.x rejected client-side (RFC1918)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not reach network'); + }); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'http://10.0.0.1', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — dry-run path +// --------------------------------------------------------------------------- + +describe('runTestRun — dry-run', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + // P3-14: test run --dry-run now emits a TriggerRunResponse shape (same as + // a real trigger response) rather than the HTTP-descriptor envelope. This + // makes `test run --dry-run --output json` consistent with `test rerun --dry-run`. + + it('dry-run: no network call; prints TriggerRunResponse shape', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + const response = JSON.parse(stdout.join('')) as Record; + // Dry-run now returns TriggerRunResponse shape (runId, status, enqueuedAt) + // rather than the HTTP-descriptor envelope (method, path, body). + expect(response.runId).toBeDefined(); + expect(response.status).toBeDefined(); + }); + + it('dry-run with --target-url: does not throw on valid public URL', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + // With the new sample-based dry-run, --target-url is validated (must be + // a public URL) but is not reflected in the canned sample response. + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ), + ).resolves.toBeDefined(); + }); + + it('dry-run with --wait: emits the descriptor envelope with a thenPoll hint (codex #128 P2-A)', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + const response = JSON.parse(stdout.join('')) as Record; + // With --wait, dry-run falls through to the descriptor envelope so the + // requested wait/poll is represented honestly — a queued TriggerRunResponse + // sample would hide it. The envelope carries method/path + a thenPoll hint. + expect(response.method).toBe('POST'); + expect(response.path).toBe('/api/cli/v1/tests/test_xyz/runs'); + expect(response.thenPoll).toBeDefined(); + // Not the queued-trigger sample shape. + expect(response.runId).toBeUndefined(); + }); + + it('dry-run chained (createContext set): merges created-test fields with the run sample (codex #128 P2-A)', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + createContext: { + testId: 'test_xyz', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-05-13T00:00:00.000Z', + }, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + const response = JSON.parse(stdout.join('')) as Record; + // The merged chain envelope keeps the created-test fields at top level + // (these would be dropped if the dry-run sample short-circuited + // printRunOrChain) and nests the run sample under `run`. + expect(response.testId).toBe('test_xyz'); + expect(response.codeVersion).toBe('v1'); + const run = response.run as Record | undefined; + expect(run).toBeDefined(); + expect(run?.runId).toBeDefined(); + }); + + it('dry-run: response has runId and status fields', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + const response = JSON.parse(stdout.join('')) as Record; + expect(typeof response.runId).toBe('string'); + expect(typeof response.status).toBe('string'); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — target-url with allowed values +// --------------------------------------------------------------------------- + +describe('runTestRun — target-url guard: allowed URLs pass through', () => { + it('allows https://example.com (public URL)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TRIGGER_RESP })); + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://example.com', + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ); + expect(result).toMatchObject({ runId: 'run_abc' }); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRun — Bug 2 dogfood round-4 2026-05-17 +// CONFLICT + --wait auto-resume on currentRunId +// --------------------------------------------------------------------------- + +describe('runTestRun — CONFLICT + --wait auto-resume (dogfood round-4)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('--wait + CONFLICT with currentRunId: auto-resumes polling, no CONFLICT envelope emitted', async () => { + // The POST /run returns 409 CONFLICT with currentRunId=run_inflight. + // With --wait, the CLI should silently attach to run_inflight's poll + // loop rather than surfacing exit 6. + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Test test_xyz already has run run_inflight in flight.', + nextAction: 'Wait for the run to finish.', + requestId: 'req_conflict', + details: { reason: 'run_in_flight', currentRunId: 'run_inflight' }, + }, + }; + const inflightRun: ReturnType = { + ...makePassedRun(), + runId: 'run_inflight', + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs')) { + return { status: 409, body: conflictBody }; + } + // GET /runs/run_inflight — return terminal + return { body: inflightRun }; + }); + const stdout: string[] = []; + const stderrLines: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + // Should resolve successfully — no CONFLICT exit 6 + expect(result).toMatchObject({ runId: 'run_inflight', status: 'passed' }); + // Advisory must mention the in-flight runId on stderr + expect(stderrLines.some(l => l.includes('run_inflight'))).toBe(true); + // Stdout should carry the final run result, not an error envelope + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.runId).toBe('run_inflight'); + expect(printed.status).toBe('passed'); + }); + + it('--wait + CONFLICT without currentRunId: surfaces exit 6 (cannot auto-resume without a target runId)', async () => { + const { credentialsPath } = makeCreds(); + const conflictBodyNoRunId = { + error: { + code: 'CONFLICT', + message: 'Test test_xyz already has a run in flight.', + nextAction: 'Wait for the run to finish.', + requestId: 'req_conflict2', + details: { reason: 'run_in_flight' }, // no currentRunId + }, + }; + const fetchImpl = makeFetch(() => ({ status: 409, body: conflictBodyNoRunId })); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + }); + + it('no --wait + CONFLICT: always surfaces exit 6 (auto-resume is wait-only behavior)', async () => { + // Without --wait, CONFLICT should always propagate — the caller + // explicitly did not ask for polling. + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Test test_xyz already has run run_existing in flight.', + nextAction: 'Check currentRunId.', + requestId: 'req_c3', + details: { reason: 'run_in_flight', currentRunId: 'run_existing' }, + }, + }; + const fetchImpl = makeFetch(() => ({ status: 409, body: conflictBody })); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + }); + + it('--wait + CONFLICT auto-resume: polling loop uses the conflict currentRunId, not a stale one', async () => { + const { credentialsPath } = makeCreds(); + const seenGetRunIds: string[] = []; + const inflightPassedRun: ReturnType = { + ...makePassedRun(), + runId: 'run_specific_inflight', + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs')) { + return { + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'in flight', + nextAction: '', + requestId: 'r1', + details: { reason: 'run_in_flight', currentRunId: 'run_specific_inflight' }, + }, + }, + }; + } + // Extract runId from GET /runs/ + const match = /\/runs\/([^?/]+)/.exec(url); + if (match?.[1]) seenGetRunIds.push(match[1]); + return { body: inflightPassedRun }; + }); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + // All GET calls must be on run_specific_inflight (not some stale uuid) + expect(seenGetRunIds.length).toBeGreaterThan(0); + expect(seenGetRunIds.every(id => id === 'run_specific_inflight')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// CONFLICT target-URL verification (codex round-1 finding 3) +// --------------------------------------------------------------------------- + +describe('runTestRun — CONFLICT target-URL verification (codex round-1 finding-3)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('CONFLICT reason=run_in_flight + --target-url matching in-flight run → auto-resume succeeds', async () => { + // Arrange: POST → 409 run_in_flight; GET /runs/run_inflight → targetUrl matches + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Test already has a run in flight.', + nextAction: '', + requestId: 'req_c10', + details: { reason: 'run_in_flight', currentRunId: 'run_inflight_url_match' }, + }, + }; + const inFlightRun: RunResponse = { + ...makePassedRun(), + runId: 'run_inflight_url_match', + targetUrl: 'https://staging.example.com', + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs')) { + return { status: 409, body: conflictBody }; + } + // GET /runs/run_inflight_url_match (fetch for URL verification + poll) + return { body: inFlightRun }; + }); + const stderrLines: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + targetUrl: 'https://staging.example.com', // matches in-flight + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(result).toMatchObject({ runId: 'run_inflight_url_match', status: 'passed' }); + // Advisory should be emitted and mention the runId + expect(stderrLines.some(l => l.includes('run_inflight_url_match'))).toBe(true); + }); + + it('CONFLICT reason=run_in_flight + --target-url X but in-flight runs against Y → exit 6, no poll', async () => { + // Arrange: POST → 409 run_in_flight; GET /runs/run_inflight → different targetUrl + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Test already has a run in flight.', + nextAction: '', + requestId: 'req_c11', + details: { reason: 'run_in_flight', currentRunId: 'run_inflight_url_mismatch' }, + }, + }; + const inFlightRun: RunResponse = { + ...makePassedRun(), + runId: 'run_inflight_url_mismatch', + targetUrl: 'https://prod.other-team.example.com', // different from requested + }; + const seenGetRunIds: string[] = []; + // Track which run IDs the poll loop hits (should be none for this case) + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs')) { + return { status: 409, body: conflictBody }; + } + // GET /runs/run_inflight_url_mismatch — for URL verification only + const match = /\/runs\/([^?/]+)/.exec(url); + if (match?.[1]) seenGetRunIds.push(match[1]); + return { body: inFlightRun }; + }); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + targetUrl: 'https://my-staging.example.com', // mismatches in-flight + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + // Must exit 6 with a CONFLICT error, not auto-resume + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CONFLICT'); + expect((err as ApiError).exitCode).toBe(6); + // Message must mention the mismatch + expect((err as ApiError).message).toContain('different target URL'); + expect((err as ApiError).message).toContain('prod.other-team.example.com'); + // nextAction must point at 'test wait ' + expect((err as ApiError).nextAction).toContain('run_inflight_url_mismatch'); + // The poll loop must NOT have been entered (only the URL-verification GET) + // seenGetRunIds has exactly one entry (the verification GET), not multiple polls + expect(seenGetRunIds).toHaveLength(1); + expect(seenGetRunIds[0]).toBe('run_inflight_url_mismatch'); + }); + + it('CONFLICT reason=idempotency_body_mismatch → exit 6, never auto-resume', async () => { + // IDEMPOTENCY_BODY_MISMATCH uses code 'IDEMPOTENCY_BODY_MISMATCH' (409), + // and must never trigger auto-resume regardless of --wait. + const { credentialsPath } = makeCreds(); + const mismatchBody = { + error: { + code: 'IDEMPOTENCY_BODY_MISMATCH', + message: 'Idempotency key reused with a different request body.', + nextAction: 'Mint a new idempotency key and retry.', + requestId: 'req_c12', + details: { reason: 'body_hash_mismatch' }, + }, + }; + const fetchImpl = makeFetch(() => ({ status: 409, body: mismatchBody })); + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('IDEMPOTENCY_BODY_MISMATCH'); + expect((err as ApiError).exitCode).toBe(6); + }); + + it('CONFLICT reason=run_in_flight + no --target-url → auto-resume, fetches real targetUrl from in-flight run', async () => { + // Finding D (codex round-2): when --target-url is not supplied, the CLI now + // fetches GET /runs/{currentRunId} to bind the REAL targetUrl to the + // synthesised triggerResponse (instead of ''). The advisory must include the + // actual target URL and the cancel hint. + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Test already has a run in flight.', + nextAction: '', + requestId: 'req_c13', + details: { reason: 'run_in_flight', currentRunId: 'run_default_target' }, + }, + }; + const inFlightRun: RunResponse = { + ...makePassedRun(), + runId: 'run_default_target', + targetUrl: 'https://default.example.com', + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs')) { + return { status: 409, body: conflictBody }; + } + return { body: inFlightRun }; + }); + const stderrLines: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + // No targetUrl supplied + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(result).toMatchObject({ runId: 'run_default_target', status: 'passed' }); + // Advisory must mention the actual target URL (not '') and the cancel hint + const advisory = stderrLines.join(' '); + expect(advisory).toContain('run_default_target'); + expect(advisory).toContain('https://default.example.com'); + expect(advisory).toContain('--target-url'); + }); +}); + +// --------------------------------------------------------------------------- +// Backend testId fallback (dogfood L1888) +// +// BE run rows never finalize server-side, so `test run --wait` falls back to +// the testId-scoped verdict so a passing BE test exits 0 instead of timing out. +// --------------------------------------------------------------------------- + +interface BeResult { + testId: string; + status: string; + startedAt: string | null; + finishedAt: string | null; + videoUrl: string | null; + failureAnalysisUrl: string | null; + snapshotId: string; + runIdIfAvailable: string | null; + codeVersion: string | null; + targetUrl: string | null; + failedStepIndex: number | null; + failureKind: string | null; + summary: { passed: number; failed: number; skipped: number }; +} + +function makeBeResult(overrides: Partial = {}): BeResult { + return { + testId: 'test_be', + status: 'passed', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: 'snap_1', + runIdIfAvailable: 'run_abc', + codeVersion: 'v1', + targetUrl: 'https://example.com', + failedStepIndex: null, + failureKind: null, + summary: { passed: 1, failed: 0, skipped: 0 }, + ...overrides, + }; +} + +function beRunRouter(args: { + type?: string; + runStatus?: RunResponse['status']; + result: () => BeResult; +}) { + const counts = { trigger: 0, type: 0, result: 0, run: 0 }; + const handler = (url: string) => { + if (url.includes('/tests/test_be/runs')) { + counts.trigger += 1; + return { body: TRIGGER_RESP }; + } + if (url.includes('/tests/test_be/result')) { + counts.result += 1; + return { body: args.result() }; + } + if (url.includes('/runs/run_abc')) { + counts.run += 1; + // A realistic BE run row: real correlation metadata (testId === the + // triggered test, project/user populated) but stuck non-terminal. + return { + body: { + ...makePassedRun(), + testId: 'test_be', + projectId: 'project_be', + userId: 'user_be', + status: args.runStatus ?? 'running', + finishedAt: null, + }, + }; + } + if (url.includes('/tests/test_be')) { + counts.type += 1; + return { + body: { + id: 'test_be', + projectId: 'p1', + name: 'BE test', + type: args.type ?? 'backend', + createdFrom: 'mcp', + status: 'running', + createdAt: '2026-05-15T10:00:00.000Z', + updatedAt: '2026-05-15T10:00:00.000Z', + }, + }; + } + return { status: 404, body: {} }; + }; + return { handler, counts }; +} + +describe('runTestRun — backend testId fallback (L1888)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('passing backend test resolves via testId result (exit 0) + advisory', async () => { + const { credentialsPath } = makeCreds(); + const router = beRunRouter({ result: () => makeBeResult({ status: 'passed' }) }); + const stderr: string[] = []; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_be', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: l => stderr.push(l), + sleep: instantSleep, + }, + ); + const run = result as RunResponse; + expect(run.status).toBe('passed'); + expect(router.counts.type).toBeGreaterThan(0); + expect(router.counts.result).toBeGreaterThan(0); + expect(stderr.join(' ')).toContain('test record'); + // codex round-2: real correlation metadata from the polled run row is + // preserved (not fabricated blank). + expect(run.runId).toBe('run_abc'); + expect(run.testId).toBe('test_be'); + expect(run.projectId).toBe('project_be'); + expect(run.userId).toBe('user_be'); + }); + + it('failing backend test resolves via fallback (CLIError exit 1) + testId artifact hint', async () => { + const { credentialsPath } = makeCreds(); + const router = beRunRouter({ + result: () => + makeBeResult({ + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 1, + summary: { passed: 0, failed: 1, skipped: 0 }, + }), + }); + const stderr: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_be', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: l => stderr.push(l), + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + expect(err.name).toBe('CLIError'); + expect(stderr.join(' ')).toContain('test failure get test_be'); + }); + + it('frontend test is untouched — terminal run row resolves with zero testId lookups', async () => { + const { credentialsPath } = makeCreds(); + const router = beRunRouter({ + type: 'frontend', + runStatus: 'passed', + result: () => makeBeResult(), + }); + // run row reports terminal on the first poll → fallback never engaged. + const handler = (url: string) => { + if (url.includes('/tests/test_be/runs')) return { body: TRIGGER_RESP }; + if (url.includes('/runs/run_abc')) return { body: makePassedRun() }; + if (url.includes('/tests/test_be/result')) { + router.counts.result += 1; + return { body: makeBeResult() }; + } + if (url.includes('/tests/test_be')) { + router.counts.type += 1; + return { body: { id: 'test_be', type: 'frontend' } }; + } + return { status: 404, body: {} }; + }; + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_be', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(handler), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect((result as RunResponse).status).toBe('passed'); + expect(router.counts.type).toBe(0); + expect(router.counts.result).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// C2 — BE steps text renders as n/a (backend) +// --------------------------------------------------------------------------- + +describe('C2 — backend run renders steps: n/a (backend) in text mode', () => { + it('renders "steps: n/a (backend)" when the BE fallback resolved the run', async () => { + // Simulates a backend test where: + // - the run row stays non-terminal (orphaned, dogfood L1888) + // - the testId result row becomes terminal → fallback fires + const { credentialsPath } = makeCreds(); + const beResult = { + testId: 'test_be_c2', + status: 'passed', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:10.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: 'snap_c2', + runIdIfAvailable: 'run_c2', + codeVersion: 'v1', + targetUrl: 'https://example.com', + failedStepIndex: null, + failureKind: null, + summary: { passed: 1, failed: 0, skipped: 0 }, + }; + const handler = (url: string) => { + if (url.includes('/tests/test_be_c2/runs')) { + return { + body: { + runId: 'run_c2', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + if (url.includes('/runs/run_c2')) { + // Return non-terminal so the fallback reads the result instead + return { + body: { + runId: 'run_c2', + testId: 'test_be_c2', + projectId: 'p1', + userId: 'u1', + status: 'queued', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: null, + finishedAt: null, + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + videoUrl: null, + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, + }, + }; + } + if (url.includes('/tests/test_be_c2/result')) { + // Return terminal result so fallback resolves + return { body: beResult }; + } + if (url.includes('/tests/test_be_c2')) { + return { body: { id: 'test_be_c2', type: 'backend' } }; + } + return { status: 404, body: {} }; + }; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_be_c2', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(handler), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + const out = stdoutLines.join('\n'); + // BE fallback used → steps line should be n/a (backend) + expect(out).toContain('steps n/a (backend)'); + // Must NOT show confusing "0/0 (passed=0, failed=0)" + expect(out).not.toContain('0/0'); + }); + + it('FE terminal run row still renders real step counts', async () => { + const { credentialsPath } = makeCreds(); + const feRun: RunResponse = { + runId: 'run_fe_c2', + testId: 'test_fe_c2', + projectId: 'p1', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 3, failedCount: 0 }, + }; + const handler = (url: string) => { + if (url.includes('/tests/test_fe_c2/runs')) { + return { + body: { + runId: 'run_fe_c2', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + if (url.includes('/runs/run_fe_c2')) return { body: feRun }; + return { status: 404, body: {} }; + }; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_fe_c2', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(handler), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + const out = stdoutLines.join('\n'); + // FE run: real step counts rendered + expect(out).toContain('steps 3/3 (passed=3, failed=0)'); + // Not the backend placeholder + expect(out).not.toContain('n/a (backend)'); + }); + + it('BE run terminal on first poll (opts.type=backend) renders steps: n/a via type hint (Fix 2)', async () => { + // Simulates a fast backend run that is already terminal on the FIRST poll, + // so `beFallbackUsed` is false (resolveAlternate is never called). + // The only BE signal is `opts.type === 'backend'` from the create-chain. + const { credentialsPath } = makeCreds(); + const passedBeRun: RunResponse = { + runId: 'run_fast_be', + testId: 'test_fast_be', + projectId: 'p1', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:02.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + // RunResponse has no per-step breakdown for BE runs; backend may send zeros. + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, + }; + const handler = (url: string) => { + if (url.includes('/tests/test_fast_be/runs')) { + return { + body: { + runId: 'run_fast_be', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + // Run row is terminal on first GET — resolveAlternate never fires. + if (url.includes('/runs/run_fast_be')) return { body: passedBeRun }; + return { status: 404, body: {} }; + }; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_fast_be', + // type hint supplied by the create-chain (or the caller when type is known) + type: 'backend', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(handler), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + const out = stdoutLines.join('\n'); + // Fast BE: type hint → n/a (backend) even though beFallbackUsed is false + expect(out).toContain('steps n/a (backend)'); + expect(out).not.toContain('0/0'); + }); +}); + +// --------------------------------------------------------------------------- +// C3 — requestId: trailer gated behind --verbose / --debug +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Fix 3 — test run idempotency-key emitted under --verbose (was --debug only) +// --------------------------------------------------------------------------- + +describe('Fix 3 — test run idempotency-key emission', () => { + it('does NOT emit idempotency-key by default (text mode, no --verbose/--debug)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TRIGGER_RESP })); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_idem', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(false); + }); + + it('emits idempotency-key under --verbose', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TRIGGER_RESP })); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: true, + dryRun: false, + testId: 'test_idem', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + + it('emits idempotency-key in JSON output mode (Fix 1: JSON-mode regression)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TRIGGER_RESP })); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_idem', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + // JSON mode: idempotency-key must still appear on stderr (stderr is never + // the JSON stream; suppressing it was a silent regression). + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); +}); + +describe('C3 — requestId trailer gated behind --verbose', () => { + it('does NOT emit requestId to stderr by default (no-wait path)', async () => { + const { credentialsPath } = makeCreds(); + // Inject a request-id header in the response so the CLI can surface it. + const fetchImpl: typeof globalThis.fetch = async () => + new Response( + JSON.stringify({ + runId: 'run_c3', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }), + { + status: 200, + headers: { 'content-type': 'application/json', 'x-request-id': 'req_c3_hidden' }, + }, + ); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_c3', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('requestId:'))).toBe(false); + }); + + it('emits requestId to stderr under --verbose (no-wait path)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl: typeof globalThis.fetch = async () => + new Response( + JSON.stringify({ + runId: 'run_c3', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }), + { + status: 200, + headers: { 'content-type': 'application/json', 'x-request-id': 'req_c3_visible' }, + }, + ); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: true, + dryRun: false, + testId: 'test_c3', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('requestId:'))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 3 — D4: RequestTimeoutError under --wait emits partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: Fix 3 — RequestTimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when poll throws RequestTimeoutError', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async (_input, _init) => { + callCount += 1; + if (callCount === 1) { + // Trigger succeeds + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // Polling call throws RequestTimeoutError (simulates per-request timeout) + throw new RequestTimeoutError(120000, 'req_timeout_test'); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + // Stdout must contain the partial object with runId + status:"running" + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + + // Stderr should mention the runId and suggest test wait + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain(TRIGGER_RESP.runId); + expect(stderrBlock).toContain('test wait'); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 5 — B2(c): --timeout hint fires on default, not on explicit timeout +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: Fix 5 — first-run timeout hint', () => { + function makeTriggerThenPassedFetch(): typeof globalThis.fetch { + let callCount = 0; + return (async (_input, _init) => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify(makePassedRun()), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + } + + it('emits the hint to stderr when timeoutIsDefault is true and output=text', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + timeoutIsDefault: true, + }, + { + credentialsPath, + fetchImpl: makeTriggerThenPassedFetch() as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(true); + }); + + it('does NOT emit the hint when timeoutIsDefault is false (explicit --timeout)', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + timeoutIsDefault: false, + }, + { + credentialsPath, + fetchImpl: makeTriggerThenPassedFetch() as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(false); + }); + + it('does NOT emit the hint in json output mode', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + timeoutIsDefault: true, + }, + { + credentialsPath, + fetchImpl: makeTriggerThenPassedFetch() as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(false); + }); +}); + +// Note: Finding B (advisory dup-name lookup short deadline) tests live in +// test.test.ts which has the makeFetch + runCreate helpers needed for +// lightweight code-file-based create tests without OOM-prone AbortController +// race setups. + +// --------------------------------------------------------------------------- +// Finding C (codex round-2) — timeout partial routes through renderer +// --------------------------------------------------------------------------- + +describe('[finding-C] runTestRun --wait RequestTimeoutError — text mode renders human-readable', () => { + it('text mode: stdout contains runId label line (not raw JSON) on RequestTimeoutError', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async (_input, _init) => { + callCount++; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + throw new RequestTimeoutError(120000, 'req_c_test'); + }; + + const stdoutLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'text', // TEXT MODE — the fix ensures this renders readable + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutBlock = stdoutLines.join('\n'); + // Must contain the human-readable label (not '{"runId":') + expect(stdoutBlock).toContain('runId'); + expect(stdoutBlock).toContain('running'); + expect(stdoutBlock).not.toMatch(/^\{/); // not a raw JSON object at the start + }); + + it('json mode: stdout has merged create-chain envelope when createContext supplied', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async (_input, _init) => { + callCount++; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + throw new RequestTimeoutError(120000, 'req_c_chain_test'); + }; + + const stdoutLines: string[] = []; + const createContext = { + testId: 'test_c_chain', + codeVersion: 'v1', + createdAt: '2026-06-07T00:00:00.000Z', + projectId: 'project_abc', + type: 'frontend' as const, + name: 'Chain Test', + }; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + createContext, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + testId: string; + run: { runId: string; status: string }; + }; + // Merged envelope: create context + run partial + expect(stdoutJson.testId).toBe('test_c_chain'); + expect(stdoutJson.run.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.run.status).toBe('running'); + }); +}); + +// --------------------------------------------------------------------------- +// Finding D (codex round-2) — 409 conflict auto-resume without --target-url +// → timeout partial carries the real in-flight targetUrl (not '') +// --------------------------------------------------------------------------- + +describe('[finding-D] 409 conflict auto-resume no --target-url + RequestTimeoutError → partial has real targetUrl', () => { + it('timeout partial carries in-flight run targetUrl (not null or "")', async () => { + // Regression: before the fix, the synthesised triggerResponse.targetUrl was '' + // (empty string) when no --target-url was supplied. The timeout partial then + // published '' as the environment provenance. After the fix, the CLI fetches + // GET /runs/{currentRunId} and binds the real URL. + const { credentialsPath } = makeCreds(); + const conflictBody = { + error: { + code: 'CONFLICT', + message: 'Run already in flight.', + nextAction: '', + requestId: 'req_d01', + details: { reason: 'run_in_flight', currentRunId: 'run_inflight_d01' }, + }, + }; + const inFlightRun: RunResponse = { + ...makePassedRun(), + runId: 'run_inflight_d01', + targetUrl: 'https://real-env.example.com', + status: 'running' as const, + }; + + // Track which calls have been made so each call serves a distinct role. + const calls: string[] = []; + const fetchImpl: typeof globalThis.fetch = async (input, init) => { + const url = typeof input === 'string' ? input : (input as Request).url; + const method = (init?.method ?? 'GET').toUpperCase(); + calls.push(`${method} ${url}`); + + // First call: trigger POST /tests/{id}/runs → 409 conflict + if (method === 'POST' && url.includes('/tests/') && url.endsWith('/runs')) { + return new Response(JSON.stringify(conflictBody), { + status: 409, + headers: { 'content-type': 'application/json' }, + }); + } + // Second call: GET /runs/{id} (getRun to fetch in-flight run details) + // This is used BOTH by the 409 handler (to get targetUrl) and by polling. + // We differentiate: the first GET /runs/{id} is the advisory lookup; + // subsequent GET /runs/{id} calls are polling → throw RequestTimeoutError. + if (method === 'GET' && url.includes('/runs/run_inflight_d01')) { + const runGetCalls = calls.filter( + c => c === `GET ${url.split('?')[0]}` || c.startsWith(`GET ${url.split('?')[0]}`), + ).length; + if (runGetCalls <= 1) { + // First GET /runs/{id} — advisory lookup + return new Response(JSON.stringify(inFlightRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // Second+ GET /runs/{id} — polling → simulate per-request timeout + throw new RequestTimeoutError(120000, 'req_d_timeout'); + } + // Any other call → timeout + throw new RequestTimeoutError(120000, 'req_d_fallback'); + }; + + const stdoutLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + // No --target-url supplied + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string | null; + }; + + // The partial must carry the REAL in-flight targetUrl — not '' or null. + expect(stdoutJson.runId).toBe('run_inflight_d01'); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe('https://real-env.example.com'); + expect(stdoutJson.targetUrl).not.toBe(''); + }); +}); + +// --------------------------------------------------------------------------- +// M4 piece-2 — `test run --all --project ` (fresh wave-ordered batch run) +// --------------------------------------------------------------------------- + +describe('runTestRunAll — batch fresh run', () => { + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + function makePassedRun(runId: string, testId: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 3, failedCount: 0 }, + }; + } + + it('routes to POST /tests/batch/run with correct body shape (projectId + source)', async () => { + const { credentialsPath } = makeCreds(); + type Captured = { url: string; method: string; body: unknown }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + }); + return { body: BATCH_FRESH_RESP }; + }); + const out: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const post = captured.find(c => c.method === 'POST' && (c.url as string).includes('batch/run')); + expect(post).toBeDefined(); + expect(post!.body).toMatchObject({ projectId: 'project_be', source: 'cli' }); + // No testIds field when --filter is absent (run all) + expect((post!.body as Record).testIds).toBeUndefined(); + }); + + it('includes testIds when --filter is set (resolves via GET /tests then filters)', async () => { + const { credentialsPath } = makeCreds(); + const allTests = [ + { + id: 'test_a', + name: 'Create user', + type: 'backend', + status: 'ready', + projectId: 'project_be', + createdFrom: 'cli', + createdAt: '', + updatedAt: '', + }, + { + id: 'test_b', + name: 'Read user', + type: 'backend', + status: 'ready', + projectId: 'project_be', + createdFrom: 'cli', + createdAt: '', + updatedAt: '', + }, + { + id: 'test_c', + name: 'Delete user', + type: 'backend', + status: 'ready', + projectId: 'project_be', + createdFrom: 'cli', + createdAt: '', + updatedAt: '', + }, + ]; + type Captured = { url: string; body: unknown }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + captured.push({ url, body: init.body ? JSON.parse(init.body as string) : undefined }); + if (method === 'GET') return { body: { items: allTests, nextToken: null } }; + return { + body: { + accepted: [{ testId: 'test_b', runId: 'run_b', enqueuedAt: '...' }], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + } satisfies BatchRunFreshResponse, + }; + }); + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + nameFilter: 'read', // case-insensitive → matches 'Read user' + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + const post = captured.find(c => (c.url as string).includes('batch/run') && c.body); + expect(post).toBeDefined(); + expect((post!.body as Record).testIds).toEqual(['test_b']); + // Should report the filter skip count on stderr + expect(stderrLines.some(l => l.includes('--filter'))).toBe(true); + }); + + it('positional + --all → exit 5 mutual exclusion (via command wiring)', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['run', 'test_xyz', '--all', '--project', 'proj_1'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('missing both positional and --all → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect(test.parseAsync(['run'], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('--all without --project → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect(test.parseAsync(['run', '--all'], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('--all --target-url → exit 5 (target-url has no effect on BE-only batch)', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync( + ['run', '--all', '--project', 'proj_1', '--target-url', 'https://example.com'], + { from: 'user' }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it(' --filter (without --all) → exit 5 (filter is --all-only)', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['run', 'test_xyz', '--filter', 'login'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--wait polls each accepted runId and returns on all-pass (exit 0)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + // Poll GET /runs/ + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + return { body: makePassedRun(runId, runId === 'run_fresh_01' ? 'test_be_01' : 'test_be_02') }; + }); + const out: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const payload = JSON.parse(out.join('\n')) as { accepted: Array<{ status: string }> }; + expect(payload.accepted.every(r => r.status === 'passed')).toBe(true); + }); + + it('--wait with a failed run → exit 1', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + const run = makePassedRun(runId, 'test_be_01'); + if (runId === 'run_fresh_01') + return { body: { ...run, status: 'failed', failureKind: 'assertion' } }; + return { body: run }; + }); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + }); + + it('skippedFrontend[] reported on stderr as advisory', async () => { + const { credentialsPath } = makeCreds(); + const respWithSkipped: BatchRunFreshResponse = { + accepted: [{ testId: 'test_be_01', runId: 'run_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + conflicts: [], + deferred: [], + skippedFrontend: ['test_fe_01', 'test_fe_02'], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { body: respWithSkipped }; + }); + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('frontend'))).toBe(true); + expect(stderrLines.some(l => l.includes('2'))).toBe(true); + }); + + it('deferred[] (rate-limited) forces exit 7 with a retry hint (non-wait)', async () => { + const { credentialsPath } = makeCreds(); + const respWithDeferred: BatchRunFreshResponse = { + accepted: [{ testId: 'test_be_01', runId: 'run_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + conflicts: [], + deferred: [{ testId: 'test_be_02' }, { testId: 'test_be_03' }], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { body: respWithDeferred }; + }); + const stderrLines: string[] = []; + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + expect(stderrLines.some(l => l.toLowerCase().includes('rate-deferred'))).toBe(true); + }); + + it('all-conflict, nothing accepted → CONFLICT (exit 6)', async () => { + const { credentialsPath } = makeCreds(); + const respAllConflict: BatchRunFreshResponse = { + accepted: [], + conflicts: [{ testId: 'test_be_01' }], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { body: respAllConflict }; + }); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 6 }); + }); + + it('skippedIntegration[] reported on stderr as advisory', async () => { + const { credentialsPath } = makeCreds(); + const respWithIntegration: BatchRunFreshResponse = { + accepted: [{ testId: 'test_be_01', runId: 'run_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [{ testId: 'test_be_integ_01' }], + }; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { body: respWithIntegration }; + }); + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect( + stderrLines.some(l => l.includes('[advisory]') && l.toLowerCase().includes('integration')), + ).toBe(true); + }); + + it('--dry-run returns sample without network call', async () => { + const fetchImpl = vi.fn() as unknown as FetchImpl; + const out: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { fetchImpl, stdout: line => out.push(line), stderr: () => undefined, sleep: instantSleep }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + // Should print a non-empty JSON body (sample or envelope) + expect(out.length).toBeGreaterThan(0); + }); + + it('--wait with timeout → exit 7', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + // Always return running so we hit timeout + return { + body: { + runId: 'run_fresh_01', + testId: 'test_be_01', + projectId: 'project_be', + userId: 'u1', + status: 'running', + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: null, + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 3, completed: 1, passedCount: 0, failedCount: 0 }, + } satisfies RunResponse, + }; + }); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 1, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + }); +}); + +// --------------------------------------------------------------------------- +// [P1] D3 deferred-retry: conflicts discovered during retries must be merged +// into the running conflicts collection and appear in the final summary/JSON. +// Without the fix the code only logged them; the final accounting still showed +// only initial conflicts → everything-deferred→retried→conflicted would exit 0 +// with zero accepted/deferred/conflicts reported. +// --------------------------------------------------------------------------- + +describe('[codex-P1] run --all deferred-retry: retry-conflicts merged into final accounting', () => { + it('deferred→conflict on retry: summary.conflicts reflects retry-returned conflicts; exits 6 when all paths resolve to conflict', async () => { + const { credentialsPath } = makeCreds(); + + // Initial dispatch: 1 deferred, 0 accepted, 0 initial conflicts. + const initialResp: BatchRunFreshResponse = { + accepted: [], + deferred: [{ testId: 'test_deferred' }], + conflicts: [], + skippedFrontend: [], + skippedIntegration: [], + }; + // Retry response: the deferred test is now in-flight (conflict). + const retryResp: BatchRunFreshResponse = { + accepted: [], + deferred: [], + conflicts: [{ testId: 'test_deferred' }], + skippedFrontend: [], + skippedIntegration: [], + }; + + let batchCallCount = 0; + const printed: Array> = []; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { body: batchCallCount === 1 ? initialResp : retryResp }; + } + return errorBody('NOT_FOUND'); + }); + + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 300, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + // Should not reach here: exit 6 because accepted=0 and conflicts>0 after retry + throw new Error('Expected runTestRunAll to throw'); + } catch (err) { + // After D3 retries drain deferred→conflict: accepted=0, conflicts=[test_deferred] + // The code should throw CONFLICT (exit 6) because accepted=0 and conflicts>0. + expect((err as ApiError).exitCode).toBe(6); + } + + // The retry must have fired (initial + 1 retry = 2 calls, since deferred drains) + expect(batchCallCount).toBeGreaterThanOrEqual(2); + + // The final JSON payload must include the retry-discovered conflict, not just the initial [] + const withConflicts = printed.find( + p => Array.isArray(p.conflicts) && (p.conflicts as unknown[]).length > 0, + ); + expect(withConflicts).toBeDefined(); + expect( + (withConflicts?.conflicts as Array<{ testId: string }>).some( + c => c.testId === 'test_deferred', + ), + ).toBe(true); + }); + + it('deferred→partially-conflict on retry: retry-conflicts appear in JSON summary; exit 0 for accepted portion', async () => { + const { credentialsPath } = makeCreds(); + + // Initial dispatch: 2 deferred, 1 accepted. + const initialResp: BatchRunFreshResponse = { + accepted: [{ testId: 'test_ok', runId: 'run_ok', enqueuedAt: '2026-06-09T10:00:00.000Z' }], + deferred: [{ testId: 'test_deferred_a' }, { testId: 'test_deferred_b' }], + conflicts: [], + skippedFrontend: [], + skippedIntegration: [], + }; + // Retry: one accepted, one becomes conflict. + const retryResp: BatchRunFreshResponse = { + accepted: [ + { + testId: 'test_deferred_a', + runId: 'run_deferred_a', + enqueuedAt: '2026-06-09T10:00:01.000Z', + }, + ], + deferred: [], + conflicts: [{ testId: 'test_deferred_b' }], + skippedFrontend: [], + skippedIntegration: [], + }; + + let batchCallCount = 0; + const printed: Array> = []; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + batchCallCount++; + return { body: batchCallCount === 1 ? initialResp : retryResp }; + } + // Poll: both accepted runs return passed + if (url.includes('/runs/run_ok')) { + return { + body: { + runId: 'run_ok', + testId: 'test_ok', + projectId: 'project_be', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + } satisfies RunResponse, + }; + } + if (url.includes('/runs/run_deferred_a')) { + return { + body: { + runId: 'run_deferred_a', + testId: 'test_deferred_a', + projectId: 'project_be', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + } satisfies RunResponse, + }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 300, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => printed.push(JSON.parse(line) as Record), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + + // summary must include the retry-discovered conflict in the conflicts count + const withSummary = printed.find(p => p.summary); + expect(withSummary).toBeDefined(); + expect((withSummary?.summary as Record).conflicts).toBe(1); + expect((withSummary?.summary as Record).passed).toBe(2); // both accepted runs passed + expect( + (withSummary?.conflicts as Array<{ testId: string }>).some( + c => c.testId === 'test_deferred_b', + ), + ).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// [P2] D3 idempotency key truncation: derived retry keys stay ≤ 256 chars +// --------------------------------------------------------------------------- + +describe('[codex-P2] run --all deferred-retry: idempotency key truncation', () => { + it('retry idempotency key does not exceed 256 chars when base key is at the 256-char limit', async () => { + const { credentialsPath } = makeCreds(); + + // A caller-supplied key exactly at the 256-char limit + const longKey = 'k'.repeat(256); + + const initialResp: BatchRunFreshResponse = { + accepted: [], + deferred: [{ testId: 'test_deferred' }], + conflicts: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const retryResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_deferred', runId: 'run_d', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + const capturedIdempotencyKeys: string[] = []; + let batchCallCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') { + const key = (init.headers as Record)?.['idempotency-key'] ?? ''; + capturedIdempotencyKeys.push(key); + batchCallCount++; + return { body: batchCallCount === 1 ? initialResp : retryResp }; + } + if (url.includes('/runs/run_d')) { + return { + body: { + runId: 'run_d', + testId: 'test_deferred', + projectId: 'project_be', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + } satisfies RunResponse, + }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 300, + maxConcurrency: 5, + idempotencyKey: longKey, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ); + + // All captured keys must be ≤ 256 chars + for (const key of capturedIdempotencyKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + // The retry key must differ from the base key (suffix was appended) + expect(capturedIdempotencyKeys.length).toBeGreaterThanOrEqual(2); + expect(capturedIdempotencyKeys[1]).not.toBe(longKey); + expect(capturedIdempotencyKeys[1]).toContain('deferred-retry'); + }); +}); + +// --------------------------------------------------------------------------- +// [B-E2E-01] Fix 1 regression — blocked/failed runs must exit 1 under --wait +// --------------------------------------------------------------------------- + +describe('[B-E2E-01] runTestRunAll --wait: non-passed runs must exit 1 (regression)', () => { + // The unit tests below pin the post-retry exit-code path so any future + // refactor that accidentally drops the `if (failed > 0) throw CLIError(…, 1)` + // guard is caught immediately (blocked status had no dedicated test before). + + const BATCH_RESP_BLOCKED: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_blocked_01', enqueuedAt: '2026-06-09T11:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_passed_02', enqueuedAt: '2026-06-09T11:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T11:00:00.000Z', + startedAt: '2026-06-09T11:00:01.000Z', + finishedAt: '2026-06-09T11:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + it('--wait with a blocked run → exit 1 (not 0)', async () => { + // Regression test: before the fix, 51 blocked/failed runs could exit 0. + // Specifically test `blocked` status which had no dedicated coverage. + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_RESP_BLOCKED }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_blocked_01') + return { body: makeTerminalRun('run_blocked_01', 'test_be_01', 'blocked') }; + if (runId === 'run_passed_02') + return { body: makeTerminalRun('run_passed_02', 'test_be_02', 'passed') }; + return errorBody('NOT_FOUND'); + }); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + }); + + it('--wait all blocked (51 blocked out of 51) → exit 1, not 0', async () => { + // Mirrors the live E2E observation: 51 blocked/failed in a 100-run batch. + // Uses a smaller batch (3 blocked) to keep the test fast. + const { credentialsPath } = makeCreds(); + const allBlocked: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-09T11:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-09T11:00:01.000Z' }, + { testId: 'test_3', runId: 'run_b3', enqueuedAt: '2026-06-09T11:00:02.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: allBlocked }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + return { body: makeTerminalRun(runId, 'test_x', 'blocked') }; + }); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + }); + + it('--wait mixed blocked+failed → exit 1 with summary.failed = 2', async () => { + const { credentialsPath } = makeCreds(); + const mixedBatch: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_p', runId: 'run_p', enqueuedAt: '2026-06-09T11:00:00.000Z' }, + { testId: 'test_b', runId: 'run_b', enqueuedAt: '2026-06-09T11:00:01.000Z' }, + { testId: 'test_f', runId: 'run_f', enqueuedAt: '2026-06-09T11:00:02.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: mixedBatch }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_p') return { body: makeTerminalRun('run_p', 'test_p', 'passed') }; + if (runId === 'run_b') return { body: makeTerminalRun('run_b', 'test_b', 'blocked') }; + if (runId === 'run_f') return { body: makeTerminalRun('run_f', 'test_f', 'failed') }; + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + let caughtError: unknown; + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + } catch (err) { + caughtError = err; + } + + expect(caughtError).toBeDefined(); + expect((caughtError as { exitCode?: number }).exitCode).toBe(1); + + // The JSON payload (written to stdout before throwing) must contain + // summary.failed = 2 (1 blocked + 1 failed). + const payload = JSON.parse(stdoutLines.join('\n')) as { + summary: { passed: number; failed: number }; + }; + expect(payload.summary.passed).toBe(1); + expect(payload.summary.failed).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +// runTestRunAll — --max-concurrency upper bound (Fix 2) +// --------------------------------------------------------------------------- + +describe('runTestRunAll — --max-concurrency validation', () => { + it('rejects --max-concurrency > 100 with VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 101, + }, + { credentialsPath, sleep: () => Promise.resolve() }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'max-concurrency' }), + }); + }); + + it('accepts --max-concurrency = 100 (boundary value, no validation error)', async () => { + const { credentialsPath } = makeCreds(); + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch(() => ({ body: BATCH_FRESH_RESP })); + // Should resolve without VALIDATION_ERROR + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 100, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ), + ).resolves.toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// dashboardUrl on run completion (colleague feedback 2026-06-10): terminal +// run output carries a Portal deep link — projectId+testId come from the +// GET /runs/{runId} wire row (single run) or opts+item (run --all). +// --------------------------------------------------------------------------- + +describe('dashboardUrl on run completion', () => { + const PROD_API = 'https://api.testsprite.com'; + + function triggerThenTerminal(run: RunResponse): typeof globalThis.fetch { + return makeFetch(url => { + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: run }; + }); + } + + it('run --wait (JSON, prod endpoint): final envelope includes dashboardUrl', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: triggerThenTerminal(makePassedRun()), + stdout: line => stdout.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_1/test/test_xyz', + ); + }); + + it('run --wait (text, prod endpoint): card ends with dashboard line', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: triggerThenTerminal(makePassedRun()), + stdout: line => stdout.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + expect(stdout.join('\n')).toContain( + 'dashboard https://www.testsprite.com/dashboard/tests/project_1/test/test_xyz', + ); + }); + + it('run --wait (unknown API host): no dashboardUrl field', async () => { + const { credentialsPath } = makeCreds(); // localhost default + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: triggerThenTerminal(makePassedRun()), + stdout: line => stdout.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBeUndefined(); + }); + + it('run --wait: empty projectId on the run row → no dashboardUrl (BE fallback guard)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const stdout: string[] = []; + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: triggerThenTerminal({ ...makePassedRun(), projectId: '' }), + stdout: line => stdout.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBeUndefined(); + }); + + it('run --all no-wait (prod endpoint): accepted items carry dashboardUrl; text has project line', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_f_02', enqueuedAt: '2026-06-10T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const jsonOut: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: line => jsonOut.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + const printed = JSON.parse(jsonOut.join('')) as { + accepted: Array<{ testId: string; dashboardUrl?: string }>; + }; + expect(printed.accepted[0]!.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_be/test/test_be_01', + ); + expect(printed.accepted[1]!.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_be/test/test_be_02', + ); + + const textOut: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: line => textOut.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + expect(textOut.join('\n')).toContain( + 'dashboard https://www.testsprite.com/dashboard/tests/project_be', + ); + }); + + it('run --all --wait (prod endpoint): summary items carry dashboardUrl + stderr Dashboard line', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const terminal: RunResponse = { + runId: 'run_f_01', + testId: 'test_be_01', + projectId: 'project_be', + userId: 'user_1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-10T10:00:00.000Z', + startedAt: '2026-06-10T10:00:01.000Z', + finishedAt: '2026-06-10T10:00:05.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }; + const jsonOut: string[] = []; + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(url => { + if (url.includes('batch/run')) return { body: batchResp }; + return { body: terminal }; + }), + stdout: line => jsonOut.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + const printed = JSON.parse(jsonOut.join('')) as { + accepted: Array<{ testId: string; dashboardUrl?: string }>; + }; + expect(printed.accepted[0]!.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_be/test/test_be_01', + ); + expect( + stderrLines.some(l => + l.includes('Dashboard: https://www.testsprite.com/dashboard/tests/project_be'), + ), + ).toBe(true); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts new file mode 100644 index 0000000..6921d57 --- /dev/null +++ b/src/commands/test.test.ts @@ -0,0 +1,7562 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Command } from 'commander'; +import { ApiError } from '../lib/errors.js'; +import { GLOBAL_OPTS_HINT } from '../lib/output.js'; +import { + type CliFailureContext, + type CliLatestResult, + type CliTest, + type CliTestCode, + type CliTestStep, + type TestDeps, + createTestCommand, + isPresignedCodeUrl, + runCodeGet, + runCodePut, + runCreate, + runCreateBatch, + runCreateFromPlan, + runDelete, + runFailureGet, + runFailureSummary, + runGet, + runList, + runPlanPut, + runResult, + runSteps, + runUpdate, +} from './test.js'; + +function disableExits(cmd: Command): void { + cmd.exitOverride(); + cmd.commands.forEach(disableExits); +} + +const FE_TEST: CliTest = { + id: 'test_fe', + projectId: 'project_alice', + name: 'Checkout happy path', + type: 'frontend', + createdFrom: 'portal', + status: 'failed', + createdAt: '2026-04-20T11:00:00.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', +}; + +const BE_TEST: CliTest = { + id: 'test_be', + projectId: 'project_alice', + name: 'Smoke — health check', + type: 'backend', + createdFrom: 'mcp', + status: 'passed', + createdAt: '2026-04-22T09:00:00.000Z', + updatedAt: '2026-05-05T11:00:30.000Z', +}; + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13502', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-p3-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +describe('createTestCommand — surface', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exposes the expected top-level subcommands', () => { + const test = createTestCommand(); + const names = test.commands.map(c => c.name()).sort(); + expect(names).toEqual([ + 'artifact', + 'code', + 'create', + 'create-batch', + 'delete', + 'delete-batch', + 'failure', + 'get', + 'list', + 'plan', + 'rerun', + 'result', + 'run', + 'steps', + 'update', + 'wait', + ]); + }); + + it('exposes the expected `code` subcommands', () => { + const test = createTestCommand(); + const code = test.commands.find(c => c.name() === 'code'); + expect(code).toBeDefined(); + expect(code!.commands.map(c => c.name()).sort()).toEqual(['get', 'put']); + }); + + it('exposes the expected `failure` subcommands', () => { + const test = createTestCommand(); + const failure = test.commands.find(c => c.name() === 'failure'); + expect(failure).toBeDefined(); + // M2.1 piece 3 adds `summary`. `get` is the bundle entry point; + // `summary` is the lightweight analysis-only triage card. + expect(failure!.commands.map(c => c.name()).sort()).toEqual(['get', 'summary']); + }); + + it('list exposes the documented filter and pagination flags (including --cursor alias)', () => { + const test = createTestCommand(); + const list = test.commands.find(c => c.name() === 'list')!; + const flagNames = list.options.map(o => o.long); + expect(flagNames).toEqual( + expect.arrayContaining([ + '--project', + '--type', + '--created-from', + '--page-size', + '--starting-token', + '--cursor', + '--max-items', + ]), + ); + }); + + it('failure get exposes --out and --failed-only flags (P5)', () => { + // P5 implements `failure get`. The only flags are --out (atomic on-disk + // bundle) and --failed-only (§7.4 narrow-budget filter). Pinning the + // surface so a future "consolidate flags" sweep doesn't drop them. + const test = createTestCommand(); + const failure = test.commands.find(c => c.name() === 'failure')!; + const failureGet = failure.commands.find(c => c.name() === 'get')!; + const flagNames = failureGet.options.map(o => o.long).sort(); + expect(flagNames).toEqual(['--failed-only', '--out']); + }); + + it('steps exposes pagination flags', () => { + const test = createTestCommand(); + const steps = test.commands.find(c => c.name() === 'steps'); + expect(steps).toBeDefined(); + const flagNames = steps!.options.map(o => o.long); + expect(flagNames).toEqual( + expect.arrayContaining(['--page-size', '--max-items', '--starting-token']), + ); + }); + + it('result exposes --include-analysis (M2.1) + M3.4 piece-5 --history flags', () => { + // M2.1 piece 3 adds `--include-analysis` to `test result`. + // M3.4 piece 5 adds `--history`, `--source`, `--since`, `--page-size`, `--cursor`. + // Pinning the surface so a future flag-consolidation sweep keeps every + // option intentional. Back-compat: bare `test result ` (no --history) + // still calls runResult and returns the M2 CliLatestResult shape. + const test = createTestCommand(); + const result = test.commands.find(c => c.name() === 'result'); + const flagNames = result!.options.map(o => o.long); + expect(flagNames).toEqual([ + '--include-analysis', + '--history', + '--source', + '--since', + '--page-size', + '--cursor', + ]); + }); + + it('code get exposes --out as its only option', () => { + // `--out` lets agents write the response to a file without a shell + // redirect (matters on Windows + when the wire shape carries a + // presigned URL we want to stream straight to disk). Pinning it + // here so a future "remove redundant flag" sweep doesn't take it. + const test = createTestCommand(); + const code = test.commands.find(c => c.name() === 'code'); + const codeGet = code!.commands.find(c => c.name() === 'get'); + const flagNames = codeGet!.options.map(o => o.long); + expect(flagNames).toEqual(['--out']); + }); + + // ------------------------------------------------------------------------- + // GLOBAL_OPTS_HINT sweep — every leaf subcommand must surface the footer + // pointing at `testsprite --help` so users discover --dry-run, --output, + // --profile, --endpoint-url, --verbose, and --debug. + // + // This addresses the dogfood entry (2026-05-15): "M3.3 subcommands omitted + // GLOBAL_OPTS_HINT". The M3.3 fix landed in fix/cli-m3.3-consolidated-fixes; + // this sweep guards the full surface (M2 + M3.x) against future regressions. + // ------------------------------------------------------------------------- + + function captureHelp(cmd: ReturnType): string { + let out = ''; + cmd.configureOutput({ + writeOut: (str: string) => { + out += str; + }, + }); + cmd.outputHelp(); + return out; + } + + it('M3.3: test run --help includes GLOBAL_OPTS_HINT', () => { + const test = createTestCommand(); + const run = test.commands.find(c => c.name() === 'run')!; + const help = captureHelp(run); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--dry-run'); + }); + + it('M3.3: test wait --help includes GLOBAL_OPTS_HINT', () => { + const test = createTestCommand(); + const wait = test.commands.find(c => c.name() === 'wait')!; + const help = captureHelp(wait); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--dry-run'); + }); + + it('M3.3: test artifact get --help includes GLOBAL_OPTS_HINT', () => { + const test = createTestCommand(); + const artifact = test.commands.find(c => c.name() === 'artifact')!; + const artifactGet = artifact.commands.find(c => c.name() === 'get')!; + const help = captureHelp(artifactGet); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--dry-run'); + }); + + it('M3.3: test failure get --help includes GLOBAL_OPTS_HINT', () => { + const test = createTestCommand(); + const failure = test.commands.find(c => c.name() === 'failure')!; + const failureGet = failure.commands.find(c => c.name() === 'get')!; + const help = captureHelp(failureGet); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--dry-run'); + }); + + it('M3.3: test failure summary --help includes GLOBAL_OPTS_HINT', () => { + const test = createTestCommand(); + const failure = test.commands.find(c => c.name() === 'failure')!; + const failureSummary = failure.commands.find(c => c.name() === 'summary')!; + const help = captureHelp(failureSummary); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--dry-run'); + }); + + it('M2 sweep: all remaining leaf subcommands include GLOBAL_OPTS_HINT', () => { + // Covers list, get, create, create-batch, steps, result, update, delete, + // code get, code put, plan put — the full M2 surface that the dogfood + // entry (2026-05-13) flagged and fix/cli-dogfood-bundle-2026-05-16 fixed. + const test = createTestCommand(); + + // Flat leaf commands (direct children of `test`) + const flatLeaves = [ + 'list', + 'get', + 'create', + 'create-batch', + 'steps', + 'result', + 'update', + 'delete', + ]; + for (const name of flatLeaves) { + const cmd = test.commands.find(c => c.name() === name)!; + expect(cmd, `test ${name} must exist`).toBeDefined(); + const help = captureHelp(cmd); + expect(help, `test ${name} --help must include GLOBAL_OPTS_HINT`).toContain(GLOBAL_OPTS_HINT); + } + + // Nested: test code get, test code put + const code = test.commands.find(c => c.name() === 'code')!; + for (const name of ['get', 'put']) { + const cmd = code.commands.find(c => c.name() === name)!; + const help = captureHelp(cmd); + expect(help, `test code ${name} --help must include GLOBAL_OPTS_HINT`).toContain( + GLOBAL_OPTS_HINT, + ); + } + + // Nested: test plan put + const plan = test.commands.find(c => c.name() === 'plan')!; + const planPut = plan.commands.find(c => c.name() === 'put')!; + const planHelp = captureHelp(planPut); + expect(planHelp, 'test plan put --help must include GLOBAL_OPTS_HINT').toContain( + GLOBAL_OPTS_HINT, + ); + }); +}); + +describe('runList', () => { + it('passes projectId, type, and createdFrom to the facade query string', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + createdFrom: 'portal', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seen[0]).toContain('projectId=project_alice'); + expect(seen[0]).toContain('type=frontend'); + expect(seen[0]).toContain('createdFrom=portal'); + }); + + it('accepts --created-from cli and passes createdFrom=cli to the wire (dogfood 2026-06-04)', async () => { + // End-to-end through parseEnumFlag: backend now stamps createFrom='cli' + // on `testsprite test create` rows, so the filter must accept 'cli'. + // If parseEnumFlag rejected it, this would throw VALIDATION_ERROR before + // any fetch and `seen` would stay empty. + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [], nextToken: null } }; + }); + const test = createTestCommand({ credentialsPath, fetchImpl, stdout: () => undefined }); + await test.parseAsync(['list', '--project', 'project_alice', '--created-from', 'cli'], { + from: 'user', + }); + expect(seen[0]).toContain('createdFrom=cli'); + }); + + it('auto-pages until nextToken is null', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch(() => { + calls += 1; + if (calls === 1) return { body: { items: [FE_TEST], nextToken: 'cursor-1' } }; + return { body: { items: [BE_TEST], nextToken: null } }; + }); + const out: string[] = []; + const page = await runList( + { profile: 'default', output: 'json', debug: false, projectId: 'project_alice' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(calls).toBe(2); + expect(page.items).toHaveLength(2); + expect(JSON.parse(out[0]!).items).toHaveLength(2); + }); + + it('--page-size returns one page (no auto-paging) and surfaces the cursor', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [FE_TEST], nextToken: 'opaque-A' } }; + }); + const page = await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + pageSize: 1, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seen).toHaveLength(1); + expect(seen[0]).toContain('pageSize=1'); + expect(page.nextToken).toBe('opaque-A'); + }); + + it('--max-items caps result count across multiple pages', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch(() => { + calls += 1; + return { + body: { + items: [ + { ...FE_TEST, id: `t_${calls}_a` }, + { ...FE_TEST, id: `t_${calls}_b` }, + ], + nextToken: calls < 3 ? `cursor-${calls}` : null, + }, + }; + }); + const page = await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + maxItems: 3, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(page.items).toHaveLength(3); + expect(page.nextToken).toBe('cursor-2'); + }); + + it('rejects --type=junk locally with VALIDATION_ERROR (no network call)', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['list', '--project', 'project_alice', '--type', 'junk'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects --created-from=junk locally with VALIDATION_ERROR', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['list', '--project', 'project_alice', '--created-from', 'junk'], { + from: 'user', + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects --page-size=0 locally with VALIDATION_ERROR (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('network should not be hit'); + }); + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + pageSize: 0, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', details: { field: 'page-size' } }); + }); + + it('forwards a server-side VALIDATION_ERROR envelope as ApiError exit 5', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 400, + body: { + error: { + code: 'VALIDATION_ERROR', + message: 'bad cursor', + nextAction: 'pass nextToken from a previous response', + requestId: 'req_test', + details: { field: 'cursor' }, + }, + }, + })); + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + startingToken: 'bogus', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toBeInstanceOf(ApiError); + }); + + it('text mode renders mixed FE/BE rows with header + status column', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST, BE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('ID'); + expect(block).toContain('NAME'); + expect(block).toContain('TYPE'); + expect(block).toContain('FROM'); + expect(block).toContain('STATUS'); + expect(block).toContain('UPDATED'); + expect(block).toContain('Checkout happy path'); + expect(block).toContain('Smoke — health check'); + // Both types render explicitly; the agent loop reads `type` directly + // from JSON, but humans glance at the column. + expect(block).toContain('frontend'); + expect(block).toContain('backend'); + // Both createdFrom variants render — verifies the column doesn't + // accidentally hardcode a value. + expect(block).toContain('portal'); + expect(block).toContain('mcp'); + }); + + it('text mode reads "No tests." when items is empty and nextToken is null', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toBe('No tests.'); + }); + + it('text mode reads "No tests on this page." with cursor when filtered out', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: 'still-more' } })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('No tests on this page.'); + expect(block).toContain('nextToken: still-more'); + }); + + it('--starting-token resumes pagination from the supplied cursor', async () => { + const { credentialsPath } = makeCreds(); + const seenCursors: Array = []; + const fetchImpl = makeFetch(url => { + const match = /cursor=([^&]+)/.exec(url); + seenCursors.push(match ? decodeURIComponent(match[1]!) : null); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + startingToken: 'resume-here', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenCursors[0]).toBe('resume-here'); + }); +}); + +// Fix 2: --cursor alias on `test list` +// Before this fix, `test list --cursor ` would emit +// "error: unknown option '--cursor'" (exit 5 via Commander error handling) +// because `test list` only had `--starting-token`. +describe('createTestCommand list — --cursor alias', () => { + it('--cursor is accepted by `test list` without "unknown option" error', async () => { + // This test exercises the Commander-level wiring, which is where the + // --cursor → startingToken merge happens (in the .action() handler). + // `runList` itself has no `cursor` field — the alias is resolved before + // `runList` is called. + const { credentialsPath } = makeCreds(); + const seenCursors: Array = []; + const fetchImpl = makeFetch(url => { + const match = /cursor=([^&]+)/.exec(url); + seenCursors.push(match ? decodeURIComponent(match[1]!) : null); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + const deps: TestDeps = { credentialsPath, fetchImpl, stdout: () => undefined }; + const test = createTestCommand(deps); + // Commander's .parseAsync with 'user' source parses bare tokens + // and flags relative to the command. --project is required. + await test.parseAsync( + ['list', '--project', 'project_alice', '--cursor', 'cursor-alias-token'], + { + from: 'user', + }, + ); + expect(seenCursors[0]).toBe('cursor-alias-token'); + }); + + it('--starting-token takes precedence over --cursor when both are supplied', async () => { + const { credentialsPath } = makeCreds(); + const seenCursors: Array = []; + const fetchImpl = makeFetch(url => { + const match = /cursor=([^&]+)/.exec(url); + seenCursors.push(match ? decodeURIComponent(match[1]!) : null); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + const deps: TestDeps = { credentialsPath, fetchImpl, stdout: () => undefined }; + const test = createTestCommand(deps); + await test.parseAsync( + [ + 'list', + '--project', + 'project_alice', + '--starting-token', + 'primary-token', + '--cursor', + 'alias-token', + ], + { from: 'user' }, + ); + expect(seenCursors[0]).toBe('primary-token'); + }); +}); + +describe('createTestCommand list — required flag', () => { + it('rejects when --project is missing with VALIDATION_ERROR (not commander)', async () => { + const test = createTestCommand(); + disableExits(test); + // We deliberately removed `.requiredOption` so the local validator + // (`requireProjectId`) runs and throws the typed envelope. Commander's + // built-in "required option" error would surface as exit 1, breaking + // the CLI error spec §2 contract that "missing required field" + // is a `VALIDATION_ERROR` (exit 5). + try { + await test.parseAsync(['list'], { from: 'user' }); + expect.unreachable('expected ApiError'); + } catch (err) { + const apiErr = err as { code?: string; exitCode?: number; details?: { field?: string } }; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.details?.field).toBe('project'); + } + }); + + it('M2.1 piece 2: --status passes the comma-separated value to the wire', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [], nextToken: null } }; + }); + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'failed,blocked', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // Server-side filter applied before pagination — the status query + // param must reach the wire so a long-tail of 50+ tests doesn't + // get fetched + filtered client-side. + expect(seen[0]).toMatch(/[?&]status=failed%2Cblocked/); + }); + + it('M2.1 piece 2: --status rejects unknown tokens locally (exit 5, no fetch)', async () => { + // Defense: a typo like `--status fail` shouldn't silently filter + // to nothing. Fail fast client-side with the accepted set in + // the error envelope. + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: { items: [], nextToken: null } }; + }); + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'fail', // typo for `failed` + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('runGet', () => { + it('GETs /tests/{id} and prints the §6.2 fields in text mode', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: FE_TEST }; + }); + const out: string[] = []; + const test = await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen[0]).toContain('/tests/test_fe'); + expect(test.id).toBe('test_fe'); + const block = out.join('\n'); + expect(block).toContain('id: test_fe'); + expect(block).toContain('projectId: project_alice'); + expect(block).toContain('status: failed'); + }); + + it('renders a `blocked` test row (M2.1 piece 1 — distinct from failed)', async () => { + // Regression for the M2.1 contract flip: pre-M2.1 the wire shape + // would have arrived as `status: failed` for the same source row. + // The text renderer must surface `blocked` byte-for-byte without + // collapsing it back to the legacy bucket. Also asserts the + // structured `details` parsed cleanly through the JSON envelope. + const blockedTest: CliTest = { + ...FE_TEST, + id: 'test_blocked', + status: 'blocked', + details: { + processingStatus: 'Idle', + testStatus: 'Blocked', + rawStatus: 'ps=Idle; ts=Blocked', + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: blockedTest })); + const out: string[] = []; + const test = await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_blocked' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(test.status).toBe('blocked'); + expect(test.details).toEqual({ + processingStatus: 'Idle', + testStatus: 'Blocked', + rawStatus: 'ps=Idle; ts=Blocked', + }); + expect(out.join('\n')).toContain('status: blocked'); + }); + + it('renders the planSteps count when the facade ships planStepCount (M3.4)', async () => { + const withPlan: CliTest = { ...FE_TEST, planStepCount: 3 }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withPlan })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toContain('planSteps: 3'); + }); + + it('omits the planSteps line when planStepCount is null or absent (M3.4)', async () => { + const noPlan: CliTest = { ...FE_TEST, planStepCount: null }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: noPlan })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).not.toContain('planSteps:'); + }); + + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Resource not found.', + nextAction: 'Check the id with `testsprite test list --project `.', + requestId: 'req_test', + details: { resource: 'test', id: 'test_missing' }, + }, + }, + })); + await expect( + runGet( + { profile: 'default', output: 'json', debug: false, testId: 'test_missing' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND', exitCode: 4 }); + }); + + it('URL-encodes test ids with `/` or `?` in them', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: FE_TEST }; + }); + await runGet( + { profile: 'default', output: 'json', debug: false, testId: 'odd/id?weird' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seen[0]).toContain('odd%2Fid%3Fweird'); + }); + + it('M2.1 piece 4: renders project: () when projectName is set', async () => { + const TEST_WITH_PROJECT_NAME: CliTest = { + ...FE_TEST, + projectId: 'project_alice', + projectName: 'Checkout', + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_WITH_PROJECT_NAME })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('project: Checkout (project_alice)'); + // Pre-M2.1 line that just had projectId should not appear. + expect(block).not.toContain('projectId: project_alice'); + }); + + it('M2.1 piece 4: falls back to projectId: when projectName is null', async () => { + const TEST_NO_PROJECT_NAME: CliTest = { + ...FE_TEST, + projectId: 'project_alice', + projectName: null, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_NO_PROJECT_NAME })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toContain('projectId: project_alice'); + }); + + // G1a — priority field surfacing + it('G1a: renders priority: p1 line when backend ships priority', async () => { + const withPriority: CliTest = { ...FE_TEST, priority: 'p1' }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withPriority })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toContain('priority: p1'); + }); + + it('G1a: omits priority line when priority is null', async () => { + const nullPriority: CliTest = { ...FE_TEST, priority: null }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: nullPriority })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).not.toContain('priority:'); + }); + + it('G1a: omits priority line when priority field is absent (pre-G1a backend)', async () => { + // FE_TEST has no priority field — matches pre-G1a wire shape + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: FE_TEST })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).not.toContain('priority:'); + }); + + it('G1a: priority is included in --output json pass-through', async () => { + const withPriority: CliTest = { ...FE_TEST, priority: 'p0' }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withPriority })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const parsed = JSON.parse(out.join('')) as CliTest; + expect(parsed.priority).toBe('p0'); + }); +}); + +// ---------- P4: code / steps / result ---------- + +const TEST_CODE_INLINE: CliTestCode = { + testId: 'test_fe', + language: 'typescript', + framework: 'playwright', + code: [ + "import { test, expect } from '@playwright/test';", + "test('checkout', async ({ page }) => {", + ' await page.goto(process.env.TARGET_URL!);', + '});', + '', + ].join('\n'), + codeVersion: 'v3', + etag: 'sha256:abc', +}; + +const TEST_CODE_PRESIGNED: CliTestCode = { + testId: 'test_large', + language: 'typescript', + framework: 'playwright', + code: 'https://s3-presigned.example.com/codes/test_large?X-Amz-fixture', + codeVersion: 'v1', + etag: null, +}; + +const STEP_PASSED: CliTestStep = { + testId: 'test_fe', + stepIndex: 4, + action: 'click', + description: 'Click the cart icon', + status: 'passed', + screenshotUrl: 'https://s3-presigned.example.com/snap/04.png?X', + htmlSnapshotUrl: 'https://s3-presigned.example.com/snap/04.html?X', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: '2026-05-05T12:34:55.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', +}; + +const STEP_FAILED: CliTestStep = { + ...STEP_PASSED, + stepIndex: 5, + action: 'click', + description: 'Click the submit button', + status: 'failed', + screenshotUrl: 'https://s3-presigned.example.com/snap/05.png?X', + htmlSnapshotUrl: 'https://s3-presigned.example.com/snap/05.html?X', + capturedAt: '2026-05-05T12:34:56.000Z', +}; + +const STEP_PENDING: CliTestStep = { + ...STEP_PASSED, + stepIndex: 6, + action: 'expect', + description: 'Expect order confirmation heading', + status: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + capturedAt: null, + updatedAt: '2026-05-05T12:34:58.000Z', +}; + +const RESULT_FAILED: CliLatestResult = { + testId: 'test_fe', + status: 'failed', + startedAt: '2026-05-05T12:34:00.000Z', + finishedAt: '2026-05-05T12:34:58.000Z', + videoUrl: 'https://s3-presigned.example.com/video/run_abc.mp4?X', + failureAnalysisUrl: 'https://s3-presigned.example.com/analysis/run_abc.json?X', + snapshotId: 'snap_2026_05_05_b2f9a1c8', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + targetUrl: 'https://staging.example.com/checkout', + failedStepIndex: 5, + failureKind: 'assertion', + summary: { passed: 4, failed: 1, skipped: 0 }, +}; + +const RESULT_PASSED: CliLatestResult = { + testId: 'test_passed', + status: 'passed', + startedAt: '2026-05-05T07:59:30.000Z', + finishedAt: '2026-05-05T08:00:12.000Z', + videoUrl: 'https://s3-presigned.example.com/video/run_xyz.mp4?X', + failureAnalysisUrl: null, + snapshotId: 'snap_2026_05_05_e1b9c2a4', + runIdIfAvailable: 'run_xyz', + codeVersion: 'v2', + targetUrl: 'https://staging.example.com/checkout', + failedStepIndex: null, + failureKind: null, + summary: { passed: 8, failed: 0, skipped: 0 }, +}; + +describe('isPresignedCodeUrl', () => { + it('treats https:// as presigned and source-looking strings as inline', () => { + expect(isPresignedCodeUrl('https://s3.example.com/x')).toBe(true); + expect(isPresignedCodeUrl('http://insecure.example.com/x')).toBe(false); + expect(isPresignedCodeUrl("import { test } from '@playwright/test';")).toBe(false); + expect(isPresignedCodeUrl('')).toBe(false); + }); +}); + +describe('runCodeGet', () => { + it('JSON mode prints the §6.3 wire shape verbatim and skips the URL fetch', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: TEST_CODE_INLINE }; + }); + const out: string[] = []; + const got = await runCodeGet( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen).toHaveLength(1); + expect(seen[0]).toContain('/tests/test_fe/code'); + expect(got).toEqual(TEST_CODE_INLINE); + expect(JSON.parse(out[0]!)).toEqual(TEST_CODE_INLINE); + }); + + it('text mode prints the inline source body byte-exact via rawStdout', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_CODE_INLINE })); + const chunks: string[] = []; + await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + rawStdout: chunk => { + chunks.push(chunk); + }, + }, + ); + // §12.7: text is human-only; JSON consumers see the wire shape. + // The CLI writes the source byte-exactly so `> file.ts` piping + // produces a runnable file; no implicit newline coercion. + expect(chunks.join('')).toBe(TEST_CODE_INLINE.code); + // The source must not be wrapped in JSON braces in text mode. + expect(chunks.join('').startsWith('{')).toBe(false); + }); + + it('text mode streams a presigned URL chunk-wise with no API key header', async () => { + const { credentialsPath } = makeCreds(); + const seenHeaders: Array = []; + const STREAMED_CHUNKS = [ + "import { test } from '@playwright/test';\n", + "test('huge', async ({ page }) => {\n", + ' /* very long file */\n', + '});\n', + ]; + const fetchImpl = ((input: Parameters[0], init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + // Presigned URL fetch — capture headers to assert no API key leak, + // and serve a multi-chunk stream so the streaming branch is + // actually exercised (not just response.text()). + seenHeaders.push(init.headers ?? {}); + const stream = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + for (const chunk of STREAMED_CHUNKS) controller.enqueue(encoder.encode(chunk)); + controller.close(); + }, + }); + return Promise.resolve( + new Response(stream, { status: 200, headers: { 'content-type': 'text/plain' } }), + ); + }) as typeof globalThis.fetch; + const chunks: string[] = []; + await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_large' }, + { + credentialsPath, + fetchImpl, + rawStdout: chunk => { + chunks.push(chunk); + }, + }, + ); + // Bytes must arrive intact end-to-end and chunked rather than as + // one buffered blob — proves we're streaming, not response.text(). + expect(chunks.join('')).toBe(STREAMED_CHUNKS.join('')); + expect(chunks.length).toBeGreaterThanOrEqual(STREAMED_CHUNKS.length); + // Presigned URL fetches must NOT include an x-api-key header. The + // URL itself is the bearer of authority. + for (const headers of seenHeaders) { + const h = new Headers(headers); + expect(h.get('x-api-key')).toBeNull(); + } + }); + + it('wraps a fetchImpl rejection on the presigned URL as TransportError (exit 10)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = ((input: Parameters[0]) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + // Simulate a DNS / TLS reset — the kind of failure that bypasses + // HttpClient retry budget because the presigned URL is fetched + // directly. the CLI error spec §7 says this must surface as + // UNAVAILABLE / exit 10, never as Commander's exit 1. + return Promise.reject(new Error('ENETUNREACH dns lookup failed')); + }) as typeof globalThis.fetch; + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_large' }, + { credentialsPath, fetchImpl, rawStdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'UNAVAILABLE', + exitCode: 10, + message: expect.stringContaining('ENETUNREACH'), + }); + }); + + it('streaming loop awaits rawStdout drain on each chunk (backpressure)', async () => { + // Pins that runCodeGet's streaming loop awaits each writeChunk + // before reading the next stream value. The fine-grained "exact + // pull count under backpressure" is pinned by Output.writeChunk's + // own unit test in output.test.ts (WHATWG streams have an + // implementation-defined chunk-lookahead window, so asserting + // exact pull counts here would couple this test to runtime + // internals). + const { credentialsPath } = makeCreds(); + type Resolver = () => void; + const writeOrder: string[] = []; + const pendingState: { resolve: Resolver | null } = { resolve: null }; + const fetchImpl = ((input: Parameters[0]) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('A')); + controller.enqueue(encoder.encode('B')); + controller.enqueue(encoder.encode('C')); + controller.close(); + }, + }); + return Promise.resolve( + new Response(stream, { status: 200, headers: { 'content-type': 'text/plain' } }), + ); + }) as typeof globalThis.fetch; + const rawStdout = (text: string): Promise => + new Promise(resolve => { + writeOrder.push(`enter:${text}`); + // Resolve previous, then queue this resolver — proves the + // loop didn't fire writeChunk(B) until A's drain settled. + const prev = pendingState.resolve; + pendingState.resolve = () => { + writeOrder.push(`drain:${text}`); + resolve(); + }; + if (prev) prev(); + }); + const runPromise = runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_large' }, + { credentialsPath, fetchImpl, rawStdout }, + ); + // Drive the chain: each enter must be preceded by the previous drain. + while (!writeOrder.includes('drain:C')) { + await new Promise(r => setImmediate(r)); + const next = pendingState.resolve; + if (next !== null) { + pendingState.resolve = null; + next(); + } + } + await runPromise; + // Exactly one enter per chunk, drains interleave with subsequent + // enters, never two enters in a row without the prior drain. + expect(writeOrder.filter(s => s.startsWith('enter:'))).toEqual([ + 'enter:A', + 'enter:B', + 'enter:C', + ]); + for (let i = 0; i < writeOrder.length - 1; i += 1) { + const cur = writeOrder[i]!; + const next = writeOrder[i + 1]!; + // No two `enter:` events back-to-back — the loop must drain in between. + if (cur.startsWith('enter:') && next.startsWith('enter:')) { + throw new Error(`back-to-back writeChunk without drain: ${cur} → ${next}`); + } + } + }); + + it('wraps a mid-stream read error as TransportError (exit 10)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = ((input: Parameters[0]) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + // Use `pull` (not synchronous `start` enqueue+error) so the + // first chunk is delivered to the consumer before the error + // fires on the next pull. With sync enqueue+error, Node's + // WHATWG stream tears down before the queued chunk is observable. + let pulls = 0; + const stream = new ReadableStream({ + pull(controller) { + pulls += 1; + if (pulls === 1) { + controller.enqueue(new TextEncoder().encode('first chunk OK\n')); + return; + } + controller.error(new Error('ECONNRESET')); + }, + }); + return Promise.resolve( + new Response(stream, { status: 200, headers: { 'content-type': 'text/plain' } }), + ); + }) as typeof globalThis.fetch; + const chunks: string[] = []; + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_large' }, + { + credentialsPath, + fetchImpl, + rawStdout: chunk => { + chunks.push(chunk); + }, + }, + ), + ).rejects.toMatchObject({ code: 'UNAVAILABLE', exitCode: 10 }); + // The first chunk that arrived before the reset is fine to have + // been written — agents that pipe to a file see a partial. The + // non-zero exit tells them not to trust it. + expect(chunks.join('')).toContain('first chunk OK'); + }); + + it('JSON mode does not follow a presigned URL — caller decides', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch(() => { + calls += 1; + return { body: TEST_CODE_PRESIGNED }; + }); + await runCodeGet( + { profile: 'default', output: 'json', debug: false, testId: 'test_large' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // Exactly one network call: the §6.3 fetch. No presigned dereference. + expect(calls).toBe(1); + }); + + it('CONFLICT envelope from /code maps to exit 6', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'Snapshot in flight; retry shortly.', + nextAction: 'retry shortly', + requestId: 'req_test', + details: { reason: 'snapshot_in_flight' }, + }, + }, + })); + await expect( + runCodeGet( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'CONFLICT', exitCode: 6 }); + }); + + it('translates a non-2xx presigned URL into an UNAVAILABLE ApiError', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = ((input: Parameters[0]) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + return Promise.resolve( + new Response('expired', { status: 403, headers: { 'content-type': 'text/plain' } }), + ); + }) as typeof globalThis.fetch; + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_large' }, + { credentialsPath, fetchImpl, rawStdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'UNAVAILABLE', exitCode: 10 }); + }); + + // ---------- --out path: file sink instead of stdout ---------- + // The the CLI validation spec §4 P4 contract: "the CLI streams the body + // to stdout (or `--out`) without buffering the whole thing in memory." + // These tests pin the file-sink branch end-to-end: text mode writes + // the source body, json mode writes the wire envelope, the presigned + // streaming path pipes through to disk preserving chunk boundaries, + // and validation errors land at the typed VALIDATION_ERROR envelope + // (exit 5). + + it('--out (text mode) writes the inline body to a file instead of stdout', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_CODE_INLINE })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-')); + const target = join(dir, 'inline.ts'); + let stdoutCalls = 0; + let rawStdoutCalls = 0; + await runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: target, + }, + { + credentialsPath, + fetchImpl, + stdout: () => { + stdoutCalls += 1; + }, + rawStdout: () => { + rawStdoutCalls += 1; + }, + }, + ); + // File holds the byte-exact source body. No JSON braces. + expect(readFileSync(target, 'utf-8')).toBe(TEST_CODE_INLINE.code); + // Critical: stdout/rawStdout MUST NOT receive any of the body. + // A regression where --out also wrote to stdout would corrupt the + // process pipe in `testsprite ... > thing.zip` style invocations + // and is the kind of silent-failure --out exists to avoid. + expect(stdoutCalls).toBe(0); + expect(rawStdoutCalls).toBe(0); + }); + + it('--out (json mode) writes the §6.3 envelope as a single JSON document', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_CODE_INLINE })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-')); + const target = join(dir, 'envelope.json'); + await runCodeGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + out: target, + }, + { credentialsPath, fetchImpl }, + ); + const onDisk = readFileSync(target, 'utf-8'); + expect(JSON.parse(onDisk)).toEqual(TEST_CODE_INLINE); + // Wire-shape consumers piping `--out file.json | jq` should find + // a trailing newline so jq's per-doc parser handles the file. + expect(onDisk.endsWith('\n')).toBe(true); + }); + + it('--out streams a presigned URL straight into the file', async () => { + const { credentialsPath } = makeCreds(); + const STREAMED_CHUNKS = [ + "import { test } from '@playwright/test';\n", + "test('huge', async ({ page }) => {\n", + ' /* very long file */\n', + '});\n', + ]; + const fetchImpl = ((input: Parameters[0]) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/')) { + return Promise.resolve( + new Response(JSON.stringify(TEST_CODE_PRESIGNED), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + } + const stream = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + for (const chunk of STREAMED_CHUNKS) controller.enqueue(encoder.encode(chunk)); + controller.close(); + }, + }); + return Promise.resolve( + new Response(stream, { status: 200, headers: { 'content-type': 'text/plain' } }), + ); + }) as typeof globalThis.fetch; + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-')); + const target = join(dir, 'large.ts'); + await runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_large', + out: target, + }, + { credentialsPath, fetchImpl }, + ); + expect(readFileSync(target, 'utf-8')).toBe(STREAMED_CHUNKS.join('')); + }); + + it('--out rejects an empty path with VALIDATION_ERROR (exit 5) before any network I/O', async () => { + const { credentialsPath } = makeCreds(); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: '' }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + // Failing fast matters: an early validation should not waste an + // API call. If this assertion regresses, the user pays a network + // round-trip + audit-log entry on every typo. + expect(fetchCalls).toBe(0); + }); + + it('--out rejects a directory-style path with VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = (() => Promise.resolve(new Response('{}'))) as typeof globalThis.fetch; + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: '/tmp/some-dir/', + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + // Regression: a parent dir that doesn't exist used to surface as exit 1 + // (TRANSPORT_ERROR) — `createWriteStream` opens lazily and ENOENT fires + // mid-write. Synchronous parent stat keeps every `--out` user-input + // shape on the same exit-5 contract as the rest of CLI v1 validation. + it('--out rejects a path under a missing parent dir with VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: `/tmp/_p4_no_such_dir_${process.pid}_${Date.now()}/out.py`, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchCalls).toBe(0); + }); + + it('--out rejects a path whose parent is a regular file with VALIDATION_ERROR (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + // Use this very test file as the "parent" — guaranteed to exist and + // guaranteed not to be a directory. No fs writes; validator stats it. + const here = new URL(import.meta.url).pathname; + const fetchImpl = (() => Promise.resolve(new Response('{}'))) as typeof globalThis.fetch; + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: `${here}/under-a-file.py`, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + +describe('runCodePut', () => { + function writeCodeFile(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-')); + const path = join(dir, 'updated.spec.ts'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + const SAMPLE_RESPONSE = { + testId: 'test_alpha', + codeVersion: 'v4', + updatedAt: '2026-05-15T10:00:00.000Z', + }; + + it('PUTs /tests/{id}/code with body + If-Match: + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('updated body'); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body: SAMPLE_RESPONSE }; + }); + + const res = await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + expect(captured).toHaveLength(1); + const sent = captured[0]!; + expect(sent.method).toBe('PUT'); + expect(sent.url).toContain('/api/cli/v1/tests/test_alpha/code'); + expect(sent.body).toEqual({ code: 'updated body' }); + expect(sent.headers.get('if-match')).toBe('v3'); + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-code-put-[0-9a-f-]{36}$/); + expect(sent.headers.get('content-type')).toBe('application/json'); + }); + + it('forwards --language in the body when set', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('print("hi")'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + language: 'python', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ code: 'print("hi")', language: 'python' }); + }); + + it('--force sends If-Match: *', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + let seenIfMatch: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenIfMatch = new Headers(init.headers as Record).get('if-match'); + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + force: true, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenIfMatch).toBe('*'); + }); + + it('rejects --force + --expected-version combination before sending', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + force: true, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'expected-version' }), + }); + expect(called).toBe(0); + }); + + it('rejects an invalid --language value before sending', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + language: 'ruby' as never, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'language' }), + }); + expect(called).toBe(0); + }); + + it('rejects missing --code-file before sending', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile: '', + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(called).toBe(0); + }); + + it('auto-fetches current codeVersion when neither --expected-version nor --force is set', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + const seen: { method: string; url: string; ifMatch: string | null }[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + seen.push({ + method, + url, + ifMatch: new Headers(init.headers as Record).get('if-match'), + }); + if (method === 'GET') { + return { + body: { testId: 'test_alpha', language: 'typescript', code: 'old', codeVersion: 'v7' }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + const errLines: string[] = []; + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLines.push(line), + }, + ); + expect(seen.map(s => s.method)).toEqual(['GET', 'PUT']); + expect(seen[1]!.ifMatch).toBe('v7'); + expect(errLines.some(l => l.includes('auto-fetched codeVersion=v7'))).toBe(true); + }); + + it('auto-fetch on legacy null codeVersion falls back to If-Match: * with stderr advisory', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + const seen: { method: string; ifMatch: string | null }[] = []; + const fetchImpl = makeFetch((_url, init) => { + const method = init.method ?? 'GET'; + seen.push({ + method, + ifMatch: new Headers(init.headers as Record).get('if-match'), + }); + if (method === 'GET') { + return { + body: { testId: 'test_alpha', language: 'typescript', code: 'old', codeVersion: null }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + const errLines: string[] = []; + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLines.push(line), + }, + ); + expect(seen[1]!.ifMatch).toBe('*'); + expect(errLines.some(l => l.includes('legacy row'))).toBe(true); + }); + + it('on 412 PRECONDITION_FAILED, prints a typed retry hint with the server codeVersion and re-throws', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + const fetchImpl = makeFetch(() => ({ + status: 412, + body: { + error: { + code: 'PRECONDITION_FAILED', + message: 'codeVersion mismatch', + nextAction: "Re-fetch with 'test get ' and retry with the new codeVersion.", + requestId: 'req_42', + details: { currentCodeVersion: 'v5' }, + }, + }, + })); + const errLines: string[] = []; + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLines.push(line), + }, + ), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED', exitCode: 6 }); + expect(errLines.some(l => l.includes('Server is at v5, you sent v3'))).toBe(true); + expect(errLines.some(l => l.includes('--expected-version v5'))).toBe(true); + }); + + it('respects a caller-supplied --idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + idempotencyKey: 'op_codeput_1', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenKey).toBe('op_codeput_1'); + }); + + it('renders text mode with one line per field', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const out: string[] = []; + await runCodePut( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const block = out.join('\n'); + expect(block).toContain('testId test_alpha'); + expect(block).toContain('codeVersion v4'); + expect(block).toContain('updatedAt 2026-05-15T10:00:00.000Z'); + }); + + // N1: dry-run advisory wording must differ from real-run advisory + it('N1 — dry-run auto-fetch advisory includes "[dry-run]" prefix', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + // dry-run fetch returns the canned /code shape via dry-run samples + const errLines: string[] = []; + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_alpha', + codeFile, + // No expectedVersion — triggers the auto-fetch advisory path + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ + body: { + testId: 'test_alpha', + language: 'typescript', + code: 'old', + codeVersion: 'v_sample', + }, + })), + stdout: () => undefined, + stderr: line => errLines.push(line), + }, + ); + // Dry-run advisory must include the dry-run marker + expect(errLines.some(l => l.includes('[dry-run]'))).toBe(true); + expect(errLines.some(l => l.includes('auto-fetched codeVersion='))).toBe(false); + }); + + it('N1 — real-run auto-fetch advisory does NOT include "[dry-run]" prefix', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('body'); + const fetchImpl = makeFetch((_url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') { + return { + body: { testId: 'test_alpha', language: 'typescript', code: 'old', codeVersion: 'v9' }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + const errLines: string[] = []; + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_alpha', + codeFile, + // No expectedVersion — triggers the auto-fetch advisory path + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errLines.push(line) }, + ); + // Real advisory contains codeVersion, NOT the [dry-run] marker + expect(errLines.some(l => l.includes('auto-fetched codeVersion=v9'))).toBe(true); + expect(errLines.some(l => l.includes('[dry-run]'))).toBe(false); + }); + + // Fix #2 — dogfood 2026-05-14: --dry-run-simulate-error PRECONDITION_FAILED + it('--dry-run --dry-run-simulate-error PRECONDITION_FAILED: exits 6 + emits retry hint', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('any code'); + const stderrLines: string[] = []; + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + codeFile, + expectedVersion: 'v3', + dryRunSimulateError: 'PRECONDITION_FAILED', + }, + { + credentialsPath, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + // Retry hint must include the server version and testId + expect(stderrLines.some(l => l.includes('Code conflict'))).toBe(true); + expect(stderrLines.some(l => l.includes('v99'))).toBe(true); + expect(stderrLines.some(l => l.includes('--expected-version'))).toBe(true); + }); + + it('without --dry-run, a real 412 from the server still routes through the normal retry-hint path', async () => { + // Sanity check for the unflagged real-412 path. Confirms the + // simulate code added in dogfood-2026-05-14 does not displace the + // existing server-driven 412 handler. + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code'); + const fetchImpl = makeFetch(() => ({ + status: 412, + body: { + error: { + code: 'PRECONDITION_FAILED', + message: 'etag mismatch', + nextAction: 'retry', + requestId: 'req_test', + details: { currentCodeVersion: 'v7' }, + }, + }, + })); + const stderrLines: string[] = []; + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_abc', + codeFile, + expectedVersion: 'v3', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + // Real 412 path: hint from server (currentCodeVersion=v7), not from simulate (which would inject v99) + expect(stderrLines.some(l => l.includes('Code conflict'))).toBe(true); + expect(stderrLines.some(l => l.includes('v7'))).toBe(true); + }); + + it('--dry-run-simulate-error WITHOUT --dry-run is ignored — the simulate guard does not fire and the real fetch is reached', async () => { + // codex-review P2 (2026-05-28): the previous test name promised + // "without --dry-run" coverage but didn't actually pass + // dryRunSimulateError. This test does pass it (with dryRun: false) + // and asserts the simulate block did NOT synthesize the canned v99 + // payload — the request reached fetchImpl and got the server's v7 + // back instead. + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code'); + let fetchCallCount = 0; + const fetchImpl = makeFetch(() => { + fetchCallCount += 1; + return { + status: 412, + body: { + error: { + code: 'PRECONDITION_FAILED', + message: 'etag mismatch', + nextAction: 'retry', + requestId: 'req_real_server', + details: { currentCodeVersion: 'v7' }, + }, + }, + }; + }); + const stderrLines: string[] = []; + await expect( + runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, // <-- key: NOT dry-run + dryRunSimulateError: 'PRECONDITION_FAILED', // <-- but simulate IS set + testId: 'test_abc', + codeFile, + expectedVersion: 'v3', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ), + ).rejects.toMatchObject({ + code: 'PRECONDITION_FAILED', + requestId: 'req_real_server', // server-side requestId, not the simulate 'req_dry-run-simulate' + }); + // The real fetch must have been reached — simulate did not short-circuit. + expect(fetchCallCount).toBe(1); + // And the hint must come from the server's v7, not simulate's canned v99. + expect(stderrLines.some(l => l.includes('v7'))).toBe(true); + expect(stderrLines.some(l => l.includes('v99'))).toBe(false); + }); +}); + +describe('runSteps', () => { + it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [STEP_PASSED, STEP_FAILED, STEP_PENDING], nextToken: null } }; + }); + const out: string[] = []; + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen[0]).toContain('/tests/test_fe/steps'); + expect(seen[0]).toContain('pageSize=25'); + expect(page.items).toHaveLength(3); + expect(JSON.parse(out[0]!).items).toHaveLength(3); + }); + + it('text mode renders index/action/status/description and shared run metadata', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [STEP_PASSED, STEP_FAILED, STEP_PENDING], nextToken: null }, + })); + const out: string[] = []; + await runSteps( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('INDEX'); + expect(block).toContain('ACTION'); + expect(block).toContain('STATUS'); + expect(block).toContain('DESCRIPTION'); + expect(block).toContain('Click the cart icon'); + expect(block).toContain('Click the submit button'); + expect(block).toContain('Expect order confirmation heading'); + // null status renders as an em-dash so a glance reads it as + // "no verdict yet" rather than a broken cell. + expect(block).toContain('—'); + // Shared run metadata renders once at the bottom. + expect(block).toContain('runId: run_abc'); + expect(block).toContain('codeVersion: v3'); + }); + + it('text mode reads "No steps." on an empty list', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); + const out: string[] = []; + await runSteps( + { profile: 'default', output: 'text', debug: false, testId: 'test_empty' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toBe('No steps.'); + }); + + it('--max-items caps total returned across pages', async () => { + const { credentialsPath } = makeCreds(); + let calls = 0; + const fetchImpl = makeFetch(() => { + calls += 1; + return { + body: { + items: [ + { ...STEP_PASSED, stepIndex: calls * 2 - 1 }, + { ...STEP_PASSED, stepIndex: calls * 2 }, + ], + nextToken: calls < 3 ? `cursor-${calls}` : null, + }, + }; + }); + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + maxItems: 3, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(page.items).toHaveLength(3); + }); + + it('text mode flags the synthetic terminal assertion row with `*` and a "(synthetic assertion failure)" description tail', async () => { + // M2.1 piece-4 follow-up: when the test failed at the assertion + // layer (no real step in error), the backend appends one synthetic + // terminal row with `action: "assertion"`, `outcomeContributesToFailure: + // true`, and null screenshot/htmlSnapshot URLs. The text renderer + // marks it with `* ` so a 50-step list highlights the contributing + // row, and appends "(synthetic assertion failure)" so an operator + // doesn't wonder why the action is `assertion` when the test code + // never had one. + const { credentialsPath } = makeCreds(); + const synthetic: CliTestStep = { + testId: 'test_fe', + stepIndex: 7, + action: 'assertion', + description: 'The hosted build did not surface the member detail page.', + status: 'failed', + screenshotUrl: null, + htmlSnapshotUrl: null, + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-09T18:00:00.000Z', + outcomeContributesToFailure: true, + }; + const fetchImpl = makeFetch(() => ({ + body: { items: [STEP_PASSED, STEP_PENDING, synthetic], nextToken: null }, + })); + const out: string[] = []; + await runSteps( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + pageSize: 25, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toMatch(/\*\s+7\s+assertion/); + expect(block).toContain('(synthetic assertion failure)'); + // Real rows without `outcomeContributesToFailure: true` keep the + // empty 2-char marker so column alignment doesn't drift. + expect(block).not.toMatch(/\*\s+4\s+click/); + }); + + it('NOT_FOUND envelope from /steps maps to exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Resource not found.', + nextAction: 'check id', + requestId: 'req_test', + details: { resource: 'test', id: 'test_missing' }, + }, + }, + })); + await expect( + runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_missing' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND', exitCode: 4 }); + }); + + // --------------------------------------------------------------------------- + // --run-id filter (dogfood round-4 2026-05-17 fix) + // --------------------------------------------------------------------------- + + // ------- --run-id path: uses GET /runs/{runId}?includeSteps=true ------- + // FE Portal step rows don't reliably carry per-run `runIdIfAvailable`, so + // the old client-side filter always returned empty. The new path fetches + // from the authoritative run-scoped endpoint and maps RunStepDto→CliTestStep. + + /** Minimal RunResponse fixture for the run-scoped steps tests. */ + const RUN_WITH_STEPS = { + runId: 'run_scoped', + testId: 'test_fe', + projectId: 'project_alice', + userId: 'u1', + status: 'passed' as const, + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v5', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 }, + steps: [ + { + stepIndex: '0001', + type: 'action', + action: 'click .btn', + status: 'passed', + description: 'Click the submit button', + error: null, + screenshotUrl: 'https://s3.example.com/snap/01.png', + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:05.000Z', + }, + { + stepIndex: '0002', + type: 'assertion', + action: 'assert heading', + status: 'passed', + description: 'Confirm heading visible', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:10.000Z', + }, + ], + }; + + it('--run-id calls GET /runs/{runId}?includeSteps=true (not the cumulative steps endpoint)', async () => { + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: RUN_WITH_STEPS }; + }); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // Must call the run-scoped endpoint, NOT the cumulative /tests/{id}/steps + expect(seenUrls.some(u => u.includes('/runs/run_scoped'))).toBe(true); + expect(seenUrls.some(u => u.includes('includeSteps=true'))).toBe(true); + expect(seenUrls.every(u => !u.includes('/tests/test_fe/steps'))).toBe(true); + expect(page.items).toHaveLength(2); + }); + + it('--run-id maps RunStepDto fields to CliTestStep correctly', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RUN_WITH_STEPS })); + const out: string[] = []; + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const first = page.items[0]!; + // stepIndex is parsed from the zero-padded string '0001' + expect(first.stepIndex).toBe(1); + // runIdIfAvailable comes from RunResponse.runId + expect(first.runIdIfAvailable).toBe('run_scoped'); + // testId comes from RunResponse.testId + expect(first.testId).toBe('test_fe'); + // codeVersion comes from RunResponse.codeVersion + expect(first.codeVersion).toBe('v5'); + // timestamps come from RunStepDto.createdAt + expect(first.capturedAt).toBe('2026-06-01T10:00:05.000Z'); + expect(first.updatedAt).toBe('2026-06-01T10:00:05.000Z'); + // outcomeContributesToFailure: null when run passed (failedStepIndex is null) + expect(first.outcomeContributesToFailure).toBeNull(); + // JSON output should be parseable with 2 items + const printed = JSON.parse(out[0]!) as { items: unknown[] }; + expect(printed.items).toHaveLength(2); + }); + + it('--run-id: outcomeContributesToFailure=true on the failedStepIndex step', async () => { + const { credentialsPath } = makeCreds(); + const runWithFailure = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + }; + const fetchImpl = makeFetch(() => ({ body: runWithFailure })); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // step at index 2 (stepIndex='0002') should be flagged true; index 1 is a + // KNOWN non-contributor → false (not null) once failedStepIndex is known + // (CliTestStep contract: null = unclassified, false = classified non-contrib). + const step1 = page.items.find(s => s.stepIndex === 1)!; + const step2 = page.items.find(s => s.stepIndex === 2)!; + expect(step1.outcomeContributesToFailure).toBe(false); + expect(step2.outcomeContributesToFailure).toBe(true); + }); + + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { + const { credentialsPath } = makeCreds(); + // The run-scoped endpoint returns a run whose testId differs from the + // argument (same tenant, wrong test). Must NOT leak its steps — + // restores the implicit scoping of the old /tests/{testId}/steps path. + const otherTestRun = { ...RUN_WITH_STEPS, testId: 'test_OTHER' }; + const fetchImpl = makeFetch(() => ({ body: otherTestRun })); + await expect( + runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + runId: 'run_scoped', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ exitCode: 4 }); + }); + + it('--run-id: empty steps → JSON mode prints empty array, exits 0', async () => { + const { credentialsPath } = makeCreds(); + const runNoSteps = { ...RUN_WITH_STEPS, steps: [] }; + const fetchImpl = makeFetch(() => ({ body: runNoSteps })); + const out: string[] = []; + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + expect(page.items).toHaveLength(0); + const printed = JSON.parse(out[0]!) as { items: unknown[] }; + expect(printed.items).toHaveLength(0); + }); + + it('--run-id: empty steps → text mode emits advisory to stderr pointing at artifact get', async () => { + const { credentialsPath } = makeCreds(); + const runNoSteps = { ...RUN_WITH_STEPS, steps: [] }; + const fetchImpl = makeFetch(() => ({ body: runNoSteps })); + const stderrLines: string[] = []; + const out: string[] = []; + const page = await runSteps( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + runId: 'run_scoped', + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + expect(page.items).toHaveLength(0); + expect(out).toHaveLength(0); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('artifact get'))).toBe( + true, + ); + }); + + it('--run-id: 404 propagates as ApiError (exit 4, NOT_FOUND)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { code: 'NOT_FOUND', message: 'Run not found' }, + })); + await expect( + runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + runId: 'run_unknown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND' }); + }); + + it('no --run-id: multi-run response emits advisory to stderr when steps span >1 runId', async () => { + const { credentialsPath } = makeCreds(); + const stepA: CliTestStep = { ...STEP_PASSED, stepIndex: 1, runIdIfAvailable: 'run_A' }; + const stepB: CliTestStep = { ...STEP_PASSED, stepIndex: 2, runIdIfAvailable: 'run_B' }; + const fetchImpl = makeFetch(() => ({ + body: { items: [stepA, stepB], nextToken: null }, + })); + const stderrLines: string[] = []; + await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + // Advisory must mention the count and the --run-id flag + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--run-id'))).toBe(true); + }); + + it('no --run-id: single-run response produces no advisory', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [STEP_PASSED, STEP_FAILED], nextToken: null }, + })); + const stderrLines: string[] = []; + await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + }); + + it('--run-id: pagination flags (--page-size / --starting-token / --max-items) are ignored — run endpoint returns all steps', async () => { + // The run-scoped endpoint returns all steps in a single response. + // Pagination flags are only meaningful on the cumulative /tests/{id}/steps path. + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: RUN_WITH_STEPS }; + }); + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + runId: 'run_scoped', + pageSize: 5, // ignored for run-scoped path + startingToken: 'tok', // ignored for run-scoped path + maxItems: 1, // ignored for run-scoped path + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // All 2 steps are returned (maxItems=1 is NOT applied on the run-scoped path) + expect(page.items).toHaveLength(2); + // The run-scoped URL should not carry pagination query params + const runUrl = seenUrls.find(u => u.includes('/runs/run_scoped'))!; + expect(runUrl).toBeDefined(); + expect(runUrl).not.toMatch(/pageSize|cursor|startingToken/); + }); + + it('test steps subcommand exposes --run-id flag', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const steps = test.commands.find(c => c.name() === 'steps')!; + const flagNames = steps.options.map(o => o.long); + expect(flagNames).toContain('--run-id'); + }); +}); + +describe('runResult', () => { + it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: RESULT_FAILED }; + }); + const out: string[] = []; + const got = await runResult( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen[0]).toContain('/tests/test_fe/result'); + expect(got).toEqual(RESULT_FAILED); + expect(JSON.parse(out[0]!)).toEqual(RESULT_FAILED); + }); + + it('text mode for a failed run highlights failureKind + failedStepIndex up top', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_FAILED })); + const out: string[] = []; + await runResult( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + const lines = block.split('\n'); + // failureKind / failedStepIndex come BEFORE timestamps. "Highlight" + // here is positional — agents and humans reading top-down see the + // failure shape first. + const kindLine = lines.findIndex(l => l.startsWith('failureKind')); + const indexLine = lines.findIndex(l => l.startsWith('failedStepIndex')); + const startedLine = lines.findIndex(l => l.startsWith('startedAt')); + expect(kindLine).toBeGreaterThanOrEqual(0); + expect(indexLine).toBeGreaterThanOrEqual(0); + expect(kindLine).toBeLessThan(startedLine); + expect(indexLine).toBeLessThan(startedLine); + expect(block).toContain('failureKind: assertion'); + expect(block).toContain('failedStepIndex: 5'); + expect(block).toContain('summary: passed=4 failed=1 skipped=0'); + expect(block).toContain('failureAnalysisUrl: '); + }); + + it('text mode for a passed run skips failureKind/failedStepIndex/failureAnalysisUrl', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_PASSED })); + const out: string[] = []; + await runResult( + { profile: 'default', output: 'text', debug: false, testId: 'test_passed' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('status: passed'); + expect(block).not.toContain('failureKind'); + expect(block).not.toContain('failedStepIndex'); + expect(block).not.toContain('failureAnalysisUrl'); + expect(block).toContain('summary: passed=8 failed=0 skipped=0'); + }); + + it('CONFLICT envelope from /result maps to exit 6 (snapshot in flight)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'Snapshot in flight; retry shortly.', + nextAction: + 'Snapshot in flight; retry in a few seconds. The CLI re-fetches against a single `snapshotId` so partial reads are safe.', + requestId: 'req_test', + details: { reason: 'snapshot_in_flight' }, + }, + }, + })); + // Sleep helper makes CONFLICT retry-backoff instant for the test. + await expect( + runResult( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + }, + ), + ).rejects.toMatchObject({ code: 'CONFLICT', exitCode: 6 }); + }); + + it('default does NOT add ?includeAnalysis (byte-identical to pre-M2.1)', async () => { + // M2.1 piece 3 contract: omitting `--include-analysis` must + // produce a request whose URL has no `includeAnalysis` query + // param. That keeps existing automation byte-identical to pre- + // M2.1 — no surprise shape changes for callers that didn't opt in. + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: RESULT_FAILED }; + }); + await runResult( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seen[0]).not.toContain('includeAnalysis'); + }); + + it('--include-analysis sends ?includeAnalysis=true and renders the block in text mode', async () => { + const RESULT_WITH_ANALYSIS = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'Submit button is disabled because the form is invalid.', + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/components/CheckoutForm.tsx:412', + rationale: 'Disabled state originates from `isFormValid()`.', + }, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: RESULT_WITH_ANALYSIS }; + }); + const out: string[] = []; + const got = await runResult( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen[0]).toContain('includeAnalysis=true'); + expect(got.analysis).toEqual(RESULT_WITH_ANALYSIS.analysis); + const block = out.join('\n'); + expect(block).toContain('rootCause: Submit button is disabled'); + expect(block).toContain( + 'recommendedFix: kind=code reference=src/components/CheckoutForm.tsx:412', + ); + }); + + it('--include-analysis renders "(none)" when recommendedFixTarget is null (visibility policy)', async () => { + // M2.1 piece 3: a failing test with no LLM-filled fix target now + // arrives with `analysis.recommendedFixTarget: null`. The text + // renderer surfaces this with a pointed user-facing string rather + // than echoing the raw null shape. + const RESULT_WITH_NULL_FIX = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'timeout exceeded', + recommendedFixTarget: null, + failureKind: 'timeout' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_WITH_NULL_FIX })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('rootCause: timeout exceeded'); + expect(block).toContain('recommendedFix: — (analysis pipeline did not propose one)'); + }); + + // ---------- L141 — truncation indicator tests ---------- + + it('L141: JSON mode adds rootCauseHypothesisTruncated=true when hypothesis ends with ellipsis', async () => { + // The truncation heuristic requires length ≥ 500; pad to simulate a backend-truncated field. + const TRUNCATED_HYPOTHESIS = + 'The submit button is disabled because the credit-card field validation failed and isFormValid() returned false, preventing the user from proceeding to the next step. ' + + 'The root cause is that the React context value for the credit-card validation state was not properly propagated through the component tree, causing the submit button ' + + 'to remain in a disabled state even after the user had filled in all required fields. The fix requires updating the context provider to correctly forward the validation state…'; + const RESULT_WITH_TRUNCATED_ANALYSIS = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: TRUNCATED_HYPOTHESIS, + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/Checkout.tsx:42', + rationale: 'isFormValid predicate is the gatekeeper.', + }, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_WITH_TRUNCATED_ANALYSIS })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const parsed = JSON.parse(out[0]!); + expect(parsed.analysis.rootCauseHypothesisTruncated).toBe(true); + expect(parsed.analysis.recommendedFixRationaleTruncated).toBeUndefined(); + }); + + it('L141: JSON mode adds recommendedFixRationaleTruncated=true when rationale ends with ellipsis', async () => { + // The truncation heuristic requires length ≥ 500; pad to simulate a backend-truncated field. + const TRUNCATED_RATIONALE = + 'The isFormValid() predicate at line 412 checks all required fields; the credit-card field validation state is propagated via React context and reaches the submit button disabled state. ' + + 'To fix this, update the CardFieldContext provider at src/contexts/CardFieldContext.tsx to ensure the isValid flag is forwarded correctly to the useFormValidation hook consumed ' + + 'by the SubmitButton component. Once the validation state propagates correctly the button will re-enable when all fields pass their validators…'; + const RESULT_WITH_TRUNCATED_RATIONALE = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'Submit button is disabled.', + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/Checkout.tsx:412', + rationale: TRUNCATED_RATIONALE, + }, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_WITH_TRUNCATED_RATIONALE })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const parsed = JSON.parse(out[0]!); + expect(parsed.analysis.rootCauseHypothesisTruncated).toBeUndefined(); + expect(parsed.analysis.recommendedFixRationaleTruncated).toBe(true); + }); + + it('L141: JSON mode adds both truncation indicators when both fields are truncated', async () => { + // The truncation heuristic requires length ≥ 500; pad both fields. + const longHypothesis = + 'Hypothesis that was cut short by the server — the submit button remained disabled because the React context did not propagate the validation state correctly through the component tree. ' + + 'Several intermediate components consumed the context but did not forward the updated value, leaving the leaf SubmitButton with a stale disabled=true derived from the initial render. ' + + 'Traced to CardFieldContext provider at src/contexts/CardFieldContext.tsx line 112 where the isValid flag is not memoized and is recomputed incorrectly on every re-render cycle…'; + const longRationale = + 'Rationale that was also cut short by the server — to fix this update the CardFieldContext provider to ensure isValid is forwarded. The useFormValidation hook consumed by SubmitButton ' + + 'reads from the context on every render cycle; once the provider emits the correct value the button will re-enable when all required fields pass their individual validation rules. ' + + 'The key change is in CardFieldContext.tsx: wrap the derived isValid computation in useMemo so it only recomputes when the field values change, not on every parent re-render…'; + const RESULT_BOTH_TRUNCATED = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: longHypothesis, + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/Form.tsx:5', + rationale: longRationale, + }, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_BOTH_TRUNCATED })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const parsed = JSON.parse(out[0]!); + expect(parsed.analysis.rootCauseHypothesisTruncated).toBe(true); + expect(parsed.analysis.recommendedFixRationaleTruncated).toBe(true); + }); + + it('L141: JSON mode omits truncation indicators when neither field is truncated', async () => { + const RESULT_NO_TRUNCATION = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'Submit button is disabled.', + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/Checkout.tsx:412', + rationale: 'isFormValid() is the gatekeeper.', + }, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_NO_TRUNCATION })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const parsed = JSON.parse(out[0]!); + expect(parsed.analysis.rootCauseHypothesisTruncated).toBeUndefined(); + expect(parsed.analysis.recommendedFixRationaleTruncated).toBeUndefined(); + }); + + it('L141: text mode does NOT add truncation indicator markers (renderer unchanged)', async () => { + const RESULT_WITH_TRUNCATED = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'Truncated hypothesis…', + recommendedFixTarget: null, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_WITH_TRUNCATED })); + const out: string[] = []; + await runResult( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + // Text mode shows the raw (possibly truncated) value; no indicator suffix appended + expect(block).toContain('Truncated hypothesis…'); + // Specifically: no "rootCauseHypothesisTruncated" key/line added by text renderer + expect(block).not.toContain('rootCauseHypothesisTruncated'); + expect(block).not.toContain('recommendedFixRationaleTruncated'); + }); + + it('L141: truncation indicators are NOT present in returned result object (only in printed JSON)', async () => { + const RESULT_WITH_TRUNCATED = { + ...RESULT_FAILED, + analysis: { + rootCauseHypothesis: 'Truncated hypothesis…', + recommendedFixTarget: null, + failureKind: 'assertion' as const, + snapshotId: RESULT_FAILED.snapshotId, + }, + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RESULT_WITH_TRUNCATED })); + const result = await runResult( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + includeAnalysis: true, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ); + // The returned object is the raw backend response — indicators only in the printed JSON + expect(result.analysis?.rootCauseHypothesisTruncated).toBeUndefined(); + }); +}); + +// ---------- D1 — targetUrlSource advisory (null targetUrl + unresolved source) ---------- + +describe('runResult — D1 targetUrlSource', () => { + it('text mode: null targetUrl + targetUrlSource=unresolved emits advisory to stderr without crashing', async () => { + const { credentialsPath } = makeCreds(); + const unresolved: CliLatestResult = { + ...RESULT_FAILED, + targetUrl: null, + targetUrlSource: 'unresolved', + }; + const fetchImpl = makeFetch(() => ({ body: unresolved })); + const out: string[] = []; + const stderrLines: string[] = []; + await runResult( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + // Must not print literal "null" for targetUrl + const block = out.join('\n'); + expect(block).not.toContain('targetUrl: null'); + expect(block).not.toContain('targetUrl: null'); + // Advisory must appear on stderr + const advisory = stderrLines.find(l => l.includes('[advisory]') && l.includes('target URL')); + expect(advisory).toBeDefined(); + expect(advisory).toContain('unresolved'); + }); + + it('text mode: null targetUrl + targetUrlSource=null also emits advisory', async () => { + const { credentialsPath } = makeCreds(); + const nullSource: CliLatestResult = { + ...RESULT_FAILED, + targetUrl: null, + targetUrlSource: null, + }; + const fetchImpl = makeFetch(() => ({ body: nullSource })); + const stderrLines: string[] = []; + await runResult( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + const advisory = stderrLines.find(l => l.includes('[advisory]') && l.includes('target URL')); + expect(advisory).toBeDefined(); + }); + + it('text mode: normal non-null targetUrl emits no advisory', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...RESULT_FAILED, targetUrl: 'https://example.com', targetUrlSource: 'run' }, + })); + const stderrLines: string[] = []; + await runResult( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('target URL'))).toBe(false); + }); + + it('JSON mode: passes targetUrl (null) + targetUrlSource through unchanged without advisory', async () => { + const { credentialsPath } = makeCreds(); + const unresolved: CliLatestResult = { + ...RESULT_FAILED, + targetUrl: null, + targetUrlSource: 'unresolved', + }; + const fetchImpl = makeFetch(() => ({ body: unresolved })); + const out: string[] = []; + const stderrLines: string[] = []; + const got = await runResult( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + // JSON output must include both fields as-is + const parsed = JSON.parse(out.join('\n')) as CliLatestResult; + expect(parsed.targetUrl).toBeNull(); + expect(parsed.targetUrlSource).toBe('unresolved'); + // Returned value also carries the field + expect(got.targetUrlSource).toBe('unresolved'); + // No advisory in JSON mode + expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + }); +}); + +// ---------- §5.2 / M2.1 piece 3 — runFailureSummary ---------- + +describe('runFailureSummary', () => { + const SUMMARY = { + testId: 'test_fe', + status: 'failed' as const, + failureKind: 'assertion' as const, + snapshotId: 'snap_summary_xyz', + rootCauseHypothesis: 'Submit button is disabled because the credit-card field is empty.', + recommendedFixTarget: { + kind: 'code' as const, + reference: 'src/components/CheckoutForm.tsx:412', + rationale: 'Disabled state originates from `isFormValid()` predicate.', + }, + }; + + it('JSON mode prints the §5.2 wire envelope verbatim', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: SUMMARY }; + }); + const out: string[] = []; + const got = await runFailureSummary( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(seen[0]).toContain('/tests/test_fe/failure/summary'); + expect(got).toEqual(SUMMARY); + expect(JSON.parse(out[0]!)).toEqual(SUMMARY); + }); + + it('text mode renders the one-screen triage card', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SUMMARY })); + const out: string[] = []; + await runFailureSummary( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('testId: test_fe'); + expect(block).toContain('status: failed'); + expect(block).toContain('failureKind: assertion'); + expect(block).toContain('rootCauseHypothesis: Submit button is disabled'); + expect(block).toContain('recommendedFixTarget: kind=code'); + }); + + it('text mode renders "(none)" when recommendedFixTarget is null (visibility policy)', async () => { + const NULL_FIX_SUMMARY = { ...SUMMARY, recommendedFixTarget: null }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: NULL_FIX_SUMMARY })); + const out: string[] = []; + await runFailureSummary( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toContain( + 'recommendedFixTarget: — (analysis pipeline did not propose one)', + ); + }); + + it('NOT_FOUND with details.reason="no_failing_run" propagates as exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Test has no failing run.', + nextAction: + 'Test has no failing run. Use `testsprite test result ` to inspect the latest result.', + requestId: 'req_test', + details: { resource: 'test', id: 'test_passing', reason: 'no_failing_run' }, + }, + }, + })); + await expect( + runFailureSummary( + { profile: 'default', output: 'json', debug: false, testId: 'test_passing' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'NOT_FOUND', + exitCode: 4, + details: { reason: 'no_failing_run' }, + }); + }); +}); + +// ---------- §6.7 runFailureGet ---------- + +const FAILED_STEPS: CliTestStep[] = [ + { + testId: 'test_failed', + stepIndex: 4, + action: 'click', + description: 'Click the cart icon', + status: 'passed', + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/04.html?sig', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: '2026-05-05T12:34:55.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', + }, + { + testId: 'test_failed', + stepIndex: 5, + action: 'click', + description: 'Click the submit button', + status: 'failed', + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/05.html?sig', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: '2026-05-05T12:34:56.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', + }, + { + testId: 'test_failed', + stepIndex: 6, + action: 'expect', + description: 'Expect order confirmation heading', + status: null, + screenshotUrl: null, + htmlSnapshotUrl: 'https://signed.example.com/06.html?sig', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-05T12:34:58.000Z', + }, +]; + +function makeFailureContext(overrides: Partial = {}): CliFailureContext { + const result: CliLatestResult = { + testId: 'test_failed', + status: 'failed', + startedAt: '2026-05-05T12:34:00.000Z', + finishedAt: '2026-05-05T12:34:58.000Z', + videoUrl: 'https://video.example.com/run_abc.mp4?sig', + failureAnalysisUrl: null, + snapshotId: 'snap_2026_05_07_b2f9a1c8', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + targetUrl: 'https://staging.example.com/checkout', + failedStepIndex: 5, + failureKind: 'assertion', + summary: { passed: 4, failed: 1, skipped: 0 }, + ...overrides.result, + }; + return { + snapshotId: 'snap_2026_05_07_b2f9a1c8', + testId: 'test_failed', + projectId: 'project_alice', + result, + steps: FAILED_STEPS, + code: { + testId: 'test_failed', + language: 'typescript', + framework: 'playwright', + code: "import { test } from '@playwright/test';\n", + codeVersion: 'v3', + etag: null, + }, + failure: { + rootCauseHypothesis: + 'Submit button is disabled. Underlying error: AssertionError: expected visible.', + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: 'fill card field' }, + evidence: [ + { + kind: 'snapshot', + stepIndex: 4, + url: 'https://signed.example.com/ev/04.html?sig', + summary: "Step 4 'Click the cart icon' passed (captured at ...).", + }, + { + kind: 'snapshot', + stepIndex: 5, + url: 'https://signed.example.com/ev/05.html?sig', + summary: "Step 5 'Click the submit button' failed. Error: AssertionError", + }, + ], + }, + ...overrides, + }; +} + +describe('runFailureGet', () => { + it('JSON mode (no --out) prints the wire envelope verbatim to stdout', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext(); + const fetchImpl = makeFetch(url => { + expect(url).toContain('/tests/test_failed/failure'); + return { body: ctx }; + }); + const out: string[] = []; + const result = await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(result.context).toEqual(ctx); + expect(result.bundle).toBeUndefined(); + // Single JSON envelope on stdout — agents pipe directly into jq / + // their LLM consumer. + expect(out).toHaveLength(1); + expect(JSON.parse(out[0]!)).toEqual(ctx); + }); + + it('text mode (no --out) prints the human summary block', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext(); + const fetchImpl = makeFetch(() => ({ body: ctx })); + const out: string[] = []; + await runFailureGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_failed', + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('status: failed'); + expect(block).toContain('failureKind: assertion'); + expect(block).toContain('failedStepIndex: 5'); + expect(block).toContain('snapshotId: snap_2026_05_07_b2f9a1c8'); + expect(block).toContain('runId: run_abc'); + expect(block).toContain('rootCause: Submit button is disabled.'); + expect(block).toContain('recommendedFix: kind=unknown'); + expect(block).toContain('evidence: 2 items (snapshot×2)'); + expect(block).toContain('videoUrl: https://video.example.com'); + }); + + it('--out writes the §7 layout and prints a one-line confirmation', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext(); + const fetchImpl = makeFetch(url => { + // Presigned URL fetches return a 200 with a tiny body — covers + // the snapshot HTML / video / evidence-html paths. + if (url.startsWith('https://signed.example.com')) { + return { body: 'snapshot' }; + } + if (url.startsWith('https://video.example.com')) { + return { body: 'fake-video-bytes' }; + } + // /tests/.../failure + return { body: ctx }; + }); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-bundle-')); + const out: string[] = []; + const result = await runFailureGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(result.bundle).toBeDefined(); + expect(result.bundle!.dir).toBe(dir); + // §7.1: meta.json is the identity card. Its presence is the + // atomic-completion signal — agents read it first. + const metaPath = join(dir, 'meta.json'); + expect(existsSync(metaPath)).toBe(true); + const meta = JSON.parse(readFileSync(metaPath, 'utf8')) as Record; + expect(meta.snapshotId).toBe(ctx.snapshotId); + expect(meta.testId).toBe(ctx.testId); + expect(meta.failedStepIndex).toBe(5); + expect(meta.schemaVersion).toBe('cli-v1'); + // Top-level files exist. + expect(existsSync(join(dir, 'result.json'))).toBe(true); + expect(existsSync(join(dir, 'failure.json'))).toBe(true); + // language=typescript → code.ts (matches the test's framework). + expect(existsSync(join(dir, 'code.ts'))).toBe(true); + // videoUrl was non-null → video.mp4 written. + expect(existsSync(join(dir, 'video.mp4'))).toBe(true); + // Per-step snapshot files for ±1 around step 5. + expect(existsSync(join(dir, 'steps', '04-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '05-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '06-snapshot.html'))).toBe(true); + // .partial NOT present on success. + expect(existsSync(join(dir, '.partial'))).toBe(false); + // Confirmation line on stdout. + expect(out.join('\n')).toContain(`Bundle written to ${dir}`); + }); + + it('--out --failed-only narrows steps to the failed step ± 1 (drops outside-window)', async () => { + const { credentialsPath } = makeCreds(); + // Wider step list — failed at 5, neighbors {3,4,6,7}. With --failed-only + // the bundle keeps only 4/5/6. + const wideSteps: CliTestStep[] = [3, 4, 5, 6, 7].map(i => ({ + testId: 'test_failed', + stepIndex: i, + action: 'click', + description: `step ${i}`, + status: i === 5 ? ('failed' as const) : ('passed' as const), + screenshotUrl: null, + htmlSnapshotUrl: `https://signed.example.com/${String(i).padStart(2, '0')}.html`, + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-05T12:34:56.000Z', + })); + const ctx = makeFailureContext({ + steps: wideSteps, + failure: { + rootCauseHypothesis: null, + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: null }, + evidence: wideSteps.map(s => ({ + kind: 'snapshot' as const, + stepIndex: s.stepIndex, + url: `https://signed.example.com/ev/${String(s.stepIndex).padStart(2, '0')}.html`, + summary: `step ${s.stepIndex} summary`, + })), + }, + }); + const fetchImpl = makeFetch(url => { + if (url.includes('/failure')) return { body: ctx }; + return { body: 'x' }; + }); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-failedonly-')); + await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: true, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // Failed step ±1 stays; 3 and 7 dropped. + expect(existsSync(join(dir, 'steps', '04-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '05-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '06-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '03-snapshot.html'))).toBe(false); + expect(existsSync(join(dir, 'steps', '07-snapshot.html'))).toBe(false); + }); + + it('refuses a forged bundle with mismatched snapshotId (agent-safety trap)', async () => { + // §3 invariant: bundle.snapshotId === result.snapshotId, byte-for-byte. + // A forged response where they disagree must NOT be written to disk — + // an agent reading the meta would see one snapshotId, the result file + // would have another, and the bundle would be corrupt. + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext({ + result: { + ...makeFailureContext().result, + snapshotId: 'snap_DIFFERENT', // mismatch + }, + }); + const fetchImpl = makeFetch(() => ({ body: ctx })); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-forged-')); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + // Nothing visible was written; meta.json was never renamed in. + expect(existsSync(join(dir, 'meta.json'))).toBe(false); + }); + + it('refuses a forged bundle when steps disagree on runIdIfAvailable', async () => { + const { credentialsPath } = makeCreds(); + const [first, second] = FAILED_STEPS as [CliTestStep, CliTestStep, CliTestStep]; + const stepsMixed: CliTestStep[] = [ + { ...first, runIdIfAvailable: 'run_a' }, + { ...second, runIdIfAvailable: 'run_b' }, + ]; + const ctx = makeFailureContext({ steps: stepsMixed }); + const fetchImpl = makeFetch(() => ({ body: ctx })); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('writes /.partial and exits non-zero when a presigned download fails', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext(); + const fetchImpl = makeFetch(url => { + if (url.includes('/failure')) return { body: ctx }; + // Force a 403 on the presigned URL (URL expired). Per §6.3 the + // CLI must NOT retry 4xx — it surfaces UNAVAILABLE / exit 10. + return { status: 403, body: { error: 'expired' } }; + }); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-partial-')); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'UNAVAILABLE' }); + // .partial marker exists; meta.json does NOT (bundle incomplete). + expect(existsSync(join(dir, '.partial'))).toBe(true); + expect(existsSync(join(dir, 'meta.json'))).toBe(false); + const partial = JSON.parse(readFileSync(join(dir, '.partial'), 'utf8')) as Record< + string, + unknown + >; + expect(partial.snapshotId).toBe(ctx.snapshotId); + expect(typeof partial.error).toBe('string'); + }); + + it('404 NOT_FOUND propagates to ApiError (exit 4) and writes nothing', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Test has no failing run.', + nextAction: 'Test has no failing run. Use `testsprite test result `.', + requestId: 'req_test', + details: { resource: 'test', id: 'test_passing', reason: 'no_failing_run' }, + }, + }, + })); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-404-')); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_passing', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND', exitCode: 4 }); + // Nothing written on 404 — pre-existing is unmodified per §9. + expect(existsSync(join(dir, 'meta.json'))).toBe(false); + expect(existsSync(join(dir, '.partial'))).toBe(false); + }); + + it('CONFLICT envelope from /failure maps to exit 6 (snapshot in flight)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 409, + body: { + error: { + code: 'CONFLICT', + message: 'Snapshot in flight; retry shortly.', + nextAction: + 'Snapshot in flight; retry in a few seconds. The CLI re-fetches against a single `snapshotId` so partial reads are safe.', + requestId: 'req_test', + details: { reason: 'snapshot_in_flight' }, + }, + }, + })); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'CONFLICT', exitCode: 6 }); + }); + + it('in-place rewrite removes stale top-level files (codex P1)', async () => { + // Pin: a second `--out` run against an existing dir replaces stale + // top-level files (e.g. video.mp4 from a previous bundle whose + // result had a videoUrl, when the new bundle has none) and removes + // the old meta.json BEFORE renaming new files in. Without this, + // an agent reading the dir mid-rewrite could see a fresh meta + // pointing at the new snapshot but still see the prior run's + // video.mp4. See codex review on lib/bundle.ts:375 for the bug. + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-rewrite-')); + + // First run: bundle has a video. + const firstCtx = makeFailureContext(); + const firstFetch = makeFetch(url => { + if (url.includes('/failure')) return { body: firstCtx }; + return { body: 'fake-bytes' }; + }); + await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl: firstFetch, stdout: () => undefined }, + ); + expect(existsSync(join(dir, 'video.mp4'))).toBe(true); + expect(existsSync(join(dir, 'meta.json'))).toBe(true); + + // Second run: bundle has NO video (e.g. backend cleared the + // videoUrl, or the snapshot was minted before the recording + // landed). Stale video.mp4 from the first run must NOT linger. + const secondCtx = makeFailureContext({ + snapshotId: 'snap_2026_05_07_NEW', + result: { ...makeFailureContext().result, snapshotId: 'snap_2026_05_07_NEW', videoUrl: null }, + }); + const secondFetch = makeFetch(url => { + if (url.includes('/failure')) return { body: secondCtx }; + return { body: 'fake-bytes' }; + }); + await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl: secondFetch, stdout: () => undefined }, + ); + // Stale video.mp4 swept; new meta in place. + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + const meta = JSON.parse(readFileSync(join(dir, 'meta.json'), 'utf8')) as Record< + string, + unknown + >; + expect(meta.snapshotId).toBe('snap_2026_05_07_NEW'); + }); + + it('downloads log/network/console evidence URLs into per-evidence files (codex P2)', async () => { + // Pin: when the backend emits non-snapshot/screenshot evidence + // (kind: log|network|console), the CLI dereferences each URL into + // a per-evidence file under steps/, and the -evidence.json + // sidecar references the local path instead of the (soon-expired) + // presigned URL. + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext({ + failure: { + rootCauseHypothesis: null, + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: null }, + evidence: [ + // Snapshot evidence is the existing path — sanity check it + // still goes into -snapshot.html. + { + kind: 'snapshot' as const, + stepIndex: 5, + url: 'https://signed.example.com/ev/05.html?sig', + summary: 'snapshot summary', + }, + { + kind: 'console' as const, + stepIndex: 5, + url: 'https://signed.example.com/ev/05-console.json?sig', + summary: 'console summary', + }, + { + kind: 'log' as const, + stepIndex: 5, + url: 'https://signed.example.com/ev/05-log.txt?sig', + summary: 'log summary', + }, + ], + }, + }); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/failure')) return { body: ctx }; + seenUrls.push(url); + return { body: 'evidence-body' }; + }); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-evidence-')); + await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // Per codex round-1 P2 (re-run): every `evidence[].url` must be + // dereferenced. The snapshot evidence URL here + // (`/ev/05.html?sig`) differs from `step.htmlSnapshotUrl` + // (`/05.html?sig`), so it cannot be deduped against the + // already-downloaded step file — it must be streamed to its own + // sidecar file. Console + log get their own files too. + expect(existsSync(join(dir, 'steps', '05-snapshot-0.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '05-console-1.json'))).toBe(true); + expect(existsSync(join(dir, 'steps', '05-log-2.txt'))).toBe(true); + // The sidecar JSON references the local path, not the URL. + const sidecar = JSON.parse( + readFileSync(join(dir, 'steps', '05-evidence.json'), 'utf8'), + ) as Array<{ kind: string; path?: string; url?: string }>; + expect(sidecar).toHaveLength(3); + expect(sidecar[0]!.kind).toBe('snapshot'); + expect(sidecar[0]!.path).toBe('steps/05-snapshot-0.html'); + expect(sidecar[0]!.url).toBeUndefined(); + expect(sidecar[1]!.kind).toBe('console'); + expect(sidecar[1]!.path).toBe('steps/05-console-1.json'); + expect(sidecar[1]!.url).toBeUndefined(); + expect(sidecar[2]!.kind).toBe('log'); + expect(sidecar[2]!.path).toBe('steps/05-log-2.txt'); + // All three evidence URLs were actually fetched. + expect(seenUrls).toEqual( + expect.arrayContaining([ + expect.stringContaining('05.html?sig'), + expect.stringContaining('05-console.json'), + expect.stringContaining('05-log.txt'), + ]), + ); + }); + + it('reuses the step snapshot/screenshot file when an evidence URL matches it (codex P2 dedupe)', async () => { + const { credentialsPath } = makeCreds(); + const baseCtx = makeFailureContext(); + // Find the step whose htmlSnapshotUrl is set (the 05 step) and + // reuse its URL in the evidence list. Path-match should kick in + // and NO new file is downloaded for the snapshot evidence. + const step05 = baseCtx.steps.find(s => s.stepIndex === 5)!; + const ctx = makeFailureContext({ + failure: { + rootCauseHypothesis: null, + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: null }, + evidence: [ + { + kind: 'snapshot' as const, + stepIndex: 5, + url: step05.htmlSnapshotUrl!, + summary: 'snapshot dedupe', + }, + ], + }, + }); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/failure')) return { body: ctx }; + seenUrls.push(url); + return { body: 'evidence-body' }; + }); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-evidence-dedupe-')); + await runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + out: dir, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // The snapshot evidence remapped to the existing step snapshot + // file, not a new sidecar file. + expect(existsSync(join(dir, 'steps', '05-snapshot.html'))).toBe(true); + expect(existsSync(join(dir, 'steps', '05-snapshot-0.html'))).toBe(false); + const sidecar = JSON.parse( + readFileSync(join(dir, 'steps', '05-evidence.json'), 'utf8'), + ) as Array<{ kind: string; path?: string; url?: string }>; + expect(sidecar).toHaveLength(1); + expect(sidecar[0]!.kind).toBe('snapshot'); + expect(sidecar[0]!.path).toBe('steps/05-snapshot.html'); + expect(sidecar[0]!.url).toBeUndefined(); + // Snapshot URL was fetched exactly once (for the step file), not + // a second time for the evidence sidecar. + const snapshotFetches = seenUrls.filter(u => u === step05.htmlSnapshotUrl); + expect(snapshotFetches).toHaveLength(1); + }); + + it('refuses a bundle where result.codeVersion and code.codeVersion disagree (codex P3)', async () => { + const { credentialsPath } = makeCreds(); + const ctx = makeFailureContext({ + result: { ...makeFailureContext().result, codeVersion: 'v3' }, + code: { ...makeFailureContext().code, codeVersion: 'v4' }, + }); + const fetchImpl = makeFetch(() => ({ body: ctx })); + await expect( + runFailureGet( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_failed', + failedOnly: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ reason: 'code_version_mismatch' }), + }); + }); +}); + +describe('runCreate', () => { + function writeCodeFile(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-p2-')); + const path = join(dir, 'test.spec.ts'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + const SAMPLE_RESPONSE = { + testId: 'test_new', + type: 'frontend' as const, + codeVersion: 'v1', + createdAt: '2026-05-13T10:00:00.000Z', + }; + + it('POSTs /tests with the canonical body + Idempotency-Key + returns the response', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("smoke", async () => {});\n'); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + captured.push({ + url, + method, + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + // Fix 4: best-effort duplicate-name check issues a GET /tests?projectId=... before POST. + // Return empty listing for that pre-flight call; SAMPLE_RESPONSE for the actual POST. + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESPONSE }; + }); + + const out: string[] = []; + const res = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'sign-up happy', + description: 'Cover the happy-path signup flow.', + priority: 'p1', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + // Fix 4 adds a pre-flight GET /tests listing before the POST; filter to POST only. + const postCalls = captured.filter(c => c.method === 'POST'); + expect(postCalls).toHaveLength(1); + const sent = postCalls[0]!; + expect(sent.method).toBe('POST'); + expect(sent.url).toContain('/api/cli/v1/tests'); + expect(sent.body).toEqual({ + projectId: 'project_alice', + type: 'frontend', + name: 'sign-up happy', + description: 'Cover the happy-path signup flow.', + priority: 'p1', + code: 'test("smoke", async () => {});\n', + }); + // A UUIDv4 idempotency-key prefixed `cli-create-` is minted per call. + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-create-[0-9a-f-]{36}$/); + expect(sent.headers.get('content-type')).toBe('application/json'); + expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); + }); + + it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + idempotencyKey: 'op_42', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenKey).toBe('op_42'); + }); + + it('rejects > 350 KB code body locally with PAYLOAD_TOO_LARGE (no fetch issued)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('a'.repeat(350 * 1024 + 1)); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'PAYLOAD_TOO_LARGE', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('a missing --code-file surfaces VALIDATION_ERROR before any fetch', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile: '/tmp/this-file-does-not-exist-xyz123.spec.ts', + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'code-file' }), + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('missing --project surfaces VALIDATION_ERROR (input gate before fs)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + // @ts-expect-error — exercising the runtime gate + projectId: undefined, + type: 'frontend', + name: 'n', + codeFile: '/tmp/whatever.txt', + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('AUTH_FORBIDDEN from the server (read-only key) propagates as exit 3', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + const fetchImpl = makeFetch(() => ({ + status: 403, + body: { + error: { + code: 'AUTH_FORBIDDEN', + message: 'API key does not grant the required scope.', + nextAction: + 'This API key does not have the required scope. Ask your account owner to extend it.', + requestId: 'req_test', + details: { requiredScope: 'write:tests' }, + }, + }, + })); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'AUTH_FORBIDDEN', exitCode: 3 }); + }); + + it('IDEMPOTENCY_BODY_MISMATCH from the server is NOT retried (caller bug, exit 6)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('a different body'); + let postCalls = 0; + const fetchImpl = makeFetch((_url, init) => { + const method = init.method ?? 'GET'; + // Fix 4: best-effort GET /tests listing fires before the POST; return empty listing. + if (method === 'GET') return { status: 200, body: { items: [] } }; + postCalls += 1; + return { + status: 409, + body: { + error: { + code: 'IDEMPOTENCY_BODY_MISMATCH', + message: 'Idempotency-Key was reused with a different request body.', + nextAction: 'Generate a new Idempotency-Key for the changed request.', + requestId: 'req_test', + details: { reason: 'body-mismatch', storedAt: '2026-05-13T09:00:00.000Z' }, + }, + }, + }; + }); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + idempotencyKey: 'op_42', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'IDEMPOTENCY_BODY_MISMATCH', exitCode: 6 }); + // IDEMPOTENCY_BODY_MISMATCH is NOT retried — only 1 POST should have been made. + expect(postCalls).toBe(1); + }); + + it('--dry-run returns the canned sample (no fetchImpl needed; credentialsPath optional)', async () => { + // Omitting both fetchImpl and credentialsPath proves dry-run skips + // credential I/O AND substitutes the in-process dry-run fetch + // (`makeHttpClient` swaps in `createDryRunFetch()` per `client-factory.ts`). + const codeFile = writeCodeFile('any code'); + const res = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(res).toMatchObject({ + testId: expect.any(String), + type: 'frontend', + codeVersion: 'v1', + createdAt: expect.any(String), + }); + }); + + it('--dry-run skips fs entirely so a missing --code-file is not a blocker', async () => { + // Codex round-1 fix: dry-run must work with dummy inputs (no real + // disk dependency) to match the M2 P6 contract — operators shake + // out the wire shape without a real test file on hand. + const res = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile: '/tmp/this-file-does-not-exist-dry-run-xyz.spec.ts', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(res).toMatchObject({ testId: expect.any(String), codeVersion: 'v1' }); + }); + + it('emits the generated idempotency-key to stderr under --verbose or --output json (Fix 1)', async () => { + // Fix 1 (Fix 1 review fix): idempotency-key trailers are suppressed ONLY + // in text mode without --verbose/--debug. In JSON output mode, the trailer + // must still appear on stderr — stderr never pollutes JSON stdout, and + // suppressing it was a silent regression for scripts running --output json. + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + + // Text mode, no flags: key must NOT appear (noise reduction for humans). + const stderrLinesTextDefault: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLinesTextDefault.push(line), + }, + ); + expect(stderrLinesTextDefault.some(l => l.startsWith('idempotency-key:'))).toBe(false); + + // JSON output mode: key MUST appear on stderr (Fix 1 regression guard). + const stderrLinesJson: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLinesJson.push(line), + }, + ); + expect(stderrLinesJson.some(l => /^idempotency-key: cli-create-[0-9a-f-]{36}$/.test(l))).toBe( + true, + ); + + // Text mode + --verbose: key must appear. + const stderrLinesVerbose: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + verbose: true, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLinesVerbose.push(line), + }, + ); + expect( + stderrLinesVerbose.some(l => /^idempotency-key: cli-create-[0-9a-f-]{36}$/.test(l)), + ).toBe(true); + }); + + it('does NOT echo a caller-supplied --idempotency-key (no second-channel disclosure)', async () => { + // Counter-test for the round-1 surface: when the caller pinned the + // key themselves, they already know it — don't re-print it. Keeps + // the stderr line a "FYI for auto-generated keys" signal that the + // operator can scan for. + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + idempotencyKey: 'op_42', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(false); + }); + + it('text mode prints one line per response field (no shell-incompatible chars)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('testId test_new'); + expect(block).toContain('type frontend'); + expect(block).toContain('codeVersion v1'); + expect(block).toContain('createdAt 2026-05-13T10:00:00.000Z'); + }); + + // C1 — --target-url advisory for backend tests (fires only when --run is also set) + it('[C1] emits advisory on stderr when --type backend + --target-url + --run are all set', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("be", async () => {});'); + // Two responses: POST /tests (create) then POST /tests/{id}/runs (trigger) + const fetchImpl = makeFetch(url => { + if (url.includes('/runs')) { + return { + body: { + runId: 'run_c1', + status: 'queued', + enqueuedAt: '2026-06-04T00:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://staging.example.com', + }, + }; + } + return { body: { ...SAMPLE_RESPONSE, testId: 'test_c1', type: 'backend' } }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'be test', + codeFile, + targetUrl: 'https://staging.example.com', + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect( + stderrLines.some( + l => l.includes('[advisory]') && l.includes('--target-url') && l.includes('backend'), + ), + ).toBe(true); + }); + + it('[C1] does NOT emit the backend advisory for frontend tests with --target-url', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("fe", async () => {});'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + // No advisory expected for frontend + expect( + stderrLines.some( + l => l.includes('[advisory]') && l.includes('--target-url') && l.includes('backend'), + ), + ).toBe(false); + }); + + it('[C1] does NOT emit the backend advisory for backend tests without --target-url', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("be", async () => {});'); + const fetchImpl = makeFetch(() => ({ body: { ...SAMPLE_RESPONSE, type: 'backend' } })); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'be test', + codeFile, + // no targetUrl + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--target-url'))).toBe( + false, + ); + }); + + it('[C1 Fix 4] does NOT emit advisory when --type backend + --target-url but --run is absent', async () => { + // targetUrl only matters at run time; a bare `test create --type backend + // --target-url` without --run should NOT emit the advisory (it would be + // confusing noise since the test isn't being executed). + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("be", async () => {});'); + const fetchImpl = makeFetch(() => ({ body: { ...SAMPLE_RESPONSE, type: 'backend' } })); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'be test', + codeFile, + targetUrl: 'https://staging.example.com', + // --run not set → advisory must be suppressed + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--target-url'))).toBe( + false, + ); + }); + + it('[C1 Fix 4] emits advisory when --type backend + --target-url + --run are all set', async () => { + // When --run is also passed the test will actually execute and the advisory + // is meaningful. We verify the guard passes in this combined case. + // We use --wait: false so the test doesn't need to mock a full run poll. + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("be", async () => {});'); + // Two responses: POST /tests (create), POST /tests/{id}/runs (trigger) + let callCount = 0; + const fetchImpl = makeFetch(url => { + callCount += 1; + if (url.includes('/runs')) { + return { + body: { + runId: 'run_c1_fix4', + status: 'queued', + enqueuedAt: '2026-06-04T00:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://staging.example.com', + }, + }; + } + return { body: { ...SAMPLE_RESPONSE, testId: 'test_c1_fix4', type: 'backend' } }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'be test run', + codeFile, + targetUrl: 'https://staging.example.com', + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(callCount).toBeGreaterThan(0); + expect( + stderrLines.some( + l => l.includes('[advisory]') && l.includes('--target-url') && l.includes('backend'), + ), + ).toBe(true); + }); + + // Fix 4 — B3: duplicate-name advisory + it('Fix 4 — emits advisory on stderr when a test with the same name exists, but still proceeds', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// test code'); + let postCalled = false; + const existingTest = { + id: 'test_existing_abc', + projectId: 'project_alice', + name: 'Sign-up happy', + type: 'frontend', + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + // listing response with a same-name test + return { body: { items: [existingTest], nextToken: null } }; + } + if ((init.method ?? 'GET') === 'POST') { + postCalled = true; + return { body: SAMPLE_RESPONSE }; + } + return { body: SAMPLE_RESPONSE }; + }); + + const stderrLines: string[] = []; + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'sign-up happy', // case-insensitive match + codeFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + // Create still proceeds + expect(result.testId).toBe('test_new'); + expect(postCalled).toBe(true); + // Advisory on stderr mentioning the existing testId + const advisoryLine = stderrLines.find( + l => l.includes('[advisory]') && l.includes('test_existing_abc'), + ); + expect(advisoryLine).toBeDefined(); + expect(advisoryLine).toContain('test update'); + }); + + it('Fix 4 — swallows listing error and still proceeds with create', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// test code'); + let listCalled = false; + let postCalled = false; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + listCalled = true; + // Simulate a network or permission error (will be thrown via status 403) + return { + status: 403, + body: { + error: { + code: 'AUTH_FORBIDDEN', + message: 'Forbidden', + nextAction: '', + requestId: 'r1', + details: {}, + }, + }, + }; + } + if ((init.method ?? 'GET') === 'POST') { + postCalled = true; + return { body: SAMPLE_RESPONSE }; + } + return { body: SAMPLE_RESPONSE }; + }); + + const stderrLines: string[] = []; + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'another test', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + // Listing was attempted + expect(listCalled).toBe(true); + // But create still proceeded (error was swallowed) + expect(postCalled).toBe(true); + expect(result.testId).toBe('test_new'); + // No advisory emitted (no match found / error swallowed) + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('already exists'))).toBe( + false, + ); + }); +}); + +// --------------------------------------------------------------------------- +// [B-E2E-03] Fix 3 regression — whitespace-only --name must exit 5 (trim guard) +// --------------------------------------------------------------------------- + +describe('[B-E2E-03] runCreate: whitespace-only --name → VALIDATION_ERROR exit 5', () => { + function writeCodeFile(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-fix3-name-')); + const path = join(dir, 'test.spec.ts'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + it('single space --name → exit 5 (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// code'); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_abc', + type: 'frontend', + name: ' ', + codeFile, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('all-whitespace --name (multiple spaces) → exit 5 (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// code'); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_abc', + type: 'frontend', + name: ' ', + codeFile, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('tab-only --name → exit 5 (no network call)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// code'); + const fetchImpl = vi.fn(); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_abc', + type: 'frontend', + name: '\t\t', + codeFile, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('non-blank --name is accepted (control: no regression)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// code'); + const fetchImpl = makeFetch((url, init) => { + // Return no-match for advisory GET, and success response for POST + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { + body: { + testId: 'test_fix3_ctrl', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-06-09T00:00:00.000Z', + }, + }; + }); + // Should NOT throw — name has real content + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_abc', + type: 'frontend', + name: 'My real test name', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + // If we get here without throwing, the test passed + }); +}); + +// --------------------------------------------------------------------------- +// M4 piece-2 — BE dependency authoring: --produces/--needs/--category +// --------------------------------------------------------------------------- + +describe('runCreate — M4 BE dependency authoring flags', () => { + function writeCodeFile(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-m4-dep-')); + const path = join(dir, 'test.spec.ts'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + const BE_SAMPLE_RESPONSE = { + testId: 'test_be_dep_01', + type: 'backend' as const, + codeVersion: 'v1', + createdAt: '2026-06-09T00:00:00.000Z', + }; + + function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, + ): typeof globalThis.fetch { + return (async (input: Parameters[0], init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + } + + function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', + ): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-m4-creds-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; + } + + it('threads produces[] → body.produces when --type backend', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// producer code'); + type Captured = { body: Record }; + const captured: Captured[] = []; + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + captured.push({ body: JSON.parse(init.body as string) }); + return { body: BE_SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'create user', + codeFile, + produces: ['user_id', 'session_token'], + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ); + const postBody = captured.find(c => 'produces' in c.body)?.body; + expect(postBody).toBeDefined(); + expect(postBody!.produces).toEqual(['user_id', 'session_token']); + // wire field is `produces` (maps to captures on server) + expect('consumes' in postBody!).toBe(false); + expect('category' in postBody!).toBe(false); + }); + + it('threads needs[] → body.consumes (wire field) when --type backend', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// consumer code'); + type Captured = { body: Record }; + const captured: Captured[] = []; + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + captured.push({ body: JSON.parse(init.body as string) }); + return { body: BE_SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'read user', + codeFile, + needs: ['user_id'], + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ); + const postBody = captured.find(c => 'consumes' in c.body)?.body; + expect(postBody).toBeDefined(); + // CLI flag is --needs but wire field must be `consumes` + expect(postBody!.consumes).toEqual(['user_id']); + expect('produces' in postBody!).toBe(false); + }); + + it('threads category → body.category when --type backend', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// teardown code'); + type Captured = { body: Record }; + const captured: Captured[] = []; + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + captured.push({ body: JSON.parse(init.body as string) }); + return { body: BE_SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'cleanup session', + codeFile, + category: 'teardown', + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ); + const postBody = captured.find(c => 'category' in c.body)?.body; + expect(postBody).toBeDefined(); + expect(postBody!.category).toBe('teardown'); + }); + + it('threads all three dep fields when all are set', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// all-deps code'); + type Captured = { body: Record }; + const captured: Captured[] = []; + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + captured.push({ body: JSON.parse(init.body as string) }); + return { body: BE_SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'full dep test', + codeFile, + produces: ['order_id'], + needs: ['user_id', 'session_token'], + category: 'teardown', + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ); + const postBody = captured.find(c => 'produces' in c.body)?.body; + expect(postBody!.produces).toEqual(['order_id']); + expect(postBody!.consumes).toEqual(['user_id', 'session_token']); + expect(postBody!.category).toBe('teardown'); + }); + + it('does NOT include produces/consumes/category on the wire when arrays are empty', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// plain be code'); + type Captured = { body: Record }; + const captured: Captured[] = []; + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + captured.push({ body: JSON.parse(init.body as string) }); + return { body: BE_SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'plain test', + codeFile, + produces: [], // empty → omitted + needs: [], // empty → omitted + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ); + const postBody = captured[captured.length - 1]?.body; + expect('produces' in (postBody ?? {})).toBe(false); + expect('consumes' in (postBody ?? {})).toBe(false); + }); + + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --produces', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + produces: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + needs: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --category', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + category: 'teardown', + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('[plan-from guard] --plan-from + --produces → exit 5 (dep flags are BE-only; plan-steps are FE)', async () => { + const test = createTestCommand(); + disableExits(test); + // The guard fires at the top of the --plan-from branch, before the plan file + // is read — so a non-existent path still surfaces the dep-flag rejection first. + await expect( + test.parseAsync( + ['create', '--plan-from', '/nonexistent-plan.json', '--produces', 'user_id'], + { from: 'user' }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('[FE guard] does NOT throw for --type backend with dep flags (no false-positive)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// be code'); + const fetch = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { body: { items: [], nextToken: null } }; + return { body: BE_SAMPLE_RESPONSE }; + }); + // Should resolve without error + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'producer', + codeFile, + produces: ['x'], + category: 'teardown', + }, + { credentialsPath, fetchImpl: fetch, stdout: () => undefined, stderr: () => undefined }, + ), + ).resolves.toMatchObject({ testId: BE_SAMPLE_RESPONSE.testId }); + }); +}); + +describe('runPlanPut', () => { + function writeStepsFile(payload: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-p6-')); + const path = join(dir, 'refined-plan.json'); + writeFileSync(path, JSON.stringify(payload), 'utf8'); + return path; + } + + const STEPS = [ + { type: 'action', description: 'navigate to /login' }, + { type: 'action', description: 'fill email = a@b.c' }, + { type: 'action', description: 'click submit without password' }, + { type: 'assertion', description: "error toast 'invalid password' appears" }, + ]; + + const SAMPLE_RESPONSE = { + testId: 'test_alpha', + planStepsHash: 'sha256:abc123', + stepCount: 4, + updatedAt: '2026-05-14T10:00:00.000Z', + }; + + it('PUTs /tests/{id}/plan-steps with { planSteps } body + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body: SAMPLE_RESPONSE }; + }); + + const res = await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + expect(captured).toHaveLength(1); + const sent = captured[0]!; + expect(sent.method).toBe('PUT'); + expect(sent.url).toContain('/api/cli/v1/tests/test_alpha/plan-steps'); + expect(sent.body).toEqual({ planSteps: STEPS }); + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-plan-put-[0-9a-f-]{36}$/); + // No If-Match-Step-Count header by default — FE is last-writer-wins. + expect(sent.headers.get('if-match-step-count')).toBeNull(); + expect(sent.headers.get('content-type')).toBe('application/json'); + }); + + // Dogfood 2026-05-18 — sibling of the --plan-from BOM regression. `--steps` + // hits a separate JSON read path; lock both. + it('accepts a steps file with a UTF-8 BOM (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p6-bom-')); + const path = join(dir, 'refined-plan.json'); + writeFileSync(path, '' + JSON.stringify({ planSteps: STEPS }), 'utf8'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile: path, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ planSteps: STEPS }); + }); + + it('accepts a bare planSteps[] array (without the wrapping object)', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile(STEPS); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ planSteps: STEPS }); + }); + + it('sets If-Match-Step-Count when --expected-step-count is passed', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + let seen: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seen = new Headers(init.headers as Record).get('if-match-step-count'); + return { body: SAMPLE_RESPONSE }; + }); + await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + expectedStepCount: 4, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seen).toBe('4'); + }); + + it('rejects an invalid --expected-step-count before sending', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + expectedStepCount: -1, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'expected-step-count' }), + }); + expect(called).toBe(0); + }); + + it('rejects a missing steps file with VALIDATION_ERROR', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile: '/tmp/does-not-exist-piece6.json', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects an empty planSteps array with a field pointer', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: [] }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps' }), + }); + }); + + it('rejects a step with an invalid type', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ + planSteps: [{ type: 'observe', description: 'see thing' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps[0].type' }), + }); + }); + + it('rejects oversize plan-step body (>200 steps) pre-flight', async () => { + const { credentialsPath } = makeCreds(); + const planSteps = Array.from({ length: 201 }, (_, i) => ({ + type: 'action' as const, + description: `step ${i}`, + })); + const stepsFile = writeStepsFile({ planSteps }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps' }), + }); + }); + + it('propagates a server 400 BE-rejection envelope unchanged (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + const fetchImpl = makeFetch(() => ({ + status: 400, + body: { + error: { + code: 'VALIDATION_ERROR', + message: 'Backend test plan-steps cannot be updated via the CLI.', + nextAction: + "Use 'testsprite test code put --code-file ' to update backend test code directly.", + requestId: 'req_be_reject', + details: { field: 'type', reason: 'backend not supported' }, + }, + }, + })); + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_be_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + nextAction: expect.stringContaining('test code put'), + }); + }); + + it('respects a caller-supplied --idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + stepsFile, + idempotencyKey: 'op_plan_put_77', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenKey).toBe('op_plan_put_77'); + }); + + it('renders text mode with one line per field', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const out: string[] = []; + await runPlanPut( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_alpha', + stepsFile, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const block = out.join('\n'); + expect(block).toContain('testId test_alpha'); + expect(block).toContain('planStepsHash sha256:abc123'); + expect(block).toContain('stepCount 4'); + expect(block).toContain('updatedAt 2026-05-14T10:00:00.000Z'); + }); + + // Fix #1 — dogfood 2026-05-15: dry-run stepCount echoes actual input count + it('--dry-run: stepCount echoes the actual number of steps in the --steps file', async () => { + const { credentialsPath } = makeCreds(); + const twoSteps = [ + { type: 'action', description: 'navigate to /login' }, + { type: 'assertion', description: 'page title is Login' }, + ]; + const stepsFile = writeStepsFile({ planSteps: twoSteps }); + const res = await runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + stepsFile, + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + // Input had 2 steps; dry-run used to return the canned stepCount=3 + expect(res.stepCount).toBe(2); + }); + + // Fix #2 — dogfood 2026-05-14: --dry-run-simulate-error PRECONDITION_FAILED + it('--dry-run --dry-run-simulate-error PRECONDITION_FAILED: exits 6 + emits retry hint', async () => { + const { credentialsPath } = makeCreds(); + const stepsFile = writeStepsFile({ planSteps: STEPS }); + const stderrLines: string[] = []; + await expect( + runPlanPut( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + stepsFile, + dryRunSimulateError: 'PRECONDITION_FAILED', + }, + { + credentialsPath, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + // Retry hint must be present on stderr + expect(stderrLines.some(l => l.includes('Plan-steps conflict'))).toBe(true); + expect(stderrLines.some(l => l.includes('--expected-step-count'))).toBe(true); + }); +}); + +describe('runUpdate', () => { + const SAMPLE_RESPONSE = { + testId: 'test_alpha', + updatedFields: ['name', 'description'], + updatedAt: '2026-05-14T10:00:00.000Z', + }; + + it('PUTs /tests/{id} with the body fields the caller set + idempotency header', async () => { + const { credentialsPath } = makeCreds(); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body: SAMPLE_RESPONSE }; + }); + + const out: string[] = []; + const res = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'renamed test', + description: 'updated description', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + expect(captured).toHaveLength(1); + const sent = captured[0]!; + expect(sent.method).toBe('PUT'); + expect(sent.url).toContain('/api/cli/v1/tests/test_alpha'); + expect(sent.body).toEqual({ + name: 'renamed test', + description: 'updated description', + }); + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-update-[0-9a-f-]{36}$/); + expect(sent.headers.get('content-type')).toBe('application/json'); + expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); + }); + + it('sends only the fields the caller set — omits undefined ones from the body', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + priority: 'p2', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ priority: 'p2' }); + }); + + it('respects a caller-supplied --idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'whatever', + idempotencyKey: 'op_99', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenKey).toBe('op_99'); + }); + + it('suppresses the minted idempotency-key in text mode without --verbose/--debug (Fix 1)', async () => { + // Text mode + no flags → key suppressed (not noise in default human output). + // JSON mode and --verbose/--debug always emit it on stderr (never pollutes + // the JSON stdout stream; Fix 1 regression guard). + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + + // Text mode default: key must NOT appear + const errLinesText: string[] = []; + await runUpdate( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_alpha', + name: 'n', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLinesText.push(line), + }, + ); + expect(errLinesText.some(l => l.startsWith('idempotency-key:'))).toBe(false); + + // JSON mode: key MUST appear on stderr (Fix 1 — was silently suppressed) + const errLinesJson: string[] = []; + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'n', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLinesJson.push(line), + }, + ); + expect(errLinesJson.some(l => l.match(/^idempotency-key: cli-update-[0-9a-f-]{36}$/))).toBe( + true, + ); + + // Text mode + --verbose: key must appear + const errLinesVerbose: string[] = []; + await runUpdate( + { + profile: 'default', + output: 'text', + debug: false, + verbose: true, + testId: 'test_alpha', + name: 'n', + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLinesVerbose.push(line), + }, + ); + expect(errLinesVerbose.some(l => l.match(/^idempotency-key: cli-update-[0-9a-f-]{36}$/))).toBe( + true, + ); + }); + + it('rejects no-op invocations (none of name/description/priority set) before sending', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'fields' }), + }); + expect(called).toBe(0); + }); + + it('rejects an invalid --priority value before sending', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + priority: 'urgent' as never, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'priority' }), + }); + expect(called).toBe(0); + }); + + it('renders text mode with one line per updated field', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const out: string[] = []; + await runUpdate( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_alpha', + name: 'renamed', + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('testId test_alpha'); + expect(block).toContain('updatedFields name, description'); + expect(block).toContain('updatedAt 2026-05-14T10:00:00.000Z'); + }); + + // Regression guard: --dry-run on `test update` resolves the canned sample + // and exits 0. Pins the PUT verb (backend route is `@Put('/:testId')`) so + // a future "fix" that flips both sides to PATCH doesn't silently break + // `test update` against the deployed backend. + it('--dry-run: exits 0 and returns canned updateTest sample (status 200, not 500)', async () => { + const { credentialsPath } = makeCreds(); + const res = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + name: 'dry-run rename', + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + // Must resolve to the canned sample, not throw + expect(res.testId).toBe('test_dryrun_update_2026'); + expect(Array.isArray(res.updatedFields)).toBe(true); + expect(res.updatedAt).toBeTruthy(); + }); + + // Fix #1 — dogfood 2026-05-15: dry-run should echo user's actual flags + it('--dry-run: updatedFields echoes only the flags the caller supplied (not canned defaults)', async () => { + const { credentialsPath } = makeCreds(); + const res = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + name: 'new name only', + // description intentionally omitted + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + // With only --name, updatedFields must be ['name'], not ['name','description'] + expect(res.updatedFields).toEqual(['name']); + }); + + it('--dry-run: updatedFields echoes name + description when both flags are passed', async () => { + const { credentialsPath } = makeCreds(); + const res = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_abc', + name: 'n', + description: 'd', + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + expect(res.updatedFields).toEqual(expect.arrayContaining(['name', 'description'])); + expect(res.updatedFields).toHaveLength(2); + }); +}); + +describe('runDelete', () => { + const SAMPLE_RESPONSE = { + testId: 'test_alpha', + deletedAt: '2026-05-14T10:00:00.000Z', + }; + + it('refuses without --confirm (exit-5 VALIDATION_ERROR), does not fetch', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runDelete( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + confirm: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'confirm' }), + }); + expect(called).toBe(0); + }); + + it('DELETEs /tests/{id} with --confirm and emits the idempotency-key (json mode)', async () => { + const { credentialsPath } = makeCreds(); + type Captured = { url: string; method: string; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + headers: new Headers(init.headers as Record), + }); + return { body: SAMPLE_RESPONSE }; + }); + const out: string[] = []; + const errLines: string[] = []; + const res = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + confirm: true, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errLines.push(line), + }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + expect(captured).toHaveLength(1); + expect(captured[0]!.method).toBe('DELETE'); + expect(captured[0]!.url).toContain('/api/cli/v1/tests/test_alpha'); + expect(captured[0]!.headers.get('idempotency-key')).toMatch(/^cli-delete-[0-9a-f-]{36}$/); + // No restore hint — hard-delete is permanent. + expect(errLines.some(l => l.includes('Restore'))).toBe(false); + }); + + it('respects a caller-supplied --idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + confirm: true, + idempotencyKey: 'op_delete_1', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenKey).toBe('op_delete_1'); + }); + + it('--dry-run bypasses the --confirm requirement', async () => { + const { credentialsPath } = makeCreds(); + // dry-run swaps fetch in via client-factory; we just ensure the + // command path completes without throwing the --confirm guard. + const res = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_alpha', + confirm: false, + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + // dry-run sample is canned per `src/lib/dry-run/samples.ts` (`deleteTest`) + expect(res.testId).toBe('test_dryrun_delete_2026'); + expect(res.deletedAt).toBe('2026-05-13T00:00:00.000Z'); + }); + + it('renders text mode with one line per field', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const out: string[] = []; + await runDelete( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_alpha', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const block = out.join('\n'); + expect(block).toContain('testId test_alpha'); + expect(block).toContain('deletedAt 2026-05-14T10:00:00.000Z'); + expect(block).not.toContain('restorableUntil'); + }); +}); + +describe('runCreateFromPlan', () => { + function writePlanFile(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + + const FE_PLAN = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'login rejects empty password', + description: 'Verify /login rejects empty password with a toast.', + planSteps: [ + { type: 'action', description: 'navigate to /login' }, + { type: 'action', description: 'click submit without filling password' }, + { type: 'assertion', description: "error toast 'invalid password' appears" }, + ], + }; + + const SAMPLE_RESPONSE = { + testId: 'test_planned', + type: 'frontend' as const, + codeVersion: 'v1', + createdAt: '2026-05-14T10:00:00.000Z', + }; + + it('POSTs /tests with planSteps body + idempotency-key from a valid FE plan file', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { body: SAMPLE_RESPONSE }; + }); + + const res = await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + // Fix 4 adds a best-effort dup-name GET before the POST, so captured may + // contain 2 entries. Find the POST specifically. + const sent = captured.find(c => c.method === 'POST')!; + expect(sent).toBeDefined(); + expect(sent.url).toContain('/api/cli/v1/tests'); + expect(sent.body).toEqual({ + projectId: 'project_alice', + type: 'frontend', + name: 'login rejects empty password', + description: 'Verify /login rejects empty password with a toast.', + planSteps: FE_PLAN.planSteps, + }); + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-create-plan-[0-9a-f-]{36}$/); + }); + + // Dogfood 2026-05-18 (Joseph): PowerShell 5.1's `Set-Content -Encoding utf8` + // writes a UTF-8 BOM, which the previous JSON read rejected with a cryptic + // "Unexpected token ''" error (the BOM renders invisibly). The CLI now + // strips a leading BOM before parsing. + it('accepts a plan file with a UTF-8 BOM (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-bom-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, '' + JSON.stringify(FE_PLAN), 'utf8'); + type Captured = { body: unknown }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((_url, init) => { + captured.push({ body: init.body ? JSON.parse(init.body as string) : undefined }); + return { body: SAMPLE_RESPONSE }; + }); + + const res = await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: path, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + expect(res).toEqual(SAMPLE_RESPONSE); + // Fix 4 adds a best-effort dup-name GET before the POST; filter by body presence. + const postEntry = captured.find(c => c.body !== undefined)!; + expect(postEntry).toBeDefined(); + expect(postEntry.body).toEqual({ + projectId: 'project_alice', + type: 'frontend', + name: 'login rejects empty password', + description: 'Verify /login rejects empty password with a toast.', + planSteps: FE_PLAN.planSteps, + }); + }); + + it('rejects a BE plan pre-flight with the documented code-file nextAction', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ ...FE_PLAN, type: 'backend' }); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + message: 'Backend tests via the CLI require --code-file.', + nextAction: expect.stringContaining('--code-file'), + }); + expect(called).toBe(0); + }); + + it('--run triggers a POST /runs after create (M3.3 piece-3 implemented)', async () => { + // M3.3 has landed: --run no longer exits 7 UNSUPPORTED; it fires + // POST /tests/{testId}/runs after the create succeeds. + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + if (url.includes('/runs/')) { + // GET /runs/{runId} — return terminal run so poll exits + return { + body: { + runId: 'run_m33', + testId: 'test_planned', + projectId: 'project_alice', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 3, failedCount: 0 }, + }, + }; + } + if (url.includes('/runs')) { + // POST /tests/{testId}/runs + return { + body: { + runId: 'run_m33', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + const result = await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + run: true, + wait: true, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + // Should have made at least: POST /tests, POST /tests/.../runs, GET /runs/... + expect(seenUrls.some(u => u.includes('/runs'))).toBe(true); + // Returns the create response + expect(result).toMatchObject({ testId: 'test_planned' }); + }); + + // codex #128 P2: the `--run` chain derives `:run`. A near-limit + // base key would derive a >256-char run key, which `runTestRun` rejects — + // but only AFTER the create POST already fired, orphaning a test with no + // run. The fix validates the derived key up-front (before any I/O). + it('create --run with a 253-char --idempotency-key fails fast (exit 5) before the create POST (codex #128 P2)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(); + const longKey = 'k'.repeat(253); // 253 + ':run'.length (4) = 257 > 256 + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile: '/tmp/whatever.spec.ts', + idempotencyKey: longKey, + run: true, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'idempotencyKey' }), + }); + // The create POST must NOT have fired — no orphan test. + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('create --run with a 252-char --idempotency-key passes the chain-key guard (boundary: derived == 256)', async () => { + // 252 + ':run'.length (4) === 256 exactly — must NOT be rejected by the + // chain-key guard. It proceeds and fails later on the missing code-file, + // proving the key guard let it through (field is `code-file`, not + // `idempotencyKey`). + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(); + const boundaryKey = 'k'.repeat(252); + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'n', + codeFile: '/tmp/this-file-does-not-exist-p2b-boundary.spec.ts', + idempotencyKey: boundaryKey, + run: true, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'code-file' }), + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('create --plan-from --run with a 253-char --idempotency-key fails fast before the create POST (codex #128 P2)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(); + const longKey = 'k'.repeat(253); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: '/tmp/whatever-plan.json', + idempotencyKey: longKey, + run: true, + wait: false, + }, + { credentialsPath, fetchImpl: fetchImpl as never, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'idempotencyKey' }), + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + // Per codex round-1 P1: chained `test create --plan-from ... --run` + // with `--output json` must emit a SINGLE parseable JSON object on + // stdout, not the create envelope followed by the run envelope. + // Agents and CI scripts cannot `JSON.parse` two objects back-to-back. + it('chained --run --output json emits ONE merged envelope on stdout (codex P1)', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + const fetchImpl = makeFetch(url => { + if (url.includes('/runs/')) { + // GET /runs/{runId} — terminal so poll exits + return { + body: { + runId: 'run_chain_p1', + testId: 'test_planned', + projectId: 'project_alice', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + videoUrl: null, + stepSummary: { total: 3, completed: 3, passedCount: 3, failedCount: 0 }, + }, + }; + } + if (url.includes('/runs')) { + return { + body: { + runId: 'run_chain_p1', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + + const stdoutLines: string[] = []; + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + run: true, + wait: true, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + + // Exactly one logical line on stdout (the JSON object). Multiple + // \n inside the pretty-printed JSON.stringify(.. 2) are part of + // one `out.print` call; the `stdout` writer receives the full + // pretty body as one argument. + expect(stdoutLines).toHaveLength(1); + const parsed = JSON.parse(stdoutLines[0]!) as Record; + // Merged envelope: create fields at the top, run nested under `run`. + expect(parsed.testId).toBe('test_planned'); + expect(parsed.type).toBe('frontend'); + expect(parsed.createdAt).toBe('2026-05-14T10:00:00.000Z'); + expect(parsed.run).toMatchObject({ + runId: 'run_chain_p1', + status: 'passed', + testId: 'test_planned', + }); + }); + + it('chained --run --no-wait --output json also emits ONE merged envelope (codex P1)', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + const fetchImpl = makeFetch(url => { + if (url.includes('/runs')) { + return { + body: { + runId: 'run_chain_p1_nowait', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + return { body: SAMPLE_RESPONSE }; + }); + + const stdoutLines: string[] = []; + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ); + + expect(stdoutLines).toHaveLength(1); + const parsed = JSON.parse(stdoutLines[0]!) as Record; + expect(parsed.testId).toBe('test_planned'); + // Trigger envelope nested under `run`. + expect(parsed.run).toMatchObject({ + runId: 'run_chain_p1_nowait', + status: 'queued', + }); + }); + + // Per codex round-1 P2: --timeout on the --plan-from branch must use + // the same validator as the --code-file branch (parseTimeoutFlag with + // [1, 3600] integer range). Previously it went through parseNumericFlag + // which would accept --timeout 0, --timeout 1.5, and --timeout 999999. + it('rejects --plan-from --timeout 0 with VALIDATION_ERROR (codex P2)', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync( + ['create', '--plan-from', '/tmp/probe-plan.json', '--run', '--timeout', '0'], + { from: 'user' }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'timeout' }), + }); + }); + + it('rejects --plan-from --timeout 1.5 with VALIDATION_ERROR (codex P2)', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync( + ['create', '--plan-from', '/tmp/probe-plan.json', '--run', '--timeout', '1.5'], + { from: 'user' }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'timeout' }), + }); + }); + + it('rejects --plan-from --timeout 999999 (above cap) with VALIDATION_ERROR (codex P2)', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync( + ['create', '--plan-from', '/tmp/probe-plan.json', '--run', '--timeout', '999999'], + { from: 'user' }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'timeout' }), + }); + }); + + it('rejects a missing plan file with VALIDATION_ERROR', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: '/tmp/does-not-exist-piece5.json', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('rejects a plan missing required fields with a typed field pointer', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ projectId: 'p', type: 'frontend', planSteps: [] }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'name' }), + }); + }); + + it('L1778: a plan missing projectId fails fast (field projectId) BEFORE the ignored-flags warning', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + type: 'frontend', + name: 'x', + planSteps: [{ type: 'action', description: 'go' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + ignoredFlags: ['--project'], + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'projectId' }), + }); + // The misleading "ignoring --project" advisory must NOT precede the error. + expect(stderrLines.join(' ')).not.toContain('warning: --plan-from'); + }); + + it('L1778: a valid plan still warns about ignored flags (after validation passes)', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + ignoredFlags: ['--project', '--name'], + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ); + const errText = stderrLines.join(' '); + expect(errText).toContain('warning: --plan-from supplies the test definition'); + expect(errText).toContain('--project'); + expect(errText).toContain('--name'); + }); + + it('rejects a plan with an invalid step type', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + ...FE_PLAN, + planSteps: [{ type: 'go', description: 'do thing' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps[0].type' }), + }); + }); + + it('rejects oversize plan-step body (>200 steps) pre-flight', async () => { + const { credentialsPath } = makeCreds(); + const oversize = { + ...FE_PLAN, + planSteps: Array.from({ length: 201 }, (_, i) => ({ + type: 'action' as const, + description: `step ${i}`, + })), + }; + const planFile = writePlanFile(oversize); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps' }), + }); + }); + + it('respects a caller-supplied --idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + let seenKey: string | null = null; + const fetchImpl = makeFetch((_url, init) => { + seenKey = new Headers(init.headers as Record).get('idempotency-key'); + return { body: SAMPLE_RESPONSE }; + }); + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + idempotencyKey: 'op_plan_42', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenKey).toBe('op_plan_42'); + }); + + // --------------------------------------------------------------------------- + // Finding 4 — dup-name advisory must fire in runCreateFromPlan too (codex + // round-2: the round-1 advisory only covered runCreate). + // --------------------------------------------------------------------------- + + it('[finding-4] emits dup-name advisory on stderr when a same-name test exists in the plan project', async () => { + const { credentialsPath } = makeCreds(); + const existingTest = { + id: 'test_existing_plan', + projectId: 'project_alice', + name: 'login rejects empty password', // same as FE_PLAN.name + type: 'frontend', + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', + }; + let postCalled = false; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + return { body: { items: [existingTest], nextToken: null } }; + } + if ((init.method ?? 'GET') === 'POST') { + postCalled = true; + return { body: SAMPLE_RESPONSE }; + } + return { body: SAMPLE_RESPONSE }; + }); + const planFile = writePlanFile(FE_PLAN); + const stderrLines: string[] = []; + + const result = await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + + // Create still proceeds + expect(result.testId).toBe(SAMPLE_RESPONSE.testId); + expect(postCalled).toBe(true); + // Advisory on stderr mentioning the existing testId + const advisoryLine = stderrLines.find( + l => l.includes('[advisory]') && l.includes('test_existing_plan'), + ); + expect(advisoryLine).toBeDefined(); + expect(advisoryLine).toContain('test update'); + }); + + it('[finding-4] swallows listing error on plan-from path and still creates', async () => { + const { credentialsPath } = makeCreds(); + let listAttempted = false; + let postCalled = false; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + listAttempted = true; + return { + status: 500, + body: { + error: { + code: 'INTERNAL', + message: 'err', + nextAction: '', + requestId: 'r', + details: {}, + }, + }, + }; + } + if ((init.method ?? 'GET') === 'POST') { + postCalled = true; + return { body: SAMPLE_RESPONSE }; + } + return { body: SAMPLE_RESPONSE }; + }); + const planFile = writePlanFile(FE_PLAN); + + const result = await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + expect(listAttempted).toBe(true); + expect(postCalled).toBe(true); + expect(result.testId).toBe(SAMPLE_RESPONSE.testId); + }); + + it('[finding-4] skips dup-name lookup when dry-run is true', async () => { + const { credentialsPath } = makeCreds(); + let listAttempted = false; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + listAttempted = true; + } + return { body: SAMPLE_RESPONSE }; + }); + const planFile = writePlanFile(FE_PLAN); + + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile, dryRun: true }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + + // No listing call on dry-run path + expect(listAttempted).toBe(false); + }); + + // --------------------------------------------------------------------------- + // Finding B (codex round-2) — advisory dup-name lookup passes AbortSignal + // so the 5s AbortController deadline can abort a stalled fetch. + // --------------------------------------------------------------------------- + + it('[finding-B] advisory GET passes an AbortSignal (5s deadline abort gate)', async () => { + // The short-deadline advisory wraps client.get with an AbortController. + // Verify the listing fetch receives signal= so the abort can fire. + const { credentialsPath } = makeCreds(); + + let capturedSignal: AbortSignal | undefined; + let postCalled = false; + + // Use an async fetch impl that captures the signal from the listing call. + type FetchInput2 = Parameters[0]; + const asyncFetchImpl = (async (input: FetchInput2, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests?projectId=')) { + capturedSignal = init.signal ?? undefined; + return new Response(JSON.stringify({ items: [], nextToken: null }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if ((init.method ?? 'GET') === 'POST') { + postCalled = true; + return new Response(JSON.stringify(SAMPLE_RESPONSE), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify(SAMPLE_RESPONSE), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + + const planFile = writePlanFile(FE_PLAN); + + const result = await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { + credentialsPath, + fetchImpl: asyncFetchImpl as ReturnType, + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + expect(result.testId).toBe(SAMPLE_RESPONSE.testId); + expect(postCalled).toBe(true); + // The advisory GET must have been called with an AbortSignal so the + // 5s AbortController timer can cancel it. + expect(capturedSignal).toBeDefined(); + expect(capturedSignal).toBeInstanceOf(AbortSignal); + }); +}); + +describe('runCreateBatch', () => { + function writePlansJsonl(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-batch-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + + const FE_SPEC = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'one', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + + const SAMPLE_RESPONSE = { + results: [ + { specIndex: 0, testId: 'test_b0', status: 'created' as const }, + { specIndex: 1, testId: 'test_b1', status: 'created' as const }, + ], + summary: { total: 2, created: 2, failed: 0 }, + }; + + it('POSTs /tests/batch with the parsed specs + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC, { ...FE_SPEC, name: 'two' }]); + type Captured = { url: string; method: string; body: unknown; headers: Headers }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { body: SAMPLE_RESPONSE }; + }); + + const res = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(res).toEqual(SAMPLE_RESPONSE); + expect(captured).toHaveLength(1); + const sent = captured[0]!; + expect(sent.method).toBe('POST'); + expect(sent.url).toContain('/api/cli/v1/tests/batch'); + const body = sent.body as { tests: unknown[] }; + expect(body.tests).toHaveLength(2); + expect(sent.headers.get('idempotency-key')).toMatch(/^cli-create-batch-[0-9a-f-]{36}$/); + }); + + it('warns on stderr listing BE specs in a mixed batch but still proceeds', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([ + FE_SPEC, + { ...FE_SPEC, type: 'backend', name: 'be-spec' }, + { ...FE_SPEC, name: 'three' }, + ]); + let sent = 0; + const fetchImpl = makeFetch(() => { + sent += 1; + return { body: SAMPLE_RESPONSE }; + }); + const errLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errLines.push(line), + }, + ); + expect(sent).toBe(1); + expect(errLines.some(l => l.includes('warning: 1 of 3 specs have type="backend"'))).toBe(true); + expect(errLines.some(l => l.includes('indexes: 1'))).toBe(true); + }); + + it('rejects an oversize batch (>50 specs) pre-flight', async () => { + const { credentialsPath } = makeCreds(); + const plans = Array.from({ length: 51 }, () => FE_SPEC); + const plansFile = writePlansJsonl(plans); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'plans' }), + }); + expect(called).toBe(0); + }); + + it('rejects an empty JSONL file with VALIDATION_ERROR', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-empty-')); + const plansFile = join(dir, 'empty.jsonl'); + writeFileSync(plansFile, '\n \n', 'utf8'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'plans' }), + }); + }); + + it('rejects a malformed JSONL line with a typed line-number pointer', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p5-bad-')); + const plansFile = join(dir, 'bad.jsonl'); + writeFileSync(plansFile, `${JSON.stringify(FE_SPEC)}\nnot-json\n`, 'utf8'); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'plans[1]' }), + }); + }); + + it('rejects --max-concurrency < 1 before sending', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC]); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + maxConcurrency: 0, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'max-concurrency' }), + }); + }); + + it('rejects --max-concurrency > 100 (upper bound enforcement)', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC]); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + maxConcurrency: 101, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'max-concurrency' }), + }); + }); + + it('accepts --max-concurrency = 100 (boundary value)', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC]); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + // Should NOT throw VALIDATION_ERROR for exactly 100 + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + maxConcurrency: 100, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).resolves.toBeDefined(); + }); + + // Per codex round-1 P2: a 200 OK with `summary.created === 0` on a + // non-empty batch must not exit 0. Without this, a misconfigured + // batch job (every spec invalid) silently lands nothing in DDB while + // the wrapping CI pipeline considers the run green. + it('throws INTERNAL when summary.created === 0 on a non-empty batch (codex P2)', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC, { ...FE_SPEC, name: 'two' }]); + const zeroSuccessResponse = { + results: [ + { + specIndex: 0, + status: 'validation_error' as const, + error: { code: 'VALIDATION_ERROR', message: 'projectId invalid', field: 'projectId' }, + }, + { + specIndex: 1, + status: 'not_found' as const, + error: { code: 'NOT_FOUND', message: 'project not found' }, + }, + ], + summary: { total: 2, created: 0, failed: 2 }, + }; + const fetchImpl = makeFetch(() => ({ body: zeroSuccessResponse })); + const stdoutLines: string[] = []; + await expect( + runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { + credentialsPath, + fetchImpl, + stdout: l => stdoutLines.push(l), + stderr: () => undefined, + }, + ), + ).rejects.toMatchObject({ + code: 'INTERNAL', + details: expect.objectContaining({ total: 2, created: 0, failed: 2 }), + }); + // The per-spec response was printed before the throw so an operator + // sees both the per-spec error envelopes AND the non-zero exit. + expect(stdoutLines.some(line => line.includes('"created": 0'))).toBe(true); + }); + + it('keeps exit 0 on partial success (some created, some failed) — CI-friendly', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC, { ...FE_SPEC, name: 'two' }]); + const partialResponse = { + results: [ + { specIndex: 0, testId: 'test_b0', status: 'created' as const }, + { + specIndex: 1, + status: 'validation_error' as const, + error: { code: 'VALIDATION_ERROR', message: 'name too long', field: 'name' }, + }, + ], + summary: { total: 2, created: 1, failed: 1 }, + }; + const fetchImpl = makeFetch(() => ({ body: partialResponse })); + const res = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(res).toEqual(partialResponse); + }); + + // Fix #1 — dogfood 2026-05-15: dry-run result count echoes actual input count + it('--dry-run: result count matches the number of specs in the --plans file', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC, { ...FE_SPEC, name: 'two' }]); + const res = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + plans: plansFile, + }, + { credentialsPath, stdout: () => undefined, stderr: () => undefined }, + ); + // Input had 2 specs; dry-run used to return the canned 3-entry sample + expect(res.results).toHaveLength(2); + expect(res.summary.total).toBe(2); + expect(res.summary.created).toBe(2); + expect(res.results[0]?.specIndex).toBe(0); + expect(res.results[1]?.specIndex).toBe(1); + }); + + // — Duplicate plan-body advisory tests (dogfood L120, 2026-05-28) — + + it('emits advisory exactly once on stderr when ≥3 specs share identical planSteps+description', async () => { + const { credentialsPath } = makeCreds(); + const DUP_SPEC = { + ...FE_SPEC, + description: 'checkout flow', + planSteps: [ + { type: 'action' as const, description: 'click checkout' }, + { type: 'assertion' as const, description: 'order confirmed' }, + ], + }; + // 3 specs sharing identical planSteps+description, each with distinct name + const plansFile = writePlansJsonl([ + { ...DUP_SPEC, name: 'agent-A' }, + { ...DUP_SPEC, name: 'agent-B' }, + { ...DUP_SPEC, name: 'agent-C' }, + ]); + const fetchImpl = makeFetch(() => ({ + body: { + results: [ + { specIndex: 0, testId: 'test_x0', status: 'created' as const }, + { specIndex: 1, testId: 'test_x1', status: 'created' as const }, + { specIndex: 2, testId: 'test_x2', status: 'created' as const }, + ], + summary: { total: 3, created: 3, failed: 0 }, + }, + })); + const errLines: string[] = []; + await runCreateBatch( + { profile: 'default', output: 'json', debug: false, plans: plansFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errLines.push(line) }, + ); + // Advisory should appear exactly once + const advisoryLines = errLines.filter(l => l.includes('[advisory]')); + expect(advisoryLines).toHaveLength(1); + expect(advisoryLines[0]).toContain('3 spec(s) share an identical plan body'); + expect(advisoryLines[0]).toContain('prefix each name per agent'); + }); + + it('emits no advisory when <3 specs share identical planSteps+description', async () => { + const { credentialsPath } = makeCreds(); + const DUP_SPEC = { + ...FE_SPEC, + planSteps: [{ type: 'action' as const, description: 'click submit' }], + }; + // Only 2 duplicates — below threshold + const plansFile = writePlansJsonl([ + { ...DUP_SPEC, name: 'agent-A' }, + { ...DUP_SPEC, name: 'agent-B' }, + { ...FE_SPEC, name: 'distinct' }, // different planSteps body + ]); + const fetchImpl = makeFetch(() => ({ + body: { + results: [ + { specIndex: 0, testId: 'test_y0', status: 'created' as const }, + { specIndex: 1, testId: 'test_y1', status: 'created' as const }, + { specIndex: 2, testId: 'test_y2', status: 'created' as const }, + ], + summary: { total: 3, created: 3, failed: 0 }, + }, + })); + const errLines: string[] = []; + await runCreateBatch( + { profile: 'default', output: 'json', debug: false, plans: plansFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errLines.push(line) }, + ); + const advisoryLines = errLines.filter(l => l.includes('[advisory]')); + expect(advisoryLines).toHaveLength(0); + }); + + it('advisory goes to stderr and stdout remains a single parseable JSON document in --output json mode', async () => { + const { credentialsPath } = makeCreds(); + const DUP_SPEC = { + ...FE_SPEC, + planSteps: [ + { type: 'action' as const, description: 'click login' }, + { type: 'assertion' as const, description: 'dashboard visible' }, + ], + }; + const batchResponse = { + results: [ + { specIndex: 0, testId: 'test_z0', status: 'created' as const }, + { specIndex: 1, testId: 'test_z1', status: 'created' as const }, + { specIndex: 2, testId: 'test_z2', status: 'created' as const }, + ], + summary: { total: 3, created: 3, failed: 0 }, + }; + const plansFile = writePlansJsonl([ + { ...DUP_SPEC, name: 'agent-A' }, + { ...DUP_SPEC, name: 'agent-B' }, + { ...DUP_SPEC, name: 'agent-C' }, + ]); + const fetchImpl = makeFetch(() => ({ body: batchResponse })); + const stdoutLines: string[] = []; + const errLines: string[] = []; + await runCreateBatch( + { profile: 'default', output: 'json', debug: false, plans: plansFile }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => errLines.push(line), + }, + ); + // stderr carries the advisory (safe for both modes) + expect(errLines.some(l => l.includes('[advisory]'))).toBe(true); + // stdout in json mode is a single parseable JSON document + const stdoutDoc = stdoutLines.join('\n'); + const parsed: unknown = JSON.parse(stdoutDoc); + expect(parsed).toMatchObject({ summary: { total: 3, created: 3 } }); + }); +}); + +// --------------------------------------------------------------------------- +// Finding 1 — timeoutIsDefault in create-chain paths (codex round-2) +// --------------------------------------------------------------------------- +// Confirm that the first-run timeout hint fires (or doesn't) correctly when +// `test create --run --wait` and `test create --plan-from --run --wait` are +// used without or with an explicit --timeout. The chain path exercises +// runCreate / runCreateFromPlan → runTestRun with a mocked HTTP stack. +// --------------------------------------------------------------------------- + +describe('[finding-1] first-run --timeout hint flows through test create --run --wait chain', () => { + const CREATE_RESP = { + testId: 'test_chain_xyz', + type: 'frontend' as const, + codeVersion: 'v1', + createdAt: '2026-06-07T00:00:00.000Z', + }; + + const TRIGGER_RESP = { + runId: 'run_chain_001', + status: 'queued' as const, + enqueuedAt: '2026-06-07T00:01:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }; + + const PASSED_RUN = { + runId: 'run_chain_001', + testId: 'test_chain_xyz', + projectId: 'project_alice', + userId: 'user_1', + status: 'passed' as const, + source: 'cli' as const, + createdAt: '2026-06-07T00:01:00.000Z', + startedAt: '2026-06-07T00:01:01.000Z', + finishedAt: '2026-06-07T00:01:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli' as const, + failedStepIndex: null, + failureKind: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }; + + function makeChainFetch(): typeof globalThis.fetch { + return (async (input, init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const method = init?.method ?? 'GET'; + // dup-name GET /tests?projectId=... + if (method === 'GET' && url.includes('/tests')) { + return new Response(JSON.stringify({ items: [], nextToken: null }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // POST /tests (create) + if (method === 'POST' && /\/tests$/.test(url)) { + return new Response(JSON.stringify(CREATE_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // POST /runs (trigger) + if (method === 'POST' && url.includes('/runs')) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // GET /runs/{id} (poll) + return new Response(JSON.stringify(PASSED_RUN), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + } + + // Plan-from chain fetch is identical to the code-file chain fetch + const makePlanChainFetch = makeChainFetch; + + const instantSleep = () => Promise.resolve(); + + it('emits [hint] when --timeout was not set (timeoutIsDefault=true) for test create --run --wait', async () => { + const { credentialsPath } = makeCreds(); + const codeDir = mkdtempSync(join(tmpdir(), 'cli-f1-code-')); + const codeFile = join(codeDir, 'test.spec.ts'); + writeFileSync(codeFile, '// chain test code', 'utf8'); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + projectId: 'project_alice', + type: 'frontend', + name: 'chain test', + codeFile, + run: true, + wait: true, + timeout: 600, // parsed from undefined → DEFAULT_RUN_TIMEOUT_SECONDS + timeoutIsDefault: true, // <-- the fix: explicitly flag as default + }, + { + credentialsPath, + fetchImpl: makeChainFetch() as unknown as ReturnType, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(true); + }); + + it('does NOT emit [hint] when --timeout was explicitly set (timeoutIsDefault=false) for test create --run --wait', async () => { + const { credentialsPath } = makeCreds(); + const codeDir2 = mkdtempSync(join(tmpdir(), 'cli-f1-code2-')); + const codeFile = join(codeDir2, 'test2.spec.ts'); + writeFileSync(codeFile, '// chain test code 2', 'utf8'); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + projectId: 'project_alice', + type: 'frontend', + name: 'chain test 2', + codeFile, + run: true, + wait: true, + timeout: 120, + timeoutIsDefault: false, // <-- explicit timeout + }, + { + credentialsPath, + fetchImpl: makeChainFetch() as unknown as ReturnType, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(false); + }); + + it('emits [hint] for test create --plan-from --run --wait when timeoutIsDefault=true', async () => { + const { credentialsPath } = makeCreds(); + const FE_PLAN = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'plan chain test', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const dir = mkdtempSync(join(tmpdir(), 'cli-f1-plan-')); + const planFile = join(dir, 'plan.json'); + writeFileSync(planFile, JSON.stringify(FE_PLAN), 'utf8'); + + const stderrLines: string[] = []; + + await runCreateFromPlan( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + planFrom: planFile, + run: true, + wait: true, + timeout: 600, + timeoutIsDefault: true, // <-- the fix: --plan-from chain now threads this + }, + { + credentialsPath, + fetchImpl: makePlanChainFetch() as unknown as ReturnType, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(true); + }); + + it('does NOT emit [hint] for test create --plan-from --run --wait when timeoutIsDefault=false', async () => { + const { credentialsPath } = makeCreds(); + const FE_PLAN = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'plan chain test explicit', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const dir = mkdtempSync(join(tmpdir(), 'cli-f1-plan-explicit-')); + const planFile = join(dir, 'plan.json'); + writeFileSync(planFile, JSON.stringify(FE_PLAN), 'utf8'); + + const stderrLines: string[] = []; + + await runCreateFromPlan( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + planFrom: planFile, + run: true, + wait: true, + timeout: 120, + timeoutIsDefault: false, // <-- explicit timeout + }, + { + credentialsPath, + fetchImpl: makePlanChainFetch() as unknown as ReturnType, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + expect(stderrLines.some(l => l.includes('[hint]') && l.includes('--timeout'))).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 5 — dashboard URL in outputs (runCreate + runCreateBatch) +// --------------------------------------------------------------------------- + +describe('Fix 5 — dashboardUrl emission', () => { + function writeCodeFile(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-')); + const path = join(dir, 'test.spec.ts'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + const SAMPLE_RESP = { + testId: 'test_dash_01', + type: 'frontend' as const, + codeVersion: 'v1', + createdAt: '2026-06-09T10:00:00.000Z', + }; + + it('runCreate: JSON mode includes dashboardUrl when API URL is prod', async () => { + // Use prod API URL → resolvePortalUrl returns a URL + const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFile('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESP }; + }); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const printed = JSON.parse(out.join('')) as { testId: string; dashboardUrl?: string }; + expect(printed.testId).toBe('test_dash_01'); + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_dash/test/test_dash_01', + ); + }); + + it('runCreate: text mode emits Dashboard: line to stderr when API URL is prod', async () => { + const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFile('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESP }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(true); + const dashLine = stderrLines.find(l => l.startsWith('Dashboard:'))!; + expect(dashLine).toContain('www.testsprite.com/dashboard/tests/proj_dash/test/test_dash_01'); + }); + + it('runCreate: no dashboardUrl when API URL is unknown (localhost)', async () => { + const { credentialsPath } = makeCreds('sk-test', 'http://localhost:13502'); + const codeFile = writeCodeFile('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESP }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as { testId: string; dashboardUrl?: string }; + expect(printed.dashboardUrl).toBeUndefined(); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + }); + + // R1: suppress dashboardUrl under --dry-run (test create) + it('runCreate: --dry-run does NOT emit dashboardUrl in JSON mode', async () => { + // No credentialsPath / fetchImpl needed — dry-run bypasses both. + const codeFile = writeCodeFile('test("dash", async () => {});'); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + dryRun: true, + // Use prod-looking endpoint so resolvePortalUrl would match if not suppressed. + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: line => out.push(line), stderr: line => stderrLines.push(line) }, + ); + const printed = JSON.parse(out.join('')) as Record; + expect(printed['dashboardUrl']).toBeUndefined(); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + }); + + it('runCreate: --dry-run does NOT emit Dashboard: line to stderr', async () => { + const codeFile = writeCodeFile('test("dash", async () => {});'); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: () => undefined, stderr: line => stderrLines.push(line) }, + ); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + }); + + // Fix 5 plan-from coverage (E2E 2026-06-10 gap): projectId comes from the + // PLAN body, not opts — the enrichment must still fire. + it('runCreateFromPlan: JSON mode includes dashboardUrl (projectId from plan body)', async () => { + function writePlanFileDash(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-plan-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const planFile = writePlanFileDash({ + projectId: 'proj_dash_plan', + type: 'frontend', + name: 'dash plan test', + planSteps: [{ type: 'action', description: 'navigate' }], + }); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESP }; + }); + const out: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const printed = JSON.parse(out.join('')) as { testId: string; dashboardUrl?: string }; + expect(printed.testId).toBe('test_dash_01'); + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_dash_plan/test/test_dash_01', + ); + }); + + it('runCreateFromPlan: --dry-run does NOT emit dashboardUrl', async () => { + function writePlanFileDash(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-plan-dry-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + const planFile = writePlanFileDash({ + projectId: 'proj_dash_plan', + type: 'frontend', + name: 'dash plan test', + planSteps: [{ type: 'action', description: 'navigate' }], + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: line => out.push(line), stderr: line => stderrLines.push(line) }, + ); + const printed = JSON.parse(out.join('')) as Record; + expect(printed['dashboardUrl']).toBeUndefined(); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + }); + + // R1: suppress dashboardUrl under --dry-run (test create-batch) + it('runCreateBatch: --dry-run does NOT include dashboardUrl in JSON output', async () => { + function writePlansJsonl(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-batch-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + const spec = { + projectId: 'proj_dash', + type: 'frontend' as const, + name: 'dash batch', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const plansFile = writePlansJsonl([spec]); + const out: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: line => out.push(line), stderr: () => undefined }, + ); + // In dry-run the output may be a descriptor envelope (no results key) or + // a batch response — in either case no dashboardUrl must appear anywhere. + const hasUrl = out.join('').includes('dashboardUrl'); + expect(hasUrl).toBe(false); + }); + + // R3a: dashboardUrl in create --run JSON envelope + it('runCreate --run: dashboardUrl is included in the merged { ...create, run } JSON envelope', async () => { + // prod API URL so resolvePortalUrl maps correctly + const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFile('test("chain", async () => {});'); + const CREATE_RESP = { + testId: 'test_chain_01', + type: 'frontend' as const, + codeVersion: 'v1', + createdAt: '2026-06-09T10:00:00.000Z', + }; + const TRIGGER_RESP = { + runId: 'run_chain_01', + status: 'queued' as const, + enqueuedAt: '2026-06-09T10:00:01.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + if (url.includes('/runs')) return { status: 200, body: TRIGGER_RESP }; + return { status: 200, body: CREATE_RESP }; + }); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_chain', + type: 'frontend', + name: 'chain test', + codeFile, + run: true, + wait: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const printed = JSON.parse(out.join('')) as Record; + // The merged envelope must carry dashboardUrl from the create context. + expect(printed['dashboardUrl']).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_chain/test/test_chain_01', + ); + }); + + it('runCreate --run --dry-run: no dashboardUrl in the dry-run descriptor envelope', async () => { + const codeFile = writeCodeFile('test("chain", async () => {});'); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_chain', + type: 'frontend', + name: 'chain test', + codeFile, + run: true, + wait: false, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: line => out.push(line), stderr: () => undefined }, + ); + expect(out.join('').includes('dashboardUrl')).toBe(false); + }); + + // R3b: dashboardUrl in create-batch --run JSON output + it('runCreateBatch --run: per-item dashboardUrl in JSON run results', async () => { + function writePlansJsonl2(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-batch-run-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + // Use prod API URL + const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const spec = { + projectId: 'proj_batch', + type: 'frontend' as const, + name: 'batch spec', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const plansFile = writePlansJsonl2([spec]); + const BATCH_CREATE_RESP = { + results: [{ specIndex: 0, testId: 'test_batch_01', status: 'created' as const }], + summary: { total: 1, created: 1, failed: 0 }, + }; + const TRIGGER_RESP = { + runId: 'run_batch_01', + status: 'queued' as const, + enqueuedAt: '2026-06-09T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) return { body: BATCH_CREATE_RESP }; + if (url.includes('/runs')) return { body: TRIGGER_RESP }; + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + const out: string[] = []; + // Use sleep injection to avoid real delays. The run status is 'queued' (no --wait), + // so runBatchRun throws CLIError(1) at the end — catch it to inspect stdout first. + try { + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + } catch { + // CLIError exit 1 expected because status is 'queued' (not 'passed'). + // JSON was already emitted to stdout before the throw. + } + const printed = JSON.parse(out.join('')) as { results?: Array> }; + expect(printed.results).toHaveLength(1); + expect(printed.results![0]!['dashboardUrl']).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_batch/test/test_batch_01', + ); + }); + + it('runCreateBatch --run --dry-run: no dashboardUrl in dry-run run results', async () => { + function writePlansJsonl3(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dash-batch-run-dry-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + const spec = { + projectId: 'proj_batch', + type: 'frontend' as const, + name: 'batch dry-run spec', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const plansFile = writePlansJsonl3([spec]); + const out: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: line => out.push(line), stderr: () => undefined }, + ); + expect(out.join('').includes('dashboardUrl')).toBe(false); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts new file mode 100644 index 0000000..e254958 --- /dev/null +++ b/src/commands/test.ts @@ -0,0 +1,8537 @@ +import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { dirname, extname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { Command } from 'commander'; +import { + emitDryRunBanner, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { + assertContextIntegrity, + buildMeta, + pickCodeExtension, + resolveBundleDir, + stepFilenamePrefix, + writeBundle, + type WriteBundleResult, +} from '../lib/bundle.js'; +import { findSample } from '../lib/dry-run/samples.js'; +import { + ApiError, + CLIError, + RequestTimeoutError, + TransportError, + localValidationError, +} from '../lib/errors.js'; +import { + assertIdempotencyKey, + requireArrayLength, + requireEnum, + requireString, +} from '../lib/validate.js'; +import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; +import type { FetchImpl } from '../lib/http.js'; +import type { HttpClient } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { + fetchSinglePage, + paginate, + validatePaginationFlags, + type Page, + type PaginationFlags, +} from '../lib/pagination.js'; +import { pollRunUntilTerminal, TimeoutError } from '../lib/poll.js'; +import type { + RunResponse, + RunStatus, + RunStepDto, + TriggerRunResponse, + RerunResponse, + BatchRerunResponse, + BatchRerunAccepted, + RerunClosureMember, + ListRunsResponse, + RunHistoryItem, + RunSource, + BatchRunFreshResponse, + BatchRunFreshAccepted, +} from '../lib/runs.types.js'; +import { RUN_SOURCES } from '../lib/runs.types.js'; +import { assertNotLocal } from '../lib/target-url.js'; +import { createTicker } from '../lib/ticker.js'; +import { RateThrottle } from '../lib/rate-throttle.js'; +import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; +import { loadConfig } from '../lib/config.js'; + +/** + * `details` debug block per the CLI OpenAPI `Test` schema + * (M2.1 amendment). `processingStatus` / `testStatus` are the + * structured pair; either may be `null` when the source row has no + * analog (MCP rows have no separate processStatus). `rawStatus` is + * the deprecated pre-M2.1 mirror, kept one minor for callers that + * already parse it. All three are debug-only — automation depends on + * the typed top-level `status` field. + */ +export interface CliTestStatusDetails { + processingStatus: string | null; + testStatus: string | null; + rawStatus: string; +} + +/** + * Public Test shape per the CLI OpenAPI `Test` schema. + * `details` is optional debug context — automation must depend only on + * the typed top-level fields. `createdFrom` takes one of the three + * documented values (`portal` | `mcp` | `cli`); anything else is a + * contract violation worth surfacing rather than silently coercing. + */ +export interface CliTest { + id: string; + projectId: string; + /** + * §6.2 / M2.1 piece 4 — human-friendly project name. `null` when + * project lookup wasn't possible (record missing, ownership + * boundary, or pre-M2.1 backend that never populated the field). + * Optional on the wire so older facades that don't ship the field + * still type-check; the renderer falls back to `projectId` when + * absent. + */ + projectName?: string | null; + name: string; + type: 'frontend' | 'backend'; + createdFrom: 'portal' | 'mcp' | 'cli'; + status: CliPublicStatus; + createdAt: string; + updatedAt: string; + /** + * M3.4 — number of FE plan steps, or `null` for BE tests and rows + * without plan steps. Optional on the wire so pre-M3.4 facades that + * don't ship it still type-check; text mode shows it only when present + * and non-null. The dedicated read path for recovering the current + * count after a `test plan put --expected-step-count` 412. + */ + planStepCount?: number | null; + /** + * M2.1: structured `processingStatus` / `testStatus` pair plus the + * deprecated `rawStatus` mirror. Pre-M2.1 servers may still emit + * `{ rawStatus }` only — keep the structured fields optional on + * the wire even though M2.1 servers always populate them. The CLI + * accepts both shapes. + */ + details?: Partial; + /** + * G1a — test priority label, e.g. "p0" | "p1" | "p2" | "p3". + * Optional on the wire: pre-G1a backends omit the field; `null` means + * no priority has been set. Text mode surfaces it only when truthy. + */ + priority?: string | null; +} + +export type CliPublicStatus = + | 'draft' + | 'ready' + | 'queued' + | 'running' + | 'passed' + | 'failed' + | 'blocked' + | 'cancelled' + | 'unknown'; + +/** + * §6.3 TestCode wire shape. `code` is either the inline source body + * (when < 100 KB) or a presigned `https://` URL (when >= 100 KB). The + * caller distinguishes via {@link isPresignedCodeUrl}. + */ +export interface CliTestCode { + testId: string; + language: 'typescript' | 'javascript' | 'python'; + framework: 'playwright' | 'pytest'; + code: string; + codeVersion: string | null; + etag?: string | null; +} + +/** §6.4 TestStep wire shape. `null` is "not known", not "absent". */ +export interface CliTestStep { + testId: string; + stepIndex: number; + action: string; + description: string; + status: 'passed' | 'failed' | null; + screenshotUrl: string | null; + htmlSnapshotUrl: string | null; + runIdIfAvailable: string | null; + codeVersion: string | null; + capturedAt: string | null; + updatedAt: string; + /** + * §6.4 / M2.1 piece 4 — derived flag the facade owns. `true` only on + * step(s) that actually contributed to the test failure. `null` + * when the underlying backend row hasn't been classified yet (pre- + * M2.1 persistence). Optional on the wire so pre-M2.1 servers + * that don't emit the field still type-check. + */ + outcomeContributesToFailure?: boolean | null; +} + +/** + * §6.5 failureKind enum (M2.1 piece 4 widened from six to nine values). + * The CLI accepts unrecognized strings from the wire as `unknown` so + * adding a future enum value (e.g. `quota_exceeded`) is non-breaking + * — agents must not switch on raw strings outside the enumerated set. + */ +export type CliFailureKind = + | 'assertion' + | 'assertion_blocked' // M2.1 piece 4 + | 'routing_404' // M2.1 piece 4 + | 'network_timeout' // M2.1 piece 4 + | 'network' + | 'timeout' + | 'browser_crash' + | 'infra' + | 'unknown' + | null; + +export interface CliResultSummary { + passed: number; + failed: number; + skipped: number; +} + +/** §6.5 LatestResult wire shape. All correlation fields are required. */ +export interface CliLatestResult { + testId: string; + status: CliPublicStatus; + startedAt: string | null; + finishedAt: string | null; + videoUrl: string | null; + failureAnalysisUrl: string | null; + snapshotId: string; + runIdIfAvailable: string | null; + codeVersion: string | null; + /** + * The target URL used for this run. May be `null` when `targetUrlSource` + * is `'unresolved'` (the stored run row had no target URL and the backend + * did not fall back to the project default). + */ + targetUrl: string | null; + /** + * D1 — provenance of `targetUrl`. Present on backends that have shipped + * the D1 fix; omitted on older backends (treat as unknown when absent). + * + * - `'run'` — URL was stored explicitly on the TestRun row. + * - `'project-default'` — URL came from the project's configured default. + * - `'unresolved'` — no URL on the run row AND no project default; + * `targetUrl` will be `null`. + * - `null` — backend sent the field explicitly as null + * (semantically equivalent to `'unresolved'`). + */ + targetUrlSource?: 'run' | 'project-default' | 'unresolved' | null; + failedStepIndex: number | null; + failureKind: CliFailureKind; + summary: CliResultSummary; + /** + * §6.5.1 (M2.1 piece 3) — inline failure analysis. Present when the + * caller passed `--include-analysis` (`?includeAnalysis=true` on + * the wire); absent on the byte-identical-to-pre-M2.1 default. + */ + analysis?: CliAnalysisBlock; +} + +/** + * §6.5.1 (M2.1 piece 3) — analysis fields surfaced inline on + * `/result?includeAnalysis=true` and as the body of `/failure/summary`. + * + * Stable shape: ships even on passing/in-flight runs with every field + * inside `null`. `recommendedFixTarget` is `null` (not the always- + * `unknown` wrapper) when the analysis pipeline didn't fill it. + * `failureKind` mirrors `LatestResult.failureKind` for caller + * convenience. `snapshotId` mirrors the outer snapshot, so a caller + * comparing the inline analysis with a later `/failure` bundle can + * detect drift without a second round-trip. + */ +export interface CliAnalysisBlock { + rootCauseHypothesis: string | null; + recommendedFixTarget: CliFixTarget | null; + failureKind: CliFailureKind; + snapshotId: string; + /** + * L141 — set to `true` (JSON output only) when `rootCauseHypothesis` + * ends with `…` (U+2026), indicating the server truncated the text. + * Omitted when the field is null or untouched. This is a CLI-side + * observation; the server does not send this field. Full untruncated + * text requires backend support (backend follow-up). + */ + rootCauseHypothesisTruncated?: true; + /** + * L141 — set to `true` (JSON output only) when + * `recommendedFixTarget.rationale` ends with `…` (U+2026), indicating + * the server truncated the rationale. Omitted when not truncated. + */ + recommendedFixRationaleTruncated?: true; +} + +/** + * §5.2 (M2.1 piece 3) — body of `GET /tests/{testId}/failure/summary`. + * One-screen agent triage answer: status + failureKind + analysis, + * no bundle. Sibling of `failure get` for cases where the agent's + * first pass doesn't need video / screenshots / DOM snapshots. + */ +export interface CliFailureSummary { + testId: string; + status: CliPublicStatus; + failureKind: CliFailureKind; + snapshotId: string; + rootCauseHypothesis: string | null; + recommendedFixTarget: CliFixTarget | null; +} + +/** §6.7 narrow fix-target enum. Agents route on this; M2 emits 'unknown'. */ +export type CliFixKind = 'code' | 'selector' | 'data' | 'env' | 'unknown'; +export type CliEvidenceKind = 'screenshot' | 'snapshot' | 'log' | 'network' | 'console'; + +export interface CliFixTarget { + kind: CliFixKind; + reference: string | null; + rationale: string | null; +} + +export interface CliEvidence { + kind: CliEvidenceKind; + /** 1-based step index — matches portal display. */ + stepIndex: number; + /** Presigned S3 URL with shared 15-min TTL. */ + url: string; + /** + * LLM-generated transcription per §6.1. The CLI emits whatever the + * facade returned — never edited or generated client-side. + */ + summary: string; +} + +export interface CliFailureBlock { + rootCauseHypothesis: string | null; + /** + * §6.7 / M2.1 piece 3 visibility policy: `null` when the analysis + * pipeline didn't fill any of `kind` / `reference` / `rationale`. + * Pre-M2.1 the facade always emitted an `unknown` wrapper here; + * M2.1 drops it so agents route on `null` rather than parsing the + * always-unknown shape. + */ + recommendedFixTarget: CliFixTarget | null; + evidence: CliEvidence[]; +} + +/** §6.7 wire shape — one atomic snapshot of the latest failing run. */ +export interface CliFailureContext { + snapshotId: string; + testId: string; + projectId: string; + result: CliLatestResult; + steps: CliTestStep[]; + code: CliTestCode; + failure: CliFailureBlock; +} + +export interface TestDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + /** + * Raw stdout writer for streaming code bodies in `test code get`. No + * implicit newline; each call writes verbatim. Defaults to + * `process.stdout.write`. See `Output.writeChunk` for rationale. + */ + rawStdout?: (text: string) => void; + /** + * Injectable sleep function for the polling loop. Defaults to the + * `defaultSleep` in `poll.ts` (`setTimeout`-based). Inject an instant + * no-op in tests to avoid real delays. + */ + sleep?: (ms: number) => Promise; +} + +type CommonOptions = FactoryCommonOptions; + +interface ListOptions extends CommonOptions { + projectId: string; + type?: 'frontend' | 'backend'; + createdFrom?: 'portal' | 'mcp' | 'cli'; + /** + * §6.6 / M2.1 piece 2 — comma-separated list of public status + * values (e.g. `failed,blocked`). The CLI passes the raw string to + * the facade which validates token-by-token. Pre-validating client- + * side gives a friendlier error for typos like `--status fail`. + */ + status?: string; + pageSize?: number; + startingToken?: string; + maxItems?: number; +} + +const TEST_TYPES: ReadonlyArray<'frontend' | 'backend'> = ['frontend', 'backend']; +// 'cli' added 2026-06-04 (dogfood): backend now stamps createFrom='cli' on +// tests created via `testsprite test create`, so the `--created-from cli` +// list filter must accept it. See backend-v2.0 CLI_CREATED_FROMS. +const CREATED_FROMS: ReadonlyArray<'portal' | 'mcp' | 'cli'> = ['portal', 'mcp', 'cli']; +/** + * §6.6 / M2.1 piece 2 — public status values accepted by the + * `--status` filter. Mirrors `CLI_PUBLIC_STATUSES` on the facade + * side (cli-tests.types.ts). Client-side validation gives a friendly + * error before the request hits the wire — the facade rejects the + * same set with VALIDATION_ERROR. + */ +const PUBLIC_STATUSES: ReadonlyArray = [ + 'draft', + 'ready', + 'queued', + 'running', + 'passed', + 'failed', + 'blocked', + 'cancelled', + 'unknown', +]; + +/** + * Internal helper: resolve the effective API URL from command opts. + * Mirrors the resolution logic in `makeHttpClient` so we can compute a + * `dashboardUrl` without accessing the private `client.baseUrl`. + * Calls `loadConfig` which reads the credentials file (cheap, cached by OS). + * Used only at the emit stage, AFTER the main request completes. + */ +function resolveApiUrl(opts: CommonOptions, deps: TestDeps = {}): string { + if (opts.dryRun) return opts.endpointUrl ?? 'https://api.testsprite.com'; + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env: (deps as { env?: NodeJS.ProcessEnv }).env, + credentialsPath: (deps as { credentialsPath?: string }).credentialsPath, + }); + return config.apiUrl; +} + +export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise> { + // Validate inputs before touching credentials so a missing `--project` + // surfaces as `VALIDATION_ERROR` (exit 5) rather than `AUTH_REQUIRED` + // (exit 3) when the caller also lacks a configured key. Order matters + // for the CLI error spec §2 — bad input is a caller bug, not an auth + // gate. + requireProjectId(opts.projectId); + + const paginationFlags: PaginationFlags = validatePaginationFlags({ + pageSize: opts.pageSize, + startingToken: opts.startingToken, + maxItems: opts.maxItems, + }); + + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + // Match P2's "explicit pageSize ⇒ single-page" convention so an + // operator can grab one slice + cursor without auto-paging through a + // huge project. + const useSinglePage = opts.pageSize !== undefined && opts.maxItems === undefined; + + // M2.1 piece 2: validate `--status` tokens client-side before + // sending. Friendlier error than waiting for the server's 400 with + // a list of accepted tokens — and lets the user fix typos without + // a round trip. + validateStatusFilter(opts.status); + + const baseQuery: Record = { + projectId: opts.projectId, + type: opts.type, + createdFrom: opts.createdFrom, + status: opts.status, + }; + + let page: Page; + if (useSinglePage) { + page = await fetchSinglePage( + client, + '/tests', + paginationFlags.pageSize!, + opts.startingToken, + baseQuery, + ); + } else { + page = await paginate( + async ({ pageSize, cursor }) => + client.get>('/tests', { + query: { ...baseQuery, pageSize, cursor }, + }), + paginationFlags, + ); + } + + out.print(page, data => renderTestListText(data as Page)); + return page; +} + +/** + * §6.X / M3.2 piece-2 `CreateTestResponse` shape. `codeVersion` is the + * monotonic stamp piece-1 added to FE/BE Portal rows; the CLI re-uses + * it as the `If-Match` etag on `test code put` (piece-4). + */ +export interface CliCreateTestResponse { + testId: string; + type: 'frontend' | 'backend'; + codeVersion: string; + createdAt: string; +} + +export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; +export type CliCreatePriority = (typeof CLI_CREATE_PRIORITIES)[number]; + +/** + * 350 KB inline-code cap. Mirrors `MAX_INLINE_CODE_BYTES` in the backend + * (`CliTestsController`), which is sized to fit a full test row inside + * DDB's 400 KB item limit after metadata headroom. Enforced client-side + * as a pre-flight check so an obvious oversize file fails fast (exit 5) + * without spending a round-trip. The server enforces the same cap + * defensively. Lowered from 1 MB → 350 KB in backend PR #464; this CLI + * constant tracks it. + */ +const MAX_INLINE_CODE_BYTES = 350 * 1024; + +interface CreateOptions extends CommonOptions { + projectId: string; + type: 'frontend' | 'backend'; + name: string; + description?: string; + priority?: CliCreatePriority; + /** Source path to the test code. Read into memory; capped at 350 KB. */ + codeFile: string; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; + /** M3.3 chain: trigger a run after create. */ + run?: boolean; + /** M3.3 chain: poll until terminal when `run` is true. */ + wait?: boolean; + /** M3.3 chain: per-run timeout in seconds. */ + timeout?: number; + /** + * M4 piece-2 — BE dependency authoring flags. + * `--produces ` (repeatable): variable names this test captures. + * Maps to wire field `produces` → backend serialises as `captures` JSON. + * Backend-only; supplying with --type frontend → exit 5 (FE has no wave model). + */ + produces?: string[]; + /** + * `--needs ` (repeatable): variable names this test consumes. + * Maps to wire field `consumes`. Backend-only. + */ + needs?: string[]; + /** + * `--category `: free-text category. Use `teardown`/`cleanup` to mark + * a last-wave cleanup test in the wave planner. Backend-only. + */ + category?: string; + /** + * B2(c): true when --timeout was NOT explicitly set (the default is in + * effect). Threaded into RunTestRunOptions so the first-run hint fires. + */ + timeoutIsDefault?: boolean; + /** M3.3 chain: per-run target URL override. */ + targetUrl?: string; +} + +/** + * Chained `test create --run` / `test create --plan-from --run` derive the + * trigger idempotency key as `:run`. Validate that derived key + * BEFORE the create POST so a near-limit user-supplied base key fails fast + * (exit 5) instead of creating the test and THEN rejecting the 257+ char + * derived run key — which would leave an orphan test with no run (codex #128 + * P2). Auto-minted keys (no `--idempotency-key` supplied) are always short and + * thus exempt. `RUN_SUFFIX` must match the `:run` suffix appended in + * `runCreate` / `runCreateFromPlan` when chaining into `runTestRun`. + */ +function assertChainedRunKeyFits( + run: boolean | undefined, + idempotencyKey: string | undefined, +): void { + if (run !== true || idempotencyKey === undefined) return; + const RUN_SUFFIX = ':run'; + if (idempotencyKey.length + RUN_SUFFIX.length > 256) { + throw localValidationError( + 'idempotencyKey', + `must be at most ${256 - RUN_SUFFIX.length} characters when used with --run ` + + `(the chained trigger derives "${RUN_SUFFIX}", which must stay within the 256-char limit)`, + undefined, + 'flag', + ); + } +} + +/** + * B3 / Fix 4: best-effort duplicate-name advisory shared by `runCreate` + * and `runCreateFromPlan`. One-page lookup (pageSize=100) — not + * exhaustive but cheap. Silently swallows all errors; must never block + * or fail the caller's create. + * + * Skip when `projectId` or `name` is absent (e.g. plan not yet parsed) + * or when the caller is in dry-run mode. + */ +/** Short deadline for the advisory duplicate-name lookup (5 s). */ +const DUP_NAME_ADVISORY_TIMEOUT_MS = 5_000; + +async function emitDupNameAdvisoryIfNeeded( + client: HttpClient, + projectId: string | undefined, + name: string | undefined, + stderrFn: (line: string) => void, +): Promise { + if (!projectId || !name) return; + // B: the advisory lookup must NEVER block the create critical path. + // Use an AbortController with a 5 s deadline. When the timer fires it + // calls ac.abort(), which causes client.get (via the `signal` option) to + // throw an AbortError — caught below and swallowed. This ensures a stalled + // or retrying listing endpoint can't delay an otherwise-healthy create by + // the full request-timeout (120 s) or multiple transport retries. + // No secondary setTimeout is used to avoid leaking timers in tests. + const ac = new AbortController(); + const timer = setTimeout(() => ac.abort(), DUP_NAME_ADVISORY_TIMEOUT_MS); + try { + const listing = await client.get<{ items: CliTest[] }>( + `/tests?projectId=${encodeURIComponent(projectId)}&pageSize=100`, + { signal: ac.signal }, + ); + const nameLower = name.toLowerCase(); + const match = listing.items?.find(t => t.name.toLowerCase() === nameLower); + if (match) { + stderrFn( + `[advisory] A test named "${name}" already exists in this project (testId: ${match.id}). ` + + `Use \`testsprite test update ${match.id}\` to modify it, or proceed to create a duplicate.`, + ); + } + } catch { + // Swallow — this is best-effort; must not block the create. + } finally { + clearTimeout(timer); + } +} + +/** + * `test create --code-file ` — M3.2 piece-2 first mutation. + * + * Reads the file (with the same 350 KB pre-flight the server enforces) + * and POSTs to `/api/cli/v1/tests` with an `Idempotency-Key` header. + * Default key is `cli-create-`; a caller-supplied + * `--idempotency-key` lets retry tooling pin the same key across + * attempts so a network blip doesn't double-create. The exact key sent + * is echoed to stderr at `--debug` so an operator can reuse it. + * + * Dry-run: the request still goes through `HttpClient.post`, but the + * `client-factory` swaps in `createDryRunFetch` so no network call + * happens — the caller sees the canned response shape from + * `src/lib/dry-run/samples.ts`. The `--debug` events emit the canonical + * request envelope (URL, method, headers including Idempotency-Key) + * so the user can verify what would be sent. Matches the M2 P6 + * convention; no separate "envelope-only" output mode. + */ +export async function runCreate( + opts: CreateOptions, + deps: TestDeps = {}, +): Promise { + // P1-2: validate idempotency key before any I/O — non-ASCII chars cause a + // ByteString TypeError at the HTTP transport layer (exit 10 UNAVAILABLE). + assertIdempotencyKey(opts.idempotencyKey); + // codex #128 P2: the `--run` chain derives `:run` (see below); validate + // that derived key BEFORE the create POST so a near-limit base key fails fast + // (exit 5) instead of creating the test and THEN rejecting the 257+ char run + // key — which would orphan a created test with no run. + assertChainedRunKeyFits(opts.run, opts.idempotencyKey); + // Validate inputs before touching credentials or fs — matches the + // M2 read commands' "input gates first, then auth, then I/O" ordering. + requireProjectId(opts.projectId); + requireNonEmpty('name', opts.name); + // P1-3: client-side length checks matching server limits (name ≤200, + // description ≤2000) so the user gets instant, actionable errors instead + // of a cryptic server validation message. + if (opts.name !== undefined && opts.name.length > 200) { + throw localValidationError('name', 'must be at most 200 characters'); + } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('description', 'must be at most 2000 characters'); + } + // P2-11: extend the required-flag error message to suggest --plan-from for + // FE tests so operators who missed that flag get an actionable hint. + if (typeof opts.codeFile !== 'string' || opts.codeFile.length === 0) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Invalid request.', + nextAction: + 'Flag `--code-file` is invalid: is required. ' + + 'For frontend tests you can also supply the plan with `--plan-from ` instead of a code file.', + requestId: 'local', + details: { field: 'codeFile', reason: 'is required' }, + }, + }); + } + if (!['frontend', 'backend'].includes(opts.type)) { + throw localValidationError('type', 'must be one of: frontend, backend', [ + 'frontend', + 'backend', + ]); + } + if (opts.priority !== undefined && !CLI_CREATE_PRIORITIES.includes(opts.priority)) { + throw localValidationError('priority', `must be one of: ${CLI_CREATE_PRIORITIES.join(', ')}`, [ + ...CLI_CREATE_PRIORITIES, + ]); + } + + // M4 piece-2: --produces/--needs/--category are backend-only. FE plans have + // no wave model; fail-fast client-side to match the backend's 400 reject and + // save a round-trip. + if (opts.type === 'frontend') { + const depFlags: string[] = []; + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); + if (opts.category !== undefined) depFlags.push('--category'); + if (depFlags.length > 0) { + throw localValidationError( + depFlags[0]!, + `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + + `Remove ${depFlags.join('/')} or use --type backend.`, + ); + } + } + + // Dry-run path skips fs entirely so the operator can shake out the + // wire shape with a dummy `--code-file` (matches the M2 P6 dry-run + // contract — no real credentials, no real disk). The canned response + // from `src/lib/dry-run/samples.ts` echoes the same response shape + // a real call would produce. + const code = opts.dryRun ? DRY_RUN_PLACEHOLDER_CODE : readCodeFileGuarded(opts.codeFile); + + const idempotencyKey = opts.idempotencyKey ?? `cli-create-${randomUUID()}`; + // Surface the idempotency key on stderr so an operator who hits a + // transport-level retry-budget exhaustion can re-run with the same + // `--idempotency-key`. Without this, a generated UUID dies inside the + // process and a retry would mint a fresh key — duplicating the test + // if the original POST reached the server before the retry budget + // ran out. Stderr (not stdout) keeps json-mode output clean. + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const body: Record = { + projectId: opts.projectId, + type: opts.type, + name: opts.name, + description: opts.description, + priority: opts.priority, + code, + }; + + // M4 piece-2: thread BE dependency fields into the POST body. + // Only include when non-empty so the wire stays clean for tests that + // don't declare any dependencies (undefined is omitted by JSON.stringify). + if (opts.produces !== undefined && opts.produces.length > 0) { + body.produces = opts.produces; + } + if (opts.needs !== undefined && opts.needs.length > 0) { + body.consumes = opts.needs; + } + if (opts.category !== undefined) { + body.category = opts.category; + } + + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + // C1: --target-url is inert for backend tests (base URL is baked into the + // test code; the backend sandbox never receives targetUrl). Emit a + // pre-flight advisory so the user doesn't silently get the wrong env. + // Only fires when --run is also set, because targetUrl only matters at run + // time; a bare `test create --type backend --target-url` without --run does + // not execute the test, so the advisory would be confusing noise. + if (opts.type === 'backend' && opts.targetUrl !== undefined && opts.run === true) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn( + "[advisory] --target-url has no effect for backend tests (a backend test's base URL is defined inside its code).", + ); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + + // B3: best-effort duplicate-name advisory. Skip under --dry-run. + if (!opts.dryRun) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + await emitDupNameAdvisoryIfNeeded(client, opts.projectId, opts.name, stderrFn); + } + + const response = await client.post('/tests', { + body, + headers: { 'idempotency-key': idempotencyKey }, + }); + + // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the + // create's own print when chaining; `runTestRun` emits a single + // merged envelope `{ ...createResponse, run: }` so + // `--output json` stays parseable for agents and scripts. + if (opts.run === true) { + const runIdempotencyKey = `${idempotencyKey}:run`; + // R3a: compute dashboardUrl before the early return so it flows into + // the merged { ...createContext, run } envelope in JSON mode and + // appears on the Dashboard: stderr line in text mode. + // R1: suppress under --dry-run (fake canned test id). + const chainDashboardUrl = opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), opts.projectId, response.testId); + const createContextWithUrl = + chainDashboardUrl !== undefined ? { ...response, dashboardUrl: chainDashboardUrl } : response; + await runTestRun( + { + ...opts, + testId: response.testId, + idempotencyKey: runIdempotencyKey, + timeoutSeconds: opts.timeout ?? DEFAULT_RUN_TIMEOUT_SECONDS, + // B2(c): pass through whether --timeout was explicitly set. + // opts.timeout is already a parsed number (never undefined here) so we + // thread the dedicated flag rather than checking undefined again. + timeoutIsDefault: opts.timeoutIsDefault ?? false, + wait: opts.wait === true, + createContext: createContextWithUrl, + // Thread the known type so fast BE runs (terminal on first poll, where + // beFallbackUsed would be false) still render `steps: n/a (backend)`. + type: opts.type, + }, + deps, + ); + return response; + } + + // Fix 5: emit dashboard deep-link when projectId + testId are known client-side + // (no extra network call — both come from opts / response). + // R1: suppress under --dry-run — the test id is a fake canned value + // (e.g. "test_dryrun_create_2026") and a live-looking URL would mislead. + const dashboardUrl = opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), opts.projectId, response.testId); + if (opts.output === 'json') { + out.print(dashboardUrl !== undefined ? { ...response, dashboardUrl } : response, data => + renderCreateText(data as CliCreateTestResponse), + ); + } else { + out.print(response, data => renderCreateText(data as CliCreateTestResponse)); + if (dashboardUrl !== undefined) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn(`Dashboard: ${dashboardUrl}`); + } + } + return response; +} + +/** + * Stand-in `code` body used by `--dry-run`. The dry-run fetch impl + * never inspects the request body, so the value is just a placeholder + * — kept readable for debug-event captures. + */ +const DRY_RUN_PLACEHOLDER_CODE = '// dry-run placeholder code body'; + +/** + * Read the code body with a `stat`-first size guard so an oversize + * artifact is rejected BEFORE we load + decode the whole thing into + * memory. `readCodeFile` was the original sole-source — now wraps the + * guard so the same VALIDATION_ERROR / PAYLOAD_TOO_LARGE shapes the + * tests assert on still flow through. + */ +function readCodeFileGuarded(path: string): string { + const absolute = resolveAbsolute(path); + let stat; + try { + stat = statSync(absolute); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('code-file', `file does not exist: ${path}`); + } + if (code === 'EACCES') { + throw localValidationError('code-file', `permission denied reading ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('code-file', `cannot stat ${path}: ${reason}`); + } + if (stat.size > MAX_INLINE_CODE_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Inline code exceeds the 350 KB CLI cap (${stat.size} bytes).`, + nextAction: 'Upload via the Portal, or split into smaller tests.', + requestId: 'local', + details: { field: 'code-file', sizeBytes: stat.size, maxBytes: MAX_INLINE_CODE_BYTES }, + }, + }); + } + return readCodeFile(absolute); +} + +function readCodeFile(path: string): string { + try { + return readFileSync(resolveAbsolute(path), 'utf8'); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('code-file', `file does not exist: ${path}`); + } + if (code === 'EACCES') { + throw localValidationError('code-file', `permission denied reading ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('code-file', `cannot read ${path}: ${reason}`); + } +} + +function resolveAbsolute(path: string): string { + return isAbsolute(path) ? path : resolve(process.cwd(), path); +} + +/** + * Drop a leading UTF-8 BOM (U+FEFF) from a freshly-read file. PowerShell 5.1's + * default `Set-Content -Encoding utf8` writes a BOM; without this strip, + * `JSON.parse` fails with an invisible "Unexpected token" error that renders + * as a blank character on most consoles. Most JSON parsers strip BOM at this + * boundary — we just bring this one in line. + */ +function stripBom(raw: string): string { + return raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw; +} + +function requireNonEmpty(flagName: string, value: string): void { + if (typeof value !== 'string' || value.trim().length === 0) { + throw localValidationError(flagName, 'is required'); + } +} + +/** + * `test create` text-mode rendering. JSON-mode callers (the agent + * surface) get the wire shape verbatim via `out.print`; text mode is + * the human surface — one line per field, no shell-incompatible chars. + */ +function renderCreateText(response: CliCreateTestResponse): string { + return [ + `testId ${response.testId}`, + `type ${response.type}`, + `codeVersion ${response.codeVersion}`, + `createdAt ${response.createdAt}`, + ].join('\n'); +} + +/** + * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. + * `planStepsHash` is a sha256 over the canonicalized new array so + * agents can detect their own no-op replays without an extra read. + * `stepCount` is the post-update array length. + */ +export interface CliPutPlanStepsResponse { + testId: string; + planStepsHash: string; + stepCount: number; + updatedAt: string; +} + +/** + * piece-6 cap for the `PUT /tests/{id}/plan-steps` body. Distinct from + * piece-5's `MAX_PLAN_BODY_BYTES` because the wire shape differs + * slightly (no projectId / name / etc. on the replace path). + */ +const MAX_PLAN_STEPS_BODY_BYTES = 256 * 1024; + +interface PlanPutOptions extends CommonOptions { + testId: string; + /** Source path to the new plan-step JSON file (`{ planSteps: [...] }`). */ + stepsFile: string; + /** + * Optional defensive concurrency check. Server rejects with 412 + * when the current entity's `planSteps.length !== N`. FE has no + * `codeVersion`, so this is the only consistency knob. + */ + expectedStepCount?: number; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; + /** + * When set alongside `--dry-run`, synthesises a 412 error envelope + * so the user can preview the retry-hint output and exit code without + * a real API key. Only `PRECONDITION_FAILED` is supported today. + */ + dryRunSimulateError?: 'PRECONDITION_FAILED'; +} + +/** + * `test plan put --steps ` — M3.2 piece-6. + * + * Replace an FE test's `planSteps[]` with the array in `--steps`. The + * file is a single JSON object `{ planSteps: [...] }`; we don't echo + * the full `CliPlanInput` shape here because `projectId` / `name` etc. + * are not mutable from this endpoint. + * + * FE-only. BE tests get a 400 `VALIDATION_ERROR` from the server with + * a `nextAction` pointing at `test code put`. The CLI does **not** + * pre-fetch the test type — letting the server route saves a round + * trip and matches the piece-6 spec's "server-side routing" decision. + * + * Concurrency: FE has no `codeVersion`, so updates are last-writer- + * wins by default. Pass `--expected-step-count ` to set + * `If-Match-Step-Count`; the server rejects with 412 if the current + * array length differs. Useful for defensive callers who want to + * detect concurrent edits without a separate read. + * + * Idempotency: `cli-plan-put-` is the default key, surfaced to + * stderr so a transport retry can pin it. + */ +export async function runPlanPut( + opts: PlanPutOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + requireNonEmpty('test-id', opts.testId); + requireNonEmpty('steps', opts.stepsFile); + + if ( + opts.expectedStepCount !== undefined && + (!Number.isInteger(opts.expectedStepCount) || opts.expectedStepCount < 0) + ) { + throw localValidationError('expected-step-count', 'must be a non-negative integer'); + } + + const planSteps = readPlanStepsFileGuarded(opts.stepsFile); + + const idempotencyKey = opts.idempotencyKey ?? `cli-plan-put-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const headers: Record = { 'idempotency-key': idempotencyKey }; + if (opts.expectedStepCount !== undefined) { + headers['if-match-step-count'] = String(opts.expectedStepCount); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + + // --dry-run --dry-run-simulate-error PRECONDITION_FAILED: synthesise + // a 412 envelope so the user sees the error and exit code 6 without + // a real API key. + if (opts.dryRun && opts.dryRunSimulateError === 'PRECONDITION_FAILED') { + const expectedCount = opts.expectedStepCount ?? planSteps.length; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `Plan-steps conflict. Server has a different step count than ${expectedCount}. ` + + `Re-fetch with 'testsprite test get ${opts.testId}' to see the current planSteps[] length and retry with --expected-step-count .`, + ); + throw ApiError.fromEnvelope( + { + error: { + code: 'PRECONDITION_FAILED', + message: `[dry-run simulation] Plan-steps conflict: step count mismatch (expected ${expectedCount}, server has 99).`, + nextAction: `Re-fetch the current plan-steps length and retry with --expected-step-count 99.`, + requestId: 'req_dry-run-simulate', + details: { expectedStepCount: expectedCount, currentStepCount: 99 }, + }, + }, + 412, + ); + } + + const response = await client.put( + `/tests/${encodeURIComponent(opts.testId)}/plan-steps`, + { + body: { planSteps }, + headers, + }, + ); + out.print(response, data => renderPlanPutText(data as CliPutPlanStepsResponse)); + return response; +} + +function renderPlanPutText(response: CliPutPlanStepsResponse): string { + return [ + `testId ${response.testId}`, + `planStepsHash ${response.planStepsHash}`, + `stepCount ${response.stepCount}`, + `updatedAt ${response.updatedAt}`, + ].join('\n'); +} + +/** + * Read + validate the `--steps` file. Returns the parsed `planSteps` + * array on success; throws a typed `VALIDATION_ERROR` envelope on any + * schema problem with a `details.field` pointer the caller can act on. + * + * Stat-first guard mirrors piece-2's `readCodeFileGuarded`: oversize + * payloads fail before we load them into V8's heap. The cap here is + * 256 KB (vs. 350 KB for code) per the piece-6 spec. + */ +function readPlanStepsFileGuarded(path: string): CliPlanStep[] { + const absolute = resolveAbsolute(path); + + let stat; + try { + stat = statSync(absolute); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('steps', `file does not exist: ${path}`); + } + if (code === 'EACCES') { + throw localValidationError('steps', `permission denied reading ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('steps', `cannot stat ${path}: ${reason}`); + } + if (stat.size > MAX_PLAN_STEPS_BODY_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Plan-steps body exceeds the 256 KB CLI cap (${stat.size} bytes).`, + nextAction: 'Split into multiple smaller tests or trim step descriptions.', + requestId: 'local', + details: { + field: 'steps', + sizeBytes: stat.size, + maxBytes: MAX_PLAN_STEPS_BODY_BYTES, + }, + }, + }); + } + + let raw; + try { + raw = stripBom(readFileSync(absolute, 'utf8')); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('steps', `cannot read ${path}: ${reason}`); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('steps', `not valid JSON: ${reason}`); + } + + return assertPlanStepsShape(parsed); +} + +/** + * Type-narrow + validate a parsed `{ planSteps: [...] }` envelope. + * The file is expected to be a single JSON object with a `planSteps` + * array property; we tolerate the bare array form too (some agents + * may emit `[...]` directly) so the surface forgives a common + * mistake without surprising callers. + */ +function assertPlanStepsShape(parsed: unknown): CliPlanStep[] { + let stepsRaw: unknown; + if (Array.isArray(parsed)) { + stepsRaw = parsed; + } else if (typeof parsed === 'object' && parsed !== null) { + stepsRaw = (parsed as Record).planSteps; + } else { + throw localValidationError('steps', 'must be a JSON object with a `planSteps` array'); + } + + requireArrayLength('planSteps', stepsRaw, { min: 1, max: MAX_PLAN_STEPS, itemNoun: 'step' }); + + for (let i = 0; i < stepsRaw.length; i += 1) { + const step = stepsRaw[i]; + if (typeof step !== 'object' || step === null || Array.isArray(step)) { + throw localValidationError(`planSteps[${i}]`, 'must be an object', undefined, 'field'); + } + const s = step as Record; + requireEnum(`planSteps[${i}].type`, s.type, PLAN_STEP_TYPES); + requireString(`planSteps[${i}].description`, s.description); + } + + return stepsRaw as CliPlanStep[]; +} + +/** + * §6.X / M3.2 piece-3 `UpdateTestResponse` shape. `updatedFields` is + * the array of top-level fields that changed in this call so JSON + * consumers know what landed — useful when the agent passed all three + * flags but the server normalized one to a no-op. + */ +export interface CliUpdateTestResponse { + testId: string; + updatedFields: string[]; + updatedAt: string; +} + +interface UpdateOptions extends CommonOptions { + testId: string; + /** Optional new name; at least one of `name`/`description`/`priority` must be set. */ + name?: string; + /** Optional new description (`null` is reserved for "clear"; CLI surfaces both). */ + description?: string; + /** Optional new priority. Enum-validated CLI-side. */ + priority?: CliCreatePriority; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; +} + +/** + * `test update ` — M3.2 piece-3. + * + * Metadata-only update: `name?`, `description?`, `priority?`. Code and + * plan-steps are not part of this surface (`test code put` and + * `test plan put` are the dedicated paths). The CLI does not even + * expose `--code` / `--plan-steps` flags here so a caller cannot + * accidentally try; the server would also reject those keys, but + * keeping the surface narrow is the cheaper guard. + * + * Refuses no-op invocations (none of the three set) with a typed + * `VALIDATION_ERROR` so a careless `test update ` doesn't burn a + * request. The error includes the accepted field set so an agent + * can self-correct without reading the help text. + * + * Idempotency-Key default is `cli-update-`; a caller-supplied + * `--idempotency-key` lets retry tooling pin the key. The generated + * value is echoed to stderr so an operator who hits a transport + * retry-budget exhaustion can re-run with the same key. Surfacing + * matches piece-2's pattern. + */ +export async function runUpdate( + opts: UpdateOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + requireNonEmpty('test-id', opts.testId); + // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.length > 200) { + throw localValidationError('name', 'must be at most 200 characters'); + } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('description', 'must be at most 2000 characters'); + } + if (opts.priority !== undefined && !CLI_CREATE_PRIORITIES.includes(opts.priority)) { + throw localValidationError('priority', `must be one of: ${CLI_CREATE_PRIORITIES.join(', ')}`, [ + ...CLI_CREATE_PRIORITIES, + ]); + } + + // No-op rejection: requires at least one of the three patchable + // fields. Caught before fetching credentials or building the + // request so the user gets the cheapest possible error. + const hasName = opts.name !== undefined; + const hasDescription = opts.description !== undefined; + const hasPriority = opts.priority !== undefined; + if (!hasName && !hasDescription && !hasPriority) { + throw localValidationError( + 'fields', + 'at least one of --name / --description / --priority must be set', + ['name', 'description', 'priority'], + ); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-update-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + // Body carries only the fields the caller passed. Sending + // `{ name: undefined }` would JSON-serialize to omit the key, which + // is the intended wire shape — but we build the body deliberately + // so the contract is auditable rather than dependent on + // JSON.stringify undefined-skipping. + const body: Record = {}; + if (hasName) body.name = opts.name!; + if (hasDescription) body.description = opts.description!; + if (hasPriority) body.priority = opts.priority!; + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.put( + `/tests/${encodeURIComponent(opts.testId)}`, + { + body, + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + out.print(response, data => renderUpdateText(data as CliUpdateTestResponse)); + return response; +} + +function renderUpdateText(response: CliUpdateTestResponse): string { + return [ + `testId ${response.testId}`, + `updatedFields ${response.updatedFields.join(', ')}`, + `updatedAt ${response.updatedAt}`, + ].join('\n'); +} + +/** + * §6.X / M3.2 piece-3 `DeleteTestResponse` shape. `deletedAt` is the + * delete timestamp (an ack) — hard-delete is immediate, so there is no + * restore window. + */ +export interface CliDeleteTestResponse { + testId: string; + deletedAt: string; +} + +interface DeleteOptions extends CommonOptions { + testId: string; + /** Hard gate — required (unless `--dry-run` is set). No interactive prompts. */ + confirm: boolean; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; +} + +/** + * `test delete --confirm` — M3.2 piece-3. + * + * Permanent hard-delete via DELETE /tests/{id}. The server removes the + * test row plus its steps and code object immediately — matching the + * Portal's own delete behavior — so the test disappears everywhere at + * once. There is no restore window. + * + * **`--confirm` is required** (unless `--dry-run`). Without either, + * the CLI exits 5 `VALIDATION_ERROR` with a typed envelope explaining + * the convention. The CLI never prompts interactively — matches the + * CI-friendly contract from the CLI error spec §2. + * + * Re-delete on an already-deleted (or missing) row returns 404 from the + * server. The CLI surfaces the envelope as-is; no client-side branching. + */ +export async function runDelete( + opts: DeleteOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + requireNonEmpty('test-id', opts.testId); + + if (!opts.confirm && !opts.dryRun) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Refusing to delete without --confirm.', + nextAction: + 'This permanently deletes the test (no restore window), and the CLI ' + + 'convention is explicit confirmation for destructive operations. ' + + 'Re-run with --confirm. (--dry-run also works without --confirm.)', + requestId: 'local', + details: { field: 'confirm', reason: 'required for destructive operation' }, + }, + }); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-delete-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.delete( + `/tests/${encodeURIComponent(opts.testId)}`, + { + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + + out.print(response, data => renderDeleteText(data as CliDeleteTestResponse)); + return response; +} + +function renderDeleteText(response: CliDeleteTestResponse): string { + return [`testId ${response.testId}`, `deletedAt ${response.deletedAt}`].join('\n'); +} + +/** + * Per-test outcome record for `test delete-batch` and `test delete --all`. + */ +export interface CliBulkDeleteResult { + testId: string; + status: 'deleted' | 'skipped' | 'error'; + /** + * Present when `status === 'deleted'`. ISO 8601 timestamp from the server. + */ + deletedAt?: string; + /** Present when `status === 'error'`. Short error description. */ + error?: string; +} + +export interface CliBulkDeleteSummary { + results: CliBulkDeleteResult[]; + summary: { total: number; deleted: number; skipped: number; failed: number }; +} + +interface DeleteBatchOptions extends CommonOptions { + /** Explicit list of testIds to delete. */ + testIds: string[]; + /** --all: resolve all tests in the project and delete them. */ + all: boolean; + /** --project : required with --all. */ + projectId?: string; + /** + * --status : with --all, only delete tests whose status matches. + * Uses the same validated set as `test list --status`. + */ + statusFilter?: string; + /** Hard gate — required (unless --dry-run). */ + confirm: boolean; +} + +/** + * `test delete-batch ` and `test delete --all --project ` — dogfood L1800. + * + * Deletes tests sequentially (to avoid hammering the server) and aggregates + * results into a single summary. Gated on `--confirm` (same convention as + * `test delete`). Prints a summary line to stderr and the per-test results + * object to stdout. + * + * Exit code: 0 if all targets were deleted (or `--dry-run`); 1 if any + * deletion failed (server error); 5 if `--confirm` is missing. + */ +export async function runDeleteBatch( + opts: DeleteBatchOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (!opts.confirm && !opts.dryRun) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Refusing to bulk-delete without --confirm.', + nextAction: + 'This permanently deletes the tests (no restore window). Re-run with --confirm.', + requestId: 'local', + details: { field: 'confirm', reason: 'required for destructive operation' }, + }, + }); + } + + // Bug 1 fix: reject the ambiguous combination of explicit IDs + --all before any + // resolution happens. Without this guard the explicit IDs are silently discarded + // and ALL project tests get deleted — a data-loss footgun. + if (opts.all && opts.testIds.length > 0) { + throw localValidationError('test-ids', 'Pass either explicit test IDs or --all, not both.'); + } + + if (opts.all && !opts.projectId) { + throw localValidationError('project', '--all requires a project id — pass --project '); + } + if (!opts.all && opts.testIds.length === 0) { + throw localValidationError( + 'test-ids', + 'provide at least one , or use --all --project to delete all tests in a project', + ); + } + + // Bug 2 fix: --status without --all would silently be ignored because the filter + // is only applied inside the `if (opts.all)` block below. Reject early so the + // operator knows their flag had no effect. + if (opts.statusFilter !== undefined && !opts.all) { + throw localValidationError( + 'status', + '--status only applies with --all (it filters which project tests get deleted). ' + + 'Remove --status, or add --all --project .', + ); + } + + // Validate --status filter. + if (opts.statusFilter !== undefined) { + validateStatusFilter(opts.statusFilter); + } + + const client = makeClient(opts, deps); + + let testIds = opts.testIds; + + if (opts.all) { + // Bug 3 fix: --dry-run uses a canned fetch impl that returns sample data + // regardless of the projectId, so the resolved list does NOT reflect the + // real project scope. Warn the operator so the preview isn't mistaken for + // an accurate count. + if (opts.dryRun) { + stderrFn( + '[dry-run] WARNING: the preview below uses sample data and does NOT reflect the ' + + 'real tests in your project. Remove --dry-run to see which tests would actually ' + + 'be deleted.', + ); + } + // Resolve all tests in the project. + stderrFn(`Resolving tests in project ${opts.projectId}…`); + const allPage = await paginate( + async ({ pageSize, cursor }) => + client.get>('/tests', { + query: { projectId: opts.projectId!, pageSize, cursor }, + }), + {}, + ); + let allTests = allPage.items; + + // --status filter. + if (opts.statusFilter !== undefined && opts.statusFilter !== '') { + const allowed = new Set( + opts.statusFilter + .split(',') + .map(s => s.trim()) + .filter(s => s.length > 0), + ); + const before = allTests.length; + allTests = allTests.filter(t => allowed.has(t.status)); + const skipped = before - allTests.length; + if (skipped > 0) { + stderrFn( + `--status filter: skipping ${skipped} test${skipped !== 1 ? 's' : ''} not matching status=${opts.statusFilter}.`, + ); + } + } + + testIds = allTests.map(t => t.id); + if (testIds.length === 0) { + stderrFn(`No tests found in project ${opts.projectId} matching filters — nothing to delete.`); + const empty: CliBulkDeleteSummary = { + results: [], + summary: { total: 0, deleted: 0, skipped: 0, failed: 0 }, + }; + out.print(empty); + return empty; + } + stderrFn(`Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} to delete.`); + } + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const dryResults: CliBulkDeleteResult[] = testIds.map(id => ({ + testId: id, + status: 'deleted' as const, + deletedAt: new Date().toISOString(), + })); + const summary: CliBulkDeleteSummary = { + results: dryResults, + summary: { total: testIds.length, deleted: testIds.length, skipped: 0, failed: 0 }, + }; + out.print(summary, data => renderBulkDeleteText(data as CliBulkDeleteSummary)); + return summary; + } + + const results: CliBulkDeleteResult[] = []; + + for (const testId of testIds) { + const idempotencyKey = `cli-delete-${randomUUID()}`; + try { + const resp = await client.delete( + `/tests/${encodeURIComponent(testId)}`, + { headers: { 'idempotency-key': idempotencyKey } }, + ); + results.push({ + testId, + status: 'deleted', + deletedAt: resp.deletedAt, + }); + } catch (err) { + // 404 = already deleted / not found. Surface as 'skipped' so the + // summary count is accurate and the exit code stays 0. + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + results.push({ testId, status: 'skipped', error: 'not found (already deleted?)' }); + } else { + const msg = err instanceof Error ? err.message : String(err); + results.push({ testId, status: 'error', error: msg }); + } + } + } + + const deleted = results.filter(r => r.status === 'deleted').length; + const skipped = results.filter(r => r.status === 'skipped').length; + const failed = results.filter(r => r.status === 'error').length; + const bulk: CliBulkDeleteSummary = { + results, + summary: { total: testIds.length, deleted, skipped, failed }, + }; + + stderrFn(`Deleted ${deleted}, skipped ${skipped}, failed ${failed}.`); + + out.print(bulk, data => renderBulkDeleteText(data as CliBulkDeleteSummary)); + + if (failed > 0) { + throw new CLIError( + `${failed} deletion${failed !== 1 ? 's' : ''} failed. See results for details.`, + 1, + ); + } + return bulk; +} + +function renderBulkDeleteText(bulk: CliBulkDeleteSummary): string { + const header = `Deleted: ${bulk.summary.deleted} Skipped: ${bulk.summary.skipped} Failed: ${bulk.summary.failed}`; + const rows = bulk.results.map(r => { + if (r.status === 'deleted') { + return ` ${r.testId} deleted ${r.deletedAt}`; + } + return ` ${r.testId} ${r.status} ${r.error ?? ''}`; + }); + return [header, ...rows].join('\n'); +} + +/** + * §6.X / M3.2 piece-5 — plan-step structure. `description` is the + * natural-language instruction the browser-use Lambda interprets at + * run time. `type` is the action/assertion binary the FE pipeline + * uses to decide whether a step is mutating UI or verifying state. + */ +export interface CliPlanStep { + type: 'action' | 'assertion'; + description: string; +} + +const PLAN_STEP_TYPES: ReadonlyArray = ['action', 'assertion']; + +/** + * Plan-from input file shape. Mirrors the body the controller accepts + * at `POST /api/cli/v1/tests` with `planSteps[]` (use-cases.md UC1). + * `priority` and `description` are optional metadata; everything else + * is required. + * + * FE-only after the 2026-05-13 scope cut. `type: "backend"` plans are + * rejected pre-flight by the CLI with a `nextAction` pointing at + * `test create --type backend --code-file `. + */ +export interface CliPlanInput { + projectId: string; + type: 'frontend' | 'backend'; + name: string; + description?: string; + priority?: CliCreatePriority; + planSteps: CliPlanStep[]; +} + +/** + * §6.X / M3.2 piece-5 — response from POST /tests with planSteps[]. + * Same wire shape as the code-based create (piece-2) plus an optional + * `planSteps` echo so the agent doesn't have to keep its own copy + * between create and run. `planSteps` is absent in dry-run mode + * because the sampler doesn't inspect the request body. + */ +export interface CliCreateFromPlanResponse extends CliCreateTestResponse { + planSteps?: CliPlanStep[]; +} + +/** Per-spec result from `POST /tests/batch`. */ +export interface CliBatchSpecResult { + /** Position of the spec in the input JSONL, preserved across the response. */ + specIndex: number; + /** Spec outcome. Mirrors the server's per-spec status enum. */ + status: 'created' | 'validation_error' | 'not_found'; + /** Set on success. */ + testId?: string; + /** Set on non-success. Carries the same envelope an `ApiError` would. */ + error?: { + code: string; + message: string; + field?: string; + }; +} + +export interface CliCreateBatchResponse { + results: CliBatchSpecResult[]; + summary: { + total: number; + created: number; + failed: number; + }; +} + +/** + * Per-run result from the `--run` fan-out on `test create-batch`. + * Each entry mirrors the shape of a single `test run --wait` JSON output + * so automation can process the array the same way it processes a single run. + */ +export interface CliBatchRunResult { + /** Test that was triggered. */ + testId: string; + /** Run ID minted by the trigger call (or the in-flight runId on CONFLICT resume). */ + runId: string; + /** Terminal status if `--wait`; `queued` if no `--wait`. */ + status: string; + /** Code version resolved at trigger time. */ + codeVersion: string; + /** Resolved target URL. */ + videoUrl?: string | null; + /** Failure kind if status is `failed` or `blocked`. */ + failureKind?: string | null; + /** Error envelope when the trigger itself failed (network/auth/validation). */ + error?: { code: string; message: string; exitCode: number }; +} + +/** Envelope emitted by `test create-batch --run` in JSON mode. */ +export interface CliCreateBatchRunResponse { + results: CliBatchRunResult[]; +} + +/** + * 200-step + 256 KB caps on a single plan body. The CLI + * enforces both client-side as a pre-flight guard so an obvious + * oversize plan fails fast (exit 5) without spending a round trip. + * The server enforces the same caps defensively. + */ +const MAX_PLAN_STEPS = 200; +const MAX_PLAN_BODY_BYTES = 256 * 1024; + +/** + * Batch caps per piece-5 §Backend: 50 specs per request, 5 MB total + * body. Same fail-fast pattern as the single-plan caps. + */ +const MAX_BATCH_SPECS = 50; +const MAX_BATCH_BODY_BYTES = 5 * 1024 * 1024; + +/** + * Maximum testIds per `POST /tests/batch/rerun` request (OpenAPI + * `BatchRerunRequest.testIds` maxItems: 50). When --all resolves more + * than this, the CLI splits into chunks and aggregates the results. + */ +const MAX_BATCH_RERUN_IDS = 50; + +/** + * Default max in-flight run-triggers for `create-batch --run`. + * + * Rationale: the server caps run-triggers at 60/min/key + * (`CLI_RUN_RATE_LIMIT_PER_MIN`, default 60 → `RATE_LIMITED` / exit 11). + * A `create-batch` holds at most MAX_BATCH_SPECS (50) specs, so a default + * of 50 lets a full batch dispatch all of its runs at once and finish + * launching within a single window. With async Lambda invoke each trigger + * returns in ~1s, so this bound mainly smooths the dispatch burst. + * + * NOTE: because 50 == MAX_BATCH_SPECS, a single `create-batch --run` can + * never trip the client-side `BATCH_RUN_RATE_LIMIT` token bucket (also + * 50/min) — the server's 60/min/key is the real backstop. The client + * throttle still guards repeated runs within a window from one process. + * + * Callers can override this default via `--max-concurrency` (raising it + * cannot lift the effective rate above the server cap). + */ +export const DEFAULT_BATCH_RUN_CONCURRENCY = 50; +/** Hard upper bound for --max-concurrency. Values above this are rejected with exit 5 (VALIDATION_ERROR). */ +export const MAX_BATCH_CONCURRENCY = 100; + +/** Client-side run-trigger throttle: 50 triggers per 60-second rolling window per key (sits just under the server's 60/min/key cap). */ +export const BATCH_RUN_RATE_LIMIT = 50; +/** Rolling window duration (ms) for the client-side trigger rate throttle. */ +export const BATCH_RUN_RATE_WINDOW_MS = 60_000; +/** Maximum number of outer RATE_LIMITED retries inside the batch fan-out (beyond HTTP-layer retries). */ +export const BATCH_RUN_RATE_MAX_OUTER_RETRIES = 5; + +/** + * D3: max automatic retry attempts for deferred tests under `--wait`. + * Each attempt is preceded by a Retry-After-aware sleep (server value if + * present, else 61s default), clamped to the remaining `--timeout` budget. + * Only active under `--wait`; the non-wait path is unchanged. + */ +export const MAX_DEFERRED_RETRIES = 3; +/** D3: default deferred-retry sleep when no Retry-After is available (ms). */ +export const DEFERRED_RETRY_DEFAULT_SLEEP_MS = 61_000; + +/** + * Returns `true` when a `RATE_LIMITED` error is the **transient per-minute** + * rate limit from `RunRateLimiterGuard` — the one worth retrying. + * + * The backend surfaces two distinct situations as `RATE_LIMITED` (429): + * + * 1. **Per-minute trigger cap** (RunRateLimiterGuard): + * message = `"Run trigger rate limit exceeded: N triggers per minute per key."` + * Has `Retry-After` header + `details.retryAfterSeconds`. + * → TRANSIENT — safe to retry once the window expires. + * + * 2. **Insufficient credits** (InsufficientCreditsException): + * message starts with `"Insufficient credits: N credit(s) required."` + * No `Retry-After` header, no `details.retryAfterSeconds`. + * → PERMANENT — retrying cannot succeed; only a top-up will fix it. + * + * We prefer a structural match (presence of `retryAfterMs` on the thrown + * `ApiError`, which is set only when the HTTP response carried a `Retry-After` + * header) as the primary discriminator, with the per-minute message wording as + * a secondary guard. If both fields are absent we treat the error as permanent + * to avoid silently burning the entire retry budget on a non-recoverable state. + * + * Limitation: a hypothetical future backend that emits `RATE_LIMITED` for a + * third reason without a `Retry-After` header AND without the per-minute + * wording would be classified as permanent here. Document this if it occurs. + */ +export function isTransientRateLimit(err: ApiError): boolean { + // Fix 4 (hardening): insufficient-credits is ALWAYS permanent, regardless of + // any Retry-After header the response may carry. Check this SHORT-CIRCUIT first + // so a credits-429 with a stray Retry-After header is never retried. + if (/insufficient credits/i.test(err.message)) return false; + + // Primary transient signal: Retry-After header was present and parsed (set on + // the error by HttpClient when retryOnRateLimit: false is used and the HTTP + // layer throws after a single 429). + if (err.retryAfterMs !== undefined) return true; + // Secondary: the per-minute rate-limit message wording. + if (/run trigger rate limit exceeded/i.test(err.message)) return true; + // Details presence (retryAfterSeconds in body) also indicates the throttle path. + const retryAfterSec = err.getDetail( + 'retryAfterSeconds', + (v): v is number => typeof v === 'number' && v > 0, + ); + if (retryAfterSec !== undefined) return true; + // Absent all signals → treat as permanent (credit depletion or unknown). + return false; +} + +interface CreateFromPlanOptions extends CommonOptions { + /** Path to the JSON file containing one `CliPlanInput`. */ + planFrom: string; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; + /** + * Reserved for the M3.3 chain. When `true`, the CLI will (once M3.3 + * lands) call `POST /tests/{id}/runs` after the create returns. For + * v0.1.0 piece-5 this is wired but emits exit 7 `UNSUPPORTED` + * pointing at the Portal trigger. + */ + run?: boolean; + /** Reserved for the M3.3 chain. Honored when `--run` is set. */ + wait?: boolean; + /** Reserved for the M3.3 chain. Per-run timeout in seconds. */ + timeout?: number; + /** + * B2(c): true when --timeout was NOT explicitly set (the default is in + * effect). Threaded into RunTestRunOptions so the first-run hint fires. + */ + timeoutIsDefault?: boolean; + /** Reserved for the M3.3 chain. Per-run target URL override. */ + targetUrl?: string; + /** + * Names of `test create` flags the caller supplied that `--plan-from` + * ignores (identity lives in the JSON). Surfaced as a stderr advisory + * AFTER the plan validates, so a malformed plan (e.g. missing + * `projectId`) fails fast with a clear field error instead of the + * misleading "ignoring --project" line landing first (dogfood L1778). + */ + ignoredFlags?: string[]; +} + +/** + * `test create --plan-from ` — M3.2 piece-5. + * + * FE-only path: agent writes a `planSteps[]` JSON file describing the + * test in natural language; CLI ships it to the backend; backend + * stores it on `FrontendTestEntity` for the browser-use Lambda to + * interpret at run time. The plan is the test definition — no + * server-side LLM compile happens at create time (that was the + * 2026-05-13 BE codegen scope cut; FE was never on that path). + * + * BE plans rejected pre-flight: if `plan.json` has `type: "backend"`, + * exit 5 `VALIDATION_ERROR` with a `nextAction` pointing at + * `test create --type backend --code-file `. Same envelope the + * server would return, just emitted without burning a round trip. + * + * `--run` is reserved for the M3.3 chain — currently exits 7 + * `UNSUPPORTED` per piece-5 spec; rewires to a real `POST /runs` call + * when M3.3 lands. + */ +export async function runCreateFromPlan( + opts: CreateFromPlanOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + // codex #128 P2: validate the derived `:run` chain key before the + // create POST (see runCreate) so a near-limit base key fails fast instead + // of orphaning a created test with no run. + assertChainedRunKeyFits(opts.run, opts.idempotencyKey); + requireNonEmpty('plan-from', opts.planFrom); + + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + const plan = readPlanFromGuarded(opts.planFrom); + + // The plan validated (projectId/type/name/planSteps present). Only NOW + // warn that overlapping `test create` flags were ignored — emitting this + // before validation made a missing-projectId failure look like the + // ignored --project flag was the cause (dogfood L1778). + if (opts.ignoredFlags && opts.ignoredFlags.length > 0) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `warning: --plan-from supplies the test definition; ignoring ${opts.ignoredFlags.join(', ')}. ` + + `Edit the plan JSON to change these fields.`, + ); + } + + // FE-only after the 2026-05-13 scope cut. The server also rejects + // BE plans, but bailing here saves a round trip and matches piece-2's + // "fast-fail at the input gate" pattern. + if (plan.type === 'backend') { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Backend tests via the CLI require --code-file.', + nextAction: + "Backend tests via the CLI require '--code-file '. Use 'testsprite test create --type backend --code-file foo.py'.", + requestId: 'local', + details: { field: 'type', reason: 'backend not supported in --plan-from path' }, + }, + }); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-create-plan-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const body = { + projectId: plan.projectId, + type: plan.type, + name: plan.name, + description: plan.description, + priority: plan.priority, + planSteps: plan.planSteps, + }; + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + + // Fix 4: best-effort duplicate-name advisory — same semantics as runCreate. + // The plan's projectId + name are available after validation above. Skip + // under dry-run (no network calls); swallow all errors (advisory only). + if (!opts.dryRun) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn); + } + + const response = await client.post('/tests', { + body, + headers: { 'idempotency-key': idempotencyKey }, + }); + + // Fix 5 (plan-from coverage): the projectId for the deep-link comes from + // the validated PLAN body (not opts — `--plan-from` has no --project-id + // flag). Same dry-run suppression as runCreate (fake canned test id). + const planDashboardUrl = opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), plan.projectId, response.testId); + + // --run chain (M3.3 piece-3): trigger + optionally wait. Per codex + // round-1 P1: suppress the create's own print when chaining; + // `runTestRun` emits a single merged envelope on stdout. + if (opts.run === true) { + // Idempotency key for the run is the create key + ":run" suffix so a + // retry of the whole chain gets the same runId. Per piece-3 spec. + const runIdempotencyKey = `${idempotencyKey}:run`; + const createContextWithUrl = + planDashboardUrl !== undefined ? { ...response, dashboardUrl: planDashboardUrl } : response; + return runTestRun( + { + ...opts, + testId: response.testId, + idempotencyKey: runIdempotencyKey, + timeoutSeconds: opts.timeout ?? DEFAULT_RUN_TIMEOUT_SECONDS, + // B2(c): thread through whether --timeout was explicitly set so the + // first-run hint fires for `test create --plan-from --run --wait`. + timeoutIsDefault: opts.timeoutIsDefault ?? false, + wait: opts.wait === true, + createContext: createContextWithUrl, + }, + deps, + ).then(() => response); + } + + if (opts.output === 'json') { + out.print( + planDashboardUrl !== undefined ? { ...response, dashboardUrl: planDashboardUrl } : response, + data => renderCreateText(data as CliCreateTestResponse), + ); + } else { + out.print(response, data => renderCreateText(data as CliCreateTestResponse)); + if (planDashboardUrl !== undefined) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn(`Dashboard: ${planDashboardUrl}`); + } + } + return response; +} + +/** + * Read + validate a plan JSON file. Returns the parsed `CliPlanInput` + * on success, throws a typed `VALIDATION_ERROR` envelope on any + * schema problem (missing fields, wrong types, oversize body, etc.). + * + * Stat-first guard mirrors piece-2's `readCodeFileGuarded` — reject + * obvious oversize files BEFORE loading them into V8's heap. For + * plans the cap is 256 KB (vs. 350 KB for code). + */ +function readPlanFromGuarded(path: string): CliPlanInput { + const absolute = resolveAbsolute(path); + + let stat; + try { + stat = statSync(absolute); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('plan-from', `file does not exist: ${path}`); + } + if (code === 'EACCES') { + throw localValidationError('plan-from', `permission denied reading ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plan-from', `cannot stat ${path}: ${reason}`); + } + if (stat.size > MAX_PLAN_BODY_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Plan body exceeds the 256 KB CLI cap (${stat.size} bytes).`, + nextAction: 'Split into multiple smaller tests or trim step descriptions.', + requestId: 'local', + details: { + field: 'plan-from', + sizeBytes: stat.size, + maxBytes: MAX_PLAN_BODY_BYTES, + }, + }, + }); + } + + let raw; + try { + raw = stripBom(readFileSync(absolute, 'utf8')); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plan-from', `cannot read ${path}: ${reason}`); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plan-from', `not valid JSON: ${reason}`); + } + + return assertPlanShape(parsed); +} + +/** + * Type-narrow + validate a parsed plan input. Pulled out so the same + * checks run on `--plan-from` (single) and each JSONL line in + * `create-batch --plans`. Throws `VALIDATION_ERROR` with a typed + * `details.field` pointer so callers can fix specific issues without + * re-reading the whole file. + */ +function assertPlanShape(parsed: unknown, context: { specIndex?: number } = {}): CliPlanInput { + const prefix = context.specIndex !== undefined ? `specs[${context.specIndex}].` : ''; + + // Every field below is a JSON body path inside the plan file (or + // JSONL spec), not a CLI flag — pass `'field'` so the error message + // says `Field \`projectId\` is invalid: ...` instead of inventing a + // `--projectId` flag the user can't pass. + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw localValidationError(`${prefix}plan`, 'must be a JSON object', undefined, 'field'); + } + const obj = parsed as Record; + + requireString(`${prefix}projectId`, obj.projectId); + requireEnum(`${prefix}type`, obj.type, ['frontend', 'backend'] as const); + requireString(`${prefix}name`, obj.name); + if (obj.description !== undefined && typeof obj.description !== 'string') { + throw localValidationError( + `${prefix}description`, + 'must be a string when present', + undefined, + 'field', + ); + } + if (obj.priority !== undefined) { + requireEnum(`${prefix}priority`, obj.priority, CLI_CREATE_PRIORITIES); + } + requireArrayLength(`${prefix}planSteps`, obj.planSteps, { + min: 1, + max: MAX_PLAN_STEPS, + itemNoun: 'step', + }); + for (let i = 0; i < (obj.planSteps as unknown[]).length; i += 1) { + const step = (obj.planSteps as unknown[])[i]; + if (typeof step !== 'object' || step === null || Array.isArray(step)) { + throw localValidationError( + `${prefix}planSteps[${i}]`, + 'must be an object', + undefined, + 'field', + ); + } + const s = step as Record; + requireEnum(`${prefix}planSteps[${i}].type`, s.type, PLAN_STEP_TYPES); + requireString(`${prefix}planSteps[${i}].description`, s.description); + } + + return obj as unknown as CliPlanInput; +} + +interface CreateBatchOptions extends CommonOptions { + /** Path to the JSONL file containing one `CliPlanInput` per line. */ + plans: string; + /** + * Path to a directory containing `*.json` plan files. Globs all `*.json` + * files (sorted by name for determinism), assembles them in-process into the + * same spec array as `--plans`, then runs the existing create-batch path. + * Mutually exclusive with `--plans`. + */ + planFromDir?: string; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; + /** When true, trigger a run for each created test after the batch create. */ + run?: boolean; + /** With `--run`, max number of in-flight triggers at once (default: `DEFAULT_BATCH_RUN_CONCURRENCY` = 50). */ + maxConcurrency?: number; + /** With `--run`, poll each run until terminal status before returning. */ + wait?: boolean; + /** With `--run --wait`, per-run max seconds to wait (1..3600, default 600). */ + timeoutSeconds?: number; + /** With `--run`, override the project default env URL for each triggered run. */ + targetUrl?: string; +} + +/** + * `test create-batch --plans ` — M3.2 piece-5. + * + * Reads one `CliPlanInput` per line of the JSONL file, ships them as + * a single `POST /tests/batch` request, returns per-spec results. + * The endpoint is FE-only; any spec with `type: "backend"` returns + * `validation_error` per-spec without aborting siblings. + * + * Caps: + * - 50 specs per batch (CLI rejects locally before sending) + * - 5 MB total body (CLI checks after stringify) + * - 200 steps + 256 KB per individual plan (per-line validation) + * + * BE specs in the batch are flagged on stderr with their `specIndex` + * so the operator can see what will fail server-side, but the batch + * still proceeds — the FE specs are perfectly valid. No interactive + * prompt (the CLI is CI-friendly per piece-3's convention). + * + * Exit code: `0` if **any** spec succeeded (POSIX partial-success + * convention per use-cases.md UC2 item 6). Non-zero only when zero + * specs succeeded — in which case the underlying API failure is + * surfaced via the normal exit-code mapper. + * + * `--run` triggers each successfully-created test after the batch + * create completes. `--max-concurrency` bounds the in-flight trigger + * count. `--wait` polls each run until terminal. `--timeout` is + * per-run (not aggregate). Output in JSON mode is + * `{ results: CliBatchRunResult[] }`. Exit code: 0 if every run + * passed; 1 if any failed/blocked/cancelled; 7 if ALL runs timed out; + * falls back to 1 for mixed outcomes. + */ +export async function runCreateBatch( + opts: CreateBatchOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + // Exactly one of --plans or --plan-from-dir is required. + if (opts.planFromDir !== undefined && opts.plans !== undefined && opts.plans !== '') { + throw localValidationError( + 'plan-from-dir', + '--plan-from-dir and --plans are mutually exclusive — supply only one', + ); + } + if ((opts.planFromDir === undefined || opts.planFromDir === '') && !opts.plans) { + throw localValidationError('plans', 'one of --plans or --plan-from-dir is required'); + } + + if (opts.maxConcurrency !== undefined && !Number.isInteger(opts.maxConcurrency)) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + if ( + opts.maxConcurrency !== undefined && + (opts.maxConcurrency < 1 || opts.maxConcurrency > MAX_BATCH_CONCURRENCY) + ) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + const stderrFnEarly = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const specs = + opts.planFromDir !== undefined && opts.planFromDir !== '' + ? readPlansFromDirGuarded(opts.planFromDir, stderrFnEarly) + : readPlansJsonlGuarded(opts.plans); + + // Duplicate plan-body advisory (dogfood L120, 2026-05-28). + // If ≥3 specs share an identical planSteps body + description, the operator + // is likely scoring multiple targets against the same test definition. + // Reusing one testId across targets (a) serializes runs per-testId on the + // server and (b) overwrites video history — each run overwrites the last. + // The correct pattern is one distinct testId per (agent × plan) pair, + // e.g. prefix each name per agent. Non-blocking: batch proceeds normally. + const dupBodyCount = countDuplicatePlanBodies(specs); + if (dupBodyCount > 0) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn( + `[advisory] ${dupBodyCount} spec(s) share an identical plan body + description. If you are scoring multiple targets, keep tests distinct (e.g. prefix each name per agent) — reusing one testId serializes runs and overwrites video history per testId.`, + ); + } + + // BE-spec stderr advisory. Server returns per-spec validation_error + // for any BE spec in a batch; we flag them up front so the operator + // sees the partial failure coming. No interactive prompt — CLI is + // CI-friendly per piece-3's convention. + const beIndexes = specs.map((s, i) => (s.type === 'backend' ? i : -1)).filter(i => i !== -1); + if (beIndexes.length > 0) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn( + `warning: ${beIndexes.length} of ${specs.length} specs have type="backend" (indexes: ${beIndexes.join(', ')}) — server will return per-spec validation_error for these. FE specs will still process. Use 'test create --type backend --code-file' for BE tests.`, + ); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-create-batch-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn(`idempotency-key: ${idempotencyKey}`); + } + + const body = { tests: specs }; + const bodyBytes = Buffer.byteLength(JSON.stringify(body), 'utf8'); + if (bodyBytes > MAX_BATCH_BODY_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Batch body exceeds the 5 MB CLI cap (${bodyBytes} bytes).`, + nextAction: 'Split into multiple --plans files.', + requestId: 'local', + details: { field: 'plans', sizeBytes: bodyBytes, maxBytes: MAX_BATCH_BODY_BYTES }, + }, + }); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.post('/tests/batch', { + body, + headers: { 'idempotency-key': idempotencyKey }, + }); + + // Per codex round-1 P2: zero successes on a non-empty batch must not + // exit 0. Partial success (some created, some failed) keeps exit 0 — + // that's the documented "CI-friendly" semantic. But "submitted N specs + // and got back 0 created" is indistinguishable from total failure to + // a CI runner, and silently exit-0 would let a misconfigured batch + // job leave nothing in DDB while the wrapping pipeline considers it + // green. + // + // P3-13: in JSON mode, emit a SINGLE envelope that wraps both the results + // and the error details, rather than printing the response first and then + // throwing (which produces two separate JSON objects on different streams + // and confuses machine consumers). Text mode still renders the human summary + // first and then the error line. + if (response.summary.total > 0 && response.summary.created === 0) { + if (opts.output === 'json') { + // Single JSON envelope on stdout: wraps all-failure results + error fields. + out.print({ + results: response.results, + summary: response.summary, + error: { + code: 'INTERNAL', + message: `Batch create produced 0 successful tests out of ${response.summary.total} specs.`, + nextAction: + 'Inspect per-spec errors in `results[]` (each entry carries `status` and `error.code`). Fix the failing specs and retry; pass the same --idempotency-key to safely re-send.', + }, + }); + } else { + // Text mode: print summary first so the operator can see per-spec failures. + out.print(response, data => renderBatchText(data as CliCreateBatchResponse)); + } + throw ApiError.fromEnvelope({ + error: { + code: 'INTERNAL', + message: `Batch create produced 0 successful tests out of ${response.summary.total} specs.`, + nextAction: + 'Inspect per-spec errors in `results[]` (each entry carries `status` and `error.code`). Fix the failing specs and retry; pass the same --idempotency-key to safely re-send.', + requestId: 'local', + details: { + total: response.summary.total, + created: 0, + failed: response.summary.failed, + }, + }, + }); + } + + // Fix 5: enrich results with per-item dashboardUrl in JSON mode. + // projectId comes from specs[specIndex].projectId; testId from the result row. + // Only emitted where both are known client-side — no extra network calls. + // R1: suppress under --dry-run — test ids are fake canned values and a + // live-looking URL would mislead the caller. + const apiUrlForDashboard = resolveApiUrl(opts, deps); + const enrichedResponse: CliCreateBatchResponse = + !opts.dryRun && opts.output === 'json' + ? { + ...response, + results: response.results.map(r => { + if (r.status !== 'created' || r.testId === undefined) return r; + const spec = specs[r.specIndex]; + const projectId = spec?.projectId; + if (!projectId) return r; + const dashboardUrl = resolvePortalUrl(apiUrlForDashboard, projectId, r.testId); + return dashboardUrl !== undefined ? { ...r, dashboardUrl } : r; + }), + } + : response; + + // --run: suppress the create output in JSON mode (we'll emit a single + // merged envelope at the end). In text mode, still print the create + // summary so the operator can see what was created. + if (!opts.run) { + out.print(enrichedResponse, data => renderBatchText(data as CliCreateBatchResponse)); + } else if (opts.output !== 'json') { + out.print(enrichedResponse, data => renderBatchText(data as CliCreateBatchResponse)); + } + + // --run: fan out a trigger for each created test, then emit results. + if (opts.run === true) { + // R3b: build testId → projectId map from the create results + specs so + // runBatchRun can enrich per-item run JSON with dashboardUrl. + const runTestIdToProjectId = new Map(); + for (const r of response.results) { + if (r.status === 'created' && r.testId !== undefined) { + const projectId = specs[r.specIndex]?.projectId; + if (projectId) runTestIdToProjectId.set(r.testId, projectId); + } + } + await runBatchRun( + opts, + response, + client, + out, + deps, + opts.dryRun ? undefined : runTestIdToProjectId, + opts.dryRun ? undefined : apiUrlForDashboard, + ); + // runBatchRun handles its own exit-code logic via CLIError. + // Return the create response to satisfy the return type; callers that + // inspect the return value only do so when not using --run. + } + + return response; +} + +/** + * Fan-out trigger for `test create-batch --run`. + * + * For each test successfully created in `createResponse`, mints a fresh + * idempotency key and calls `POST /tests/{testId}/runs`. Concurrency is + * bounded by `opts.maxConcurrency` (defaults to `DEFAULT_BATCH_RUN_CONCURRENCY` = 50 when absent). With + * `--wait`, polls each run until terminal. Per-run timeout applies + * individually (not aggregate). + * + * Output: + * - JSON mode: `{ results: CliBatchRunResult[] }` on stdout (single envelope). + * - Text mode: one line per completed run as they finish, then a final + * summary line `N/M passed, X failed, Y blocked, Z cancelled`. + * + * Exit codes: + * - 0 if every run passed. + * - 1 if any run failed/blocked/cancelled (or trigger error). + * - 7 if ALL runs timed out or errored with exit 7. + * - For other uniform errors (CONFLICT=6, RATE_LIMITED=11): exit with + * that code only when ALL runs share the same code; otherwise 1. + */ +async function runBatchRun( + opts: CreateBatchOptions, + createResponse: CliCreateBatchResponse, + client: HttpClient, + out: Output, + deps: TestDeps, + /** R3b: testId → projectId mapping built from create results + specs, used to enrich + * run-path JSON items with dashboardUrl. Populated by the caller; absent (undefined) + * means no enrichment (e.g. dry-run or caller didn't supply it). */ + testIdToProjectId?: Map, + /** R3b: resolved API URL for portal link resolution. */ + apiUrlForDashboard?: string, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const timeoutSeconds = opts.timeoutSeconds ?? DEFAULT_RUN_TIMEOUT_SECONDS; + const concurrencyLimit = opts.maxConcurrency ?? DEFAULT_BATCH_RUN_CONCURRENCY; + + // Collect successfully-created testIds in specIndex order. + const testIds = createResponse.results + .filter(r => r.status === 'created' && r.testId !== undefined) + .map(r => r.testId as string); + + if (testIds.length === 0) { + // All specs failed at create time — already threw above; unreachable. + return; + } + + // Dry-run: print a descriptor envelope and return without real triggers. + if (opts.dryRun) { + const dryRunResults: CliBatchRunResult[] = testIds.map(testId => ({ + testId, + runId: `dry-run-${randomUUID()}`, + status: 'queued', + codeVersion: 'v1', + })); + const envelope = { + dryRun: true, + method: 'POST', + pathTemplate: '/api/cli/v1/tests/{testId}/runs', + maxConcurrency: opts.maxConcurrency ?? null, + wait: opts.wait ?? false, + timeoutSeconds, + testIds, + ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), + results: dryRunResults, + }; + out.print(envelope); + return; + } + + const batchRunResults: CliBatchRunResult[] = []; + const sleepFn = deps.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + + /** + * Client-side sliding-window throttle: caps outgoing triggers at + * BATCH_RUN_RATE_LIMIT (50) per BATCH_RUN_RATE_WINDOW_MS (60 s), sitting + * just under the server's 60 triggers/min/key cap as a courtesy brake + * regardless of `--max-concurrency`. + * + * Separate CLI processes cannot coordinate this counter; cross-process + * collisions are handled by the RATE_LIMITED outer retry loop below. + */ + const rateThrottle = new RateThrottle(BATCH_RUN_RATE_LIMIT, BATCH_RUN_RATE_WINDOW_MS); + + /** + * Trigger (and optionally poll) a single testId. + * + * When `opts.wait` is set, a per-spec wall-clock deadline is set + * immediately before the first trigger attempt. Every throttle/retry sleep + * and the final `pollRunUntilTerminal` call receive only the remaining + * seconds so the `--wait` budget is never re-started after retries. + * + * Returns a CliBatchRunResult. Never throws — errors are captured + * into the result's `error` field so one failure doesn't abort siblings. + */ + async function triggerOne(testId: string): Promise { + // Mint a fresh idempotency key per run — MUST NOT reuse the create key. + const runIdempotencyKey = `cli-batch-run-${randomUUID()}`; + if (opts.debug) { + stderrFn(`[batch-run] ${testId} idempotency-key: ${runIdempotencyKey}`); + } + + // MAJOR 2: record the wall-clock deadline before the first trigger attempt + // when --wait is set so that throttle/retry sleeps + the subsequent poll all + // draw from the SAME budget. Triggering at t=0, waiting 60 s for throttle, + // then starting a fresh full-timeout poll would allow the spec to consume + // 2× the intended budget. + const specDeadlineMs: number | undefined = opts.wait + ? Date.now() + timeoutSeconds * 1000 + : undefined; + + /** Returns remaining milliseconds until the per-spec deadline, or Infinity when no deadline. */ + function remainingMs(): number { + if (specDeadlineMs === undefined) return Infinity; + return Math.max(0, specDeadlineMs - Date.now()); + } + + let triggerResponse: TriggerRunResponse; + + // Outer RATE_LIMITED retry loop. + // The batch call site passes `retryOnRateLimit: false` to `triggerRunWithMeta` + // so the HTTP layer throws on the first 429 — this loop is the SOLE owner of + // rate-limit handling. Single `test run` / `test create --run` still use + // retryOnRateLimit: true (the default) and are unaffected by this loop. + let outerRateLimitAttempt = 0; + while (true) { + // Fix 2: deadline check BEFORE acquiring a throttle slot or firing a trigger. + // Without this guard, an outer retry could acquire a slot and send a new + // POST even after the --wait deadline has already expired. + if (opts.wait && remainingMs() <= 0) { + return { + testId, + runId: '', + status: 'timeout', + codeVersion: '', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${timeoutSeconds}s before trigger attempt for ${testId}.`, + exitCode: 7, + }, + }; + } + + // Acquire a slot in the client-side rate window before firing the trigger. + // If the window is full, sleep until the oldest slot ages out — clamped to + // the remaining deadline so we don't overshoot the --wait budget. + let throttleWait: number; + while ((throttleWait = rateThrottle.acquire()) > 0) { + const clampedWait = Math.min(throttleWait, remainingMs()); + if (clampedWait <= 0) { + // Deadline already passed while waiting for a throttle slot. + return { + testId, + runId: '', + status: 'timeout', + codeVersion: '', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${timeoutSeconds}s waiting to acquire throttle slot for ${testId}.`, + exitCode: 7, + }, + }; + } + if (opts.debug) { + stderrFn( + `[batch-run] ${testId} — rate throttle: waiting ${Math.ceil(clampedWait / 1000)}s before trigger`, + ); + } + await sleepFn(clampedWait); + } + + try { + const result = await client.triggerRunWithMeta( + testId, + { source: 'cli', ...(opts.targetUrl ? { targetUrl: opts.targetUrl } : {}) }, + // retryOnRateLimit: false — the outer retry loop is the SOLE owner of + // rate-limit handling for the batch path. Allowing the HTTP layer to add + // up to 3 internal retries per outer attempt would multiply trigger + // POSTs per spec (e.g. 50×3 = 150/min), blowing the server's 60/min cap. + { idempotencyKey: runIdempotencyKey, retryOnRateLimit: false }, + ); + triggerResponse = result.body; + break; // success — exit the outer retry loop + } catch (err) { + // RATE_LIMITED outer retry. Since the HTTP layer no longer retries + // RATE_LIMITED (retryOnRateLimit: false above), every 429 reaches here + // on the first attempt. + // MAJOR 3: `ApiError.retryAfterMs` now carries the parsed `Retry-After` + // header value (clamped to [1s, 300s] by HttpClient). Use it instead of + // falling back to a hardcoded 60 s when the header is present. + // + // Credit-depletion vs transient rate-limit (project knowledge): + // Both conditions surface as `RATE_LIMITED` (exit 11). Credit depletion + // is PERMANENT — no amount of waiting will fix it. Only the transient + // per-minute throttle is safe to retry. `isTransientRateLimit()` checks + // for the `Retry-After` header OR the per-minute wording to distinguish. + if (err instanceof ApiError && err.code === 'RATE_LIMITED') { + if (!isTransientRateLimit(err)) { + // Permanent condition (insufficient credits or unknown RATE_LIMITED + // variant without Retry-After). Surface immediately — never retry. + return { + testId, + runId: '', + status: 'error', + codeVersion: '', + error: { code: err.code, message: err.message, exitCode: err.exitCode }, + }; + } + + if (outerRateLimitAttempt < BATCH_RUN_RATE_MAX_OUTER_RETRIES) { + outerRateLimitAttempt++; + + // MAJOR 3: use retryAfterMs from the thrown ApiError when available + // (set by HttpClient from the HTTP Retry-After header, clamped to + // [1s, 300s]). Fall back to details.retryAfterSeconds, then 60 s. + let retryAfterMs: number; + if (err.retryAfterMs !== undefined) { + retryAfterMs = err.retryAfterMs; + } else { + const retryAfterSec = err.getDetail( + 'retryAfterSeconds', + (v): v is number => typeof v === 'number' && v > 0, + ); + retryAfterMs = Math.min((retryAfterSec ?? 60) * 1000, 120_000); + } + + // MAJOR 2: clamp to remaining deadline so we don't overshoot the + // --wait budget. + const clampedRetryMs = Math.min(retryAfterMs, remainingMs()); + if (clampedRetryMs <= 0) { + return { + testId, + runId: '', + status: 'timeout', + codeVersion: '', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${timeoutSeconds}s during rate-limit backoff for ${testId}.`, + exitCode: 7, + }, + }; + } + stderrFn( + `[batch-run] ${testId} — RATE_LIMITED (outer attempt ${outerRateLimitAttempt}/${BATCH_RUN_RATE_MAX_OUTER_RETRIES}): waiting ${Math.ceil(clampedRetryMs / 1000)}s before retry`, + ); + await sleepFn(clampedRetryMs); + continue; // retry the outer loop + } + // Exceeded outer retry cap — surface as terminal error. + return { + testId, + runId: '', + status: 'error', + codeVersion: '', + error: { code: err.code, message: err.message, exitCode: err.exitCode }, + }; + } + // Reuse the same CONFLICT + --wait auto-resume logic as single-test run. + if (opts.wait && err instanceof ApiError && err.code === 'CONFLICT') { + const conflictReason = err.getDetail( + 'reason', + (v): v is string => typeof v === 'string' && v.length > 0, + ); + const currentRunId = err.getDetail( + 'currentRunId', + (v): v is string => typeof v === 'string' && v.length > 0, + ); + if (conflictReason === 'run_in_flight' && currentRunId !== undefined) { + stderrFn( + `[batch-run] ${testId} — run already in flight (runId: ${currentRunId}). Auto-resuming wait.`, + ); + triggerResponse = { + runId: currentRunId, + status: 'queued', + enqueuedAt: new Date().toISOString(), + codeVersion: '', + targetUrl: opts.targetUrl ?? '', + }; + break; // exit the outer retry loop with a conflict-resumed response + } else { + return { + testId, + runId: '', + status: 'error', + codeVersion: '', + error: { + code: (err as ApiError).code, + message: (err as Error).message, + exitCode: (err as ApiError).exitCode, + }, + }; + } + } else if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during trigger — classify as a timeout + // (exit 7) so the all-timeout aggregation can fire, mirroring the poll + // TimeoutError path below. + return { + testId, + runId: '', + status: 'timeout', + codeVersion: '', + error: { code: 'UNSUPPORTED', message: err.message, exitCode: err.exitCode }, + }; + } else { + const apiErr = err instanceof ApiError ? err : undefined; + return { + testId, + runId: '', + status: 'error', + codeVersion: '', + error: { + code: apiErr?.code ?? 'INTERNAL', + message: err instanceof Error ? err.message : String(err), + exitCode: apiErr?.exitCode ?? 1, + }, + }; + } + } + } + + if (!opts.wait) { + // No-wait path: return the trigger response as-is. + if (opts.output !== 'json') { + stderrFn( + `[batch-run] ${testId} — triggered (runId: ${triggerResponse.runId}, status: ${triggerResponse.status})`, + ); + } + return { + testId, + runId: triggerResponse.runId, + status: triggerResponse.status, + codeVersion: triggerResponse.codeVersion, + }; + } + + // --wait path: poll until terminal. + // Fix 3: check remaining budget BEFORE computing remainingSeconds so that + // 0 remaining ms (deadline already passed) yields a timeout result without + // polling. Math.max(1, ...) would otherwise convert 0 ms → 1 s poll. + const rem = remainingMs(); + if (opts.wait && rem <= 0) { + return { + testId, + runId: triggerResponse.runId, + status: 'timeout', + codeVersion: triggerResponse.codeVersion, + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${timeoutSeconds}s before polling run ${triggerResponse.runId}.`, + exitCode: 7, + }, + }; + } + // Pass only the REMAINING seconds into pollRunUntilTerminal so trigger + // retries don't restart the timeout clock from zero. + const remainingSeconds = Math.floor(rem / 1000) || 1; + let finalRun: RunResponse; + try { + finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose + ? (msg: string) => stderrFn(`[batch-run][verbose] ${testId}: ${msg}`) + : undefined, + }); + } catch (err) { + if (err instanceof TimeoutError) { + if (opts.output !== 'json') { + stderrFn( + `[batch-run] ${testId} (runId: ${triggerResponse.runId}) — timed out after ${timeoutSeconds}s`, + ); + } + return { + testId, + runId: triggerResponse.runId, + status: 'timeout', + codeVersion: triggerResponse.codeVersion, + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${timeoutSeconds}s waiting for run ${triggerResponse.runId}.`, + exitCode: 7, + }, + }; + } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7), consistent with the poll TimeoutError path above. + return { + testId, + runId: triggerResponse.runId, + status: 'timeout', + codeVersion: triggerResponse.codeVersion, + error: { code: 'UNSUPPORTED', message: err.message, exitCode: err.exitCode }, + }; + } + const apiErr = err instanceof ApiError ? err : undefined; + return { + testId, + runId: triggerResponse.runId, + status: 'error', + codeVersion: triggerResponse.codeVersion, + error: { + code: apiErr?.code ?? 'INTERNAL', + message: err instanceof Error ? err.message : String(err), + exitCode: apiErr?.exitCode ?? 1, + }, + }; + } + + if (opts.output !== 'json') { + stderrFn( + `[batch-run] ${testId} (runId: ${finalRun.runId}) — ${finalRun.status}${finalRun.failureKind ? ` (${finalRun.failureKind})` : ''}`, + ); + } + + return { + testId: finalRun.testId, + runId: finalRun.runId, + status: finalRun.status, + codeVersion: finalRun.codeVersion, + videoUrl: finalRun.videoUrl, + failureKind: finalRun.failureKind, + }; + } + + // Bounded concurrency fan-out using a semaphore pattern. + // We process testIds one slot at a time up to the concurrency limit. + // This avoids pulling in p-limit; the logic is simple enough inline. + const remaining = [...testIds]; + const inFlight = new Set>(); + + async function drainOne(): Promise { + const testId = remaining.shift(); + if (testId === undefined) return; + const p = triggerOne(testId).then(result => { + batchRunResults.push(result); + inFlight.delete(p); + return result; + }); + inFlight.add(p); + await p; + } + + // Fill up to concurrencyLimit slots, then drain one before adding + // each new testId so we never exceed the limit. + const initial = Math.min(concurrencyLimit, testIds.length); + const startPromises: Promise[] = []; + for (let i = 0; i < initial; i++) { + startPromises.push(drainOne()); + } + // Process remaining items as slots free up. + while (remaining.length > 0) { + // Wait for any in-flight slot to free up. + if (inFlight.size > 0) { + await Promise.race(inFlight); + } + if (remaining.length > 0 && inFlight.size < concurrencyLimit) { + await drainOne(); + } + } + // Wait for all in-flight to finish. + await Promise.all([...inFlight, ...startPromises]); + + // Sort by testId order (same as input order for stable output). + batchRunResults.sort((a, b) => testIds.indexOf(a.testId) - testIds.indexOf(b.testId)); + + // Emit output. + if (opts.output === 'json') { + // R3b: enrich per-item run results with dashboardUrl when both testId and + // projectId are known (from the testIdToProjectId map built by the caller). + // Additive-optional: items where projectId is unknown are left unchanged. + const enrichedResults = + !opts.dryRun && testIdToProjectId !== undefined && apiUrlForDashboard !== undefined + ? batchRunResults.map(r => { + const projectId = testIdToProjectId.get(r.testId); + if (!projectId || !r.testId) return r; + const dashboardUrl = resolvePortalUrl(apiUrlForDashboard, projectId, r.testId); + return dashboardUrl !== undefined ? { ...r, dashboardUrl } : r; + }) + : batchRunResults; + out.print({ results: enrichedResults }); + } else { + // Text mode: print summary line. + const passed = batchRunResults.filter(r => r.status === 'passed').length; + const failed = batchRunResults.filter(r => r.status === 'failed').length; + const blocked = batchRunResults.filter(r => r.status === 'blocked').length; + const cancelled = batchRunResults.filter(r => r.status === 'cancelled').length; + const errored = batchRunResults.filter( + r => r.status === 'error' || r.status === 'timeout', + ).length; + const total = batchRunResults.length; + const parts = [`${passed}/${total} passed`]; + if (failed > 0) parts.push(`${failed} failed`); + if (blocked > 0) parts.push(`${blocked} blocked`); + if (cancelled > 0) parts.push(`${cancelled} cancelled`); + if (errored > 0) parts.push(`${errored} error/timeout`); + stderrFn(`batch-run summary: ${parts.join(', ')}`); + } + + // Determine exit code. + const allPassed = batchRunResults.every(r => r.status === 'passed'); + if (allPassed) return; // exit 0 + + // Check for a uniform non-pass exit code across all non-passed results. + const errorExitCodes = batchRunResults + .filter(r => r.error !== undefined) + .map(r => r.error!.exitCode); + const nonPassedStatuses = batchRunResults.filter(r => r.status !== 'passed'); + // Exit 7 only when EVERY run timed out — a mix of pass + timeout is "mixed + // outcomes" (exit 1), not "all timed out". `nonPassedStatuses.every(...)` + // would incorrectly fire exit 7 when 1 of N passed and the rest timed out. + const allTimeout = + batchRunResults.length > 0 && + batchRunResults.every(r => r.status === 'timeout' || r.error?.exitCode === 7); + if (allTimeout) { + throw new CLIError( + `All ${batchRunResults.length} batch run(s) timed out after ${timeoutSeconds}s.`, + 7, + ); + } + // If all non-passed results share the same specific exit code (6 or 11), use it. + if (errorExitCodes.length > 0 && errorExitCodes.length === nonPassedStatuses.length) { + const uniformCode = errorExitCodes[0]; + if ( + uniformCode !== undefined && + errorExitCodes.every(c => c === uniformCode) && + uniformCode !== 1 && + uniformCode !== 7 + ) { + throw new CLIError( + `Batch run finished: ${nonPassedStatuses.length} run(s) failed with exit code ${uniformCode}.`, + uniformCode, + ); + } + } + // Default: mixed outcomes or generic failure → exit 1. + throw new CLIError( + `Batch run finished: ${batchRunResults.filter(r => r.status !== 'passed').length} of ${batchRunResults.length} run(s) did not pass.`, + 1, + ); +} + +/** + * Read + parse a JSONL plans file. Per-line validation; spec-level + * errors fail the whole batch before we send (since the server can't + * give us a per-spec response for a parse error). Caps the number of + * specs at 50 before any per-spec work happens. + */ +function readPlansJsonlGuarded(path: string): CliPlanInput[] { + const absolute = resolveAbsolute(path); + + let stat; + try { + stat = statSync(absolute); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('plans', `file does not exist: ${path}`); + } + if (code === 'EACCES') { + throw localValidationError('plans', `permission denied reading ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plans', `cannot stat ${path}: ${reason}`); + } + if (stat.size > MAX_BATCH_BODY_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Batch file exceeds the 5 MB CLI cap (${stat.size} bytes).`, + nextAction: 'Split into multiple --plans files.', + requestId: 'local', + details: { field: 'plans', sizeBytes: stat.size, maxBytes: MAX_BATCH_BODY_BYTES }, + }, + }); + } + + let raw; + try { + raw = stripBom(readFileSync(absolute, 'utf8')); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plans', `cannot read ${path}: ${reason}`); + } + + const lines = raw + .split('\n') + .map(l => l.trim()) + .filter(l => l.length > 0); + if (lines.length === 0) { + throw localValidationError('plans', 'file is empty (no JSONL records)'); + } + if (lines.length > MAX_BATCH_SPECS) { + throw localValidationError( + 'plans', + `must contain at most ${MAX_BATCH_SPECS} specs (got ${lines.length})`, + ); + } + + const specs: CliPlanInput[] = []; + for (let i = 0; i < lines.length; i += 1) { + let parsed: unknown; + try { + parsed = JSON.parse(lines[i]!); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError(`plans[${i}]`, `not valid JSON: ${reason}`); + } + specs.push(assertPlanShape(parsed, { specIndex: i })); + } + return specs; +} + +/** + * `test create-batch --plan-from-dir ` helper — M3.2 piece-5 extension + * (dogfood L1800). + * + * Globs all `*.json` files in the directory (non-recursive, sorted by name for + * determinism), reads each as a `CliPlanInput`, assembles them into the same + * in-process array that `--plans ` produces, then runs the existing + * create-batch path. Validates each file individually and reports errors by + * filename so the caller can fix one file at a time. + * + * Caps: 50 specs total (same as JSONL); aggregate size checked against + * MAX_BATCH_BODY_BYTES (5 MB) using the JSON-serialised size of the assembled + * spec array. + */ +function readPlansFromDirGuarded(dir: string, stderrFn: (line: string) => void): CliPlanInput[] { + const absolute = resolveAbsolute(dir); + + let entries: string[]; + try { + const dirStat = statSync(absolute); + if (!dirStat.isDirectory()) { + throw localValidationError('plan-from-dir', `not a directory: ${dir}`); + } + entries = readdirSync(absolute) + .filter(f => extname(f).toLowerCase() === '.json') + .sort(); + } catch (err) { + if (err instanceof ApiError || err instanceof CLIError) throw err; + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw localValidationError('plan-from-dir', `directory does not exist: ${dir}`); + } + if (code === 'EACCES') { + throw localValidationError('plan-from-dir', `permission denied reading directory: ${dir}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plan-from-dir', `cannot read directory ${dir}: ${reason}`); + } + + if (entries.length === 0) { + throw localValidationError('plan-from-dir', `no *.json files found in directory: ${dir}`); + } + + stderrFn(`Reading ${entries.length} plan file${entries.length !== 1 ? 's' : ''} from ${dir}`); + + const specs: CliPlanInput[] = []; + let skippedCount = 0; + for (let i = 0; i < entries.length; i += 1) { + const filename = entries[i]!; + const filePath = join(absolute, filename); + + let raw: string; + try { + raw = stripBom(readFileSync(filePath, 'utf8')); + } catch (err) { + // Hard I/O error (permission denied etc.): re-throw so the user knows the + // directory is unreadable — this is not a "skip" case. + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError('plan-from-dir', `cannot read ${filename}: ${reason}`); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + // Malformed / truncated JSON — FATAL. A syntax error almost certainly + // means the file was intended as a plan but got corrupted (e.g. a + // truncated write). Silently skipping it would let automation create + // an incomplete suite and exit 0 with no indication of the lost plan. + // Only valid-JSON objects that clearly lack plan identity (see below) + // are skipped. + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError( + 'plan-from-dir', + `${filename} contains invalid JSON (syntax error / truncated): ${reason} — fix or remove the file`, + ); + } + + // Heuristic: a "clearly non-plan" file is one that parses successfully as + // a JSON object but lacks ALL core plan-identity fields (projectId AND + // planSteps). Examples: suite-index.json, README.json, lock files. + // A file that HAS some plan fields but fails full assertPlanShape validation + // (e.g. has projectId but malformed planSteps) is a BOTCHED plan → FATAL. + const isObject = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed); + const obj = isObject ? (parsed as Record) : null; + const looksLikePlan = + obj !== null && + (obj['projectId'] !== undefined || + obj['planSteps'] !== undefined || + obj['plans'] !== undefined); + + if (!looksLikePlan) { + // Clearly not a plan — skip with an advisory. + stderrFn( + `[warn] Skipping ${filename}: not a plan file (no projectId/planSteps fields — treating as metadata)`, + ); + skippedCount += 1; + continue; + } + + // Parsed object looks like a plan (has some plan fields). Apply full + // shape validation — any failure here is FATAL because this was an + // INTENDED plan that has a structural problem the user must fix. + try { + specs.push(assertPlanShape(parsed, { specIndex: specs.length })); + } catch (err) { + const reason = + err instanceof ApiError + ? err.nextAction || err.message + : err instanceof Error + ? err.message + : 'unknown error'; + throw localValidationError( + 'plan-from-dir', + `${filename} looks like a plan file but failed validation: ${reason} — fix or remove the file`, + ); + } + } + + // If every file was skipped, escalate to a fatal error. + if (specs.length === 0) { + throw localValidationError( + 'plan-from-dir', + `no valid plan files found in directory: ${dir} (${skippedCount} file${skippedCount !== 1 ? 's' : ''} skipped — not valid plan specs)`, + ); + } + + // Enforce the batch-size cap on VALID specs (after skipping non-plan files + // like suite-index.json). A directory with 50 valid plans + 1 skipped file + // should succeed; the old check on entries.length rejected it pre-skip. + if (specs.length > MAX_BATCH_SPECS) { + throw localValidationError( + 'plan-from-dir', + `directory contains ${specs.length} valid plan specs, but the batch limit is ${MAX_BATCH_SPECS} — remove some files or split into multiple batches`, + ); + } + + // Aggregate size guard — stringify the assembled array and check total bytes. + const assembled = JSON.stringify(specs); + if (assembled.length > MAX_BATCH_BODY_BYTES) { + throw ApiError.fromEnvelope({ + error: { + code: 'PAYLOAD_TOO_LARGE', + message: `Plan-from-dir batch exceeds the 5 MB CLI cap (${assembled.length} bytes after assembly).`, + nextAction: 'Split into multiple smaller directories or trim step descriptions.', + requestId: 'local', + details: { + field: 'plan-from-dir', + sizeBytes: assembled.length, + maxBytes: MAX_BATCH_BODY_BYTES, + }, + }, + }); + } + + return specs; +} + +/** + * Returns the total number of specs that are members of groups where ≥3 + * specs share an identical plan body (planSteps + description). + * + * Group key: JSON-stable stringify of the spec's planSteps array (type + + * description per step, in order) plus the optional top-level description. + * Specs without planSteps (e.g. backend specs) are excluded — they produce + * no group key and never trigger the advisory. + * + * Used by `runCreateBatch` to emit a one-shot stderr advisory when the + * operator is likely scoring multiple targets against the same plan body + * (dogfood L120, 2026-05-28). + */ +function countDuplicatePlanBodies(specs: CliPlanInput[]): number { + const counts = new Map(); + for (const spec of specs) { + if (!spec.planSteps || spec.planSteps.length === 0) continue; + // Normalise: extract only type+description from each step so incidental + // extra fields don't break grouping, then pair with the spec description. + const stepsKey = JSON.stringify( + spec.planSteps.map(s => ({ type: s.type, description: s.description })), + ); + const key = JSON.stringify({ steps: stepsKey, description: spec.description ?? '' }); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + let dupTotal = 0; + for (const count of counts.values()) { + if (count >= 3) dupTotal += count; + } + return dupTotal; +} + +function renderBatchText(response: CliCreateBatchResponse): string { + const lines = [ + `total ${response.summary.total}`, + `created ${response.summary.created}`, + `failed ${response.summary.failed}`, + '', + 'specIndex status testId', + ]; + for (const r of response.results) { + const testId = r.testId ?? '-'; + const status = r.status.padEnd(17); + const specIdx = String(r.specIndex).padStart(9); + lines.push(`${specIdx} ${status} ${testId}`); + } + return lines.join('\n'); +} + +interface GetOptions extends CommonOptions { + testId: string; +} + +export async function runGet(opts: GetOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + out.print(test, data => renderTestText(data as CliTest)); + return test; +} + +interface CodeGetOptions extends CommonOptions { + testId: string; + /** + * Optional output file path. When set, the source body (text mode) or + * the JSON envelope (json mode) is written to this file instead of + * stdout. Streaming + backpressure are preserved: presigned downloads + * pipe straight into the file's write stream so a multi-MB body never + * sits in memory waiting for the full download. Per + * the CLI validation spec §4 P4: "the CLI streams the body to stdout + * (or `--out`) without buffering the whole thing in memory." + */ + out?: string; +} + +/** + * `test code get` — fetches §6.3 `TestCode`. JSON mode prints the wire + * shape verbatim (the caller decides whether to follow `code` as a URL). + * Text mode prints the source body itself: inline bodies pass through + * directly; presigned URLs are dereferenced via the same fetch impl + * (without API-key headers — the URL is the bearer of authority). + * + * `--out ` redirects the same bytes into a file. We open the file + * before issuing the network request so a permission/dir error fails + * fast (exit 5 / VALIDATION_ERROR) without spending an API call. On + * any error after open we tear down the sink so we don't leak a + * half-written artifact. + */ +export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Promise { + // Dry-run: no fetch, no fs. Print the canned shape to stdout and, if + // the user passed `--out`, log on stderr what would have been written. + // We deliberately do NOT validate the `--out` path here in dry-run — + // a missing-parent path is a real-mode failure mode; in dry-run the + // point is "show me the shape, no side effects." + if (opts.dryRun) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + const code = await client.get(`/tests/${encodeURIComponent(opts.testId)}/code`); + if (opts.out !== undefined) { + const bytes = isPresignedCodeUrl(code.code) ? '' : `${code.code.length}`; + stderr(`[dry-run] would write code body (${bytes} bytes) to ${opts.out}`); + } + if (opts.output === 'json') { + out.print(code); + } else { + await out.writeChunk(code.code); + } + return code; + } + + const fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; + const out = fileSink ? makeFileOutput(opts.output, fileSink) : makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + try { + const code = await client.get(`/tests/${encodeURIComponent(opts.testId)}/code`); + + if (opts.output === 'json') { + out.print(code); + } else if (isPresignedCodeUrl(code.code)) { + // Text mode: dump the source body. JSON consumers want the wire + // shape; humans (and agents shelling out via `> file.ts`) want + // ready-to-edit code. Stream chunk-wise so a multi-MB generated + // suite doesn't sit in memory waiting for the full download. + // `writeChunk` awaits the rawStdout drain promise so a slow + // downstream consumer (a file on a slow disk, an NFS mount, + // or a piped `gzip`) pauses the upstream reader rather than + // letting chunks accumulate in V8's heap. + await streamPresignedBody(code.code, out, deps); + } else if (code.code === '' || code.code === null) { + // P2-10: draft test with no code yet — empty body would produce + // silent empty stdout. Print a friendly hint to stderr instead so + // the operator knows what happened, and keep exit 0. + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn('(no code generated yet — run the test first)'); + } else { + await out.writeChunk(code.code); + } + + if (fileSink) await closeOutputFile(fileSink); + return code; + } catch (err) { + if (fileSink) await closeOutputFile(fileSink).catch(() => undefined); + throw err; + } +} + +/** + * §6.X / M3.2 piece-4 `PutTestCodeResponse` shape. `codeVersion` is + * the freshly bumped value the server stamped on the entity. Used as + * the next call's `If-Match` so an agent can chain put → put without + * round-tripping through `test get` between writes. + */ +export interface CliPutTestCodeResponse { + testId: string; + codeVersion: string; + updatedAt: string; +} + +type CodePutLanguage = CliTestCode['language']; +const CODE_PUT_LANGUAGES: ReadonlyArray = ['typescript', 'javascript', 'python']; + +interface CodePutOptions extends CommonOptions { + testId: string; + /** Source path to the new code body. Read into memory; capped at 350 KB. */ + codeFile: string; + /** + * `If-Match: ` value. Mutually exclusive with `force`. + * When neither is set, the CLI auto-fetches the current + * `codeVersion` via `GET /tests/{id}/code` and uses that — a + * convenience for human callers; agents should set this explicitly. + */ + expectedVersion?: string; + /** `--force` → sends `If-Match: *`. Audit-logged with `force: true`. */ + force?: boolean; + /** Optional language override; server defaults from the test's existing language otherwise. */ + language?: CodePutLanguage; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; + /** + * When set alongside `--dry-run`, synthesises an error envelope and + * runs through the error-handler path so the user can preview the + * retry-hint output and exit code without a real API key. + * Only `PRECONDITION_FAILED` is supported today. + */ + dryRunSimulateError?: 'PRECONDITION_FAILED'; +} + +/** + * `test code put --code-file ` — M3.2 piece-4. + * + * Replace the test's code body with optimistic concurrency. Backend + * checks `If-Match: ` against the current entity row; on + * match, bumps to `v(N+1)` and writes the new body via + * `StorageService.saveCodeContent`. On mismatch, returns 412 + * `PRECONDITION_FAILED` with `currentCodeVersion` in the error body so + * the caller can retry without an extra `GET`. + * + * Concurrency flag negotiation (CLI side): + * + * --expected-version → If-Match: (preferred for agents) + * --force → If-Match: * (audit-logged with force) + * (neither) → auto-fetch via GET, then If-Match: + * + * The auto-fetch path is **a convenience for human callers**. Agents + * should always pass `--expected-version` explicitly because the + * auto-fetch introduces a TOCTOU window: a concurrent writer can bump + * the version between our GET and our PUT. The CLI emits a stderr + * advisory whenever it takes the auto-fetch path so an operator + * watching the run can see what `If-Match` value was used. + * + * `--force` and `--expected-version` are mutually exclusive — passing + * both is a caller bug and we reject locally (exit 5) rather than + * silently picking one. Same is true for missing `--code-file`. + * + * 412 handling: the CLI extracts `currentCodeVersion` from the error + * envelope's `details` block (server populates it per piece-1's + * `CliPreconditionFailedError`) and prints a typed retry hint. The + * underlying `ApiError` is re-thrown so the exit-code mapper in + * `index.ts` lands on exit 6 — the CLI does not auto-retry. + * + * Dry-run skips all I/O including the auto-fetch — we substitute the + * dry-run sample's `codeVersion` (`v3`) so the canned response makes + * sense (v3 → v4 bump). Matches piece-2's pattern. + */ +export async function runCodePut( + opts: CodePutOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + requireNonEmpty('test-id', opts.testId); + requireNonEmpty('code-file', opts.codeFile); + + if (opts.expectedVersion !== undefined && opts.force === true) { + throw localValidationError( + 'expected-version', + 'is mutually exclusive with --force; pass one or the other (or neither for auto-fetch)', + ); + } + if (opts.language !== undefined && !CODE_PUT_LANGUAGES.includes(opts.language)) { + throw localValidationError('language', `must be one of: ${CODE_PUT_LANGUAGES.join(', ')}`, [ + ...CODE_PUT_LANGUAGES, + ]); + } + + const code = opts.dryRun ? DRY_RUN_PLACEHOLDER_CODE : readCodeFileGuarded(opts.codeFile); + + const idempotencyKey = opts.idempotencyKey ?? `cli-code-put-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + // Resolve the If-Match header. Three cases: + // 1. --force → '*' (skip etag check; audit-logged) + // 2. --expected-version → that string verbatim + // 3. neither → auto-fetch via GET /tests/{id}/code, use + // returned codeVersion. Tell the user (stderr) + // so an operator watching can see the race + // window we just opened. + const client = makeClient(opts, deps); + let ifMatch: string; + if (opts.force === true) { + ifMatch = '*'; + } else if (opts.expectedVersion !== undefined) { + requireNonEmpty('expected-version', opts.expectedVersion); + ifMatch = opts.expectedVersion; + } else { + const fetched = await client.get(`/tests/${encodeURIComponent(opts.testId)}/code`); + const cv = fetched.codeVersion; + if (cv === null || cv === undefined) { + // Server hasn't stamped a codeVersion yet (legacy row). Send `*` + // so the backend's force path applies the bump. Audit will mark + // force: true — visible signal that our auto-fetch hit a + // legacy row, not a concurrent-write race. + ifMatch = '*'; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `auto-fetched codeVersion=null on legacy row; using If-Match: * for this put. Pass --expected-version explicitly to avoid this fallback.`, + ); + } else { + ifMatch = cv; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + if (opts.dryRun) { + stderr( + `[dry-run] would auto-fetch codeVersion before PUT; pass --expected-version to avoid races (sample: ${cv})`, + ); + } else { + stderr( + `auto-fetched codeVersion=${cv} for If-Match. Pass --expected-version explicitly to avoid races.`, + ); + } + } + } + + const body: { code: string; language?: CodePutLanguage } = { code }; + if (opts.language !== undefined) body.language = opts.language; + + const out = makeOutput(opts.output, deps); + try { + // --dry-run --dry-run-simulate-error PRECONDITION_FAILED: throw a + // synthetic 412 envelope so the user sees the retry-hint and exit + // code 6 without a real API key. The throw feeds into the catch + // block below — identical code path as a real server 412. + if (opts.dryRun && opts.dryRunSimulateError === 'PRECONDITION_FAILED') { + throw ApiError.fromEnvelope( + { + error: { + code: 'PRECONDITION_FAILED', + message: `[dry-run simulation] Code conflict: server is at v99, you sent ${ifMatch}.`, + nextAction: `Re-fetch the current codeVersion and retry with --expected-version v99.`, + requestId: 'req_dry-run-simulate', + details: { currentCodeVersion: 'v99' }, + }, + }, + 412, + ); + } + const response = await client.put( + `/tests/${encodeURIComponent(opts.testId)}/code`, + { + body, + headers: { + 'idempotency-key': idempotencyKey, + 'if-match': ifMatch, + }, + }, + ); + out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); + return response; + } catch (err) { + // 412 envelope carries `currentCodeVersion` in details; surface + // the retry hint on stderr so an operator (or agent reading + // stderr) can paste it back. We re-throw the ApiError unchanged + // so the exit-code mapper still lands on exit 6 — the hint is + // additive, not a substitute. + if (err instanceof ApiError && err.code === 'PRECONDITION_FAILED') { + const serverVersion = + err.getDetail('currentCodeVersion', (v): v is string => typeof v === 'string') ?? + null; + const sentVersion = ifMatch === '*' ? '*' : ifMatch; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + if (serverVersion !== null) { + stderr( + `Code conflict. Server is at ${serverVersion}, you sent ${sentVersion}. ` + + `Re-fetch with 'testsprite test get ${opts.testId}' (or 'test code get') and retry with --expected-version ${serverVersion}.`, + ); + } else { + stderr( + `Code conflict on ${opts.testId}. Re-fetch the current codeVersion and retry with --expected-version .`, + ); + } + } + throw err; + } +} + +function renderCodePutText(response: CliPutTestCodeResponse): string { + return [ + `testId ${response.testId}`, + `codeVersion ${response.codeVersion}`, + `updatedAt ${response.updatedAt}`, + ].join('\n'); +} + +interface StepsOptions extends CommonOptions { + testId: string; + pageSize?: number; + startingToken?: string; + maxItems?: number; + /** + * When set, fetch per-run steps from the authoritative run-scoped endpoint + * `GET /runs/{runId}?includeSteps=true` instead of client-filtering the + * cumulative `/tests/{id}/steps` response. + * + * Background: FE Portal step rows in `FrontendTestStepEntity` don't reliably + * carry per-run `runIdIfAvailable`, so client-side filtering of the cumulative + * list returns an empty result for every runId (even completed runs). The + * run-scoped endpoint reads from `TestRunStepEntity` directly and returns + * only the steps for that specific run. + * + * Pagination flags (`--page-size`, `--starting-token`, `--max-items`) are + * ignored when `--run-id` is supplied — the run-scoped endpoint returns all + * steps in a single response. + */ + runId?: string; +} + +/** + * Map a `RunStepDto` (from `GET /runs/{runId}?includeSteps=true`) to a + * `CliTestStep` so both the run-scoped and cumulative paths share the same + * renderer (`renderStepsText`). + * + * Fields that don't exist on `RunStepDto`: + * - `testId` — taken from the `RunResponse` + * - `runIdIfAvailable` — set to the `runId` we queried + * - `codeVersion` — taken from `RunResponse.codeVersion` + * - `capturedAt` — closest available = `RunStepDto.createdAt` + * - `updatedAt` — same as `capturedAt` (no separate updatedAt for run-scoped steps) + * - `outcomeContributesToFailure` — derived: true when the step's numeric index + * matches `RunResponse.failedStepIndex` + */ +function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestStep { + const numericIndex = parseInt(step.stepIndex, 10); + return { + testId: run.testId, + stepIndex: numericIndex, + action: step.action, + description: step.description ?? '', + status: step.status, + screenshotUrl: step.screenshotUrl, + htmlSnapshotUrl: step.htmlSnapshotUrl, + runIdIfAvailable: run.runId, + codeVersion: run.codeVersion ?? null, + capturedAt: step.createdAt, + updatedAt: step.createdAt, + // `null` = unclassified (the run has no known failed step); otherwise a + // concrete boolean — `true` for the failing step, `false` for the known + // non-contributors. (Per the CliTestStep contract: null ≠ false.) + outcomeContributesToFailure: + run.failedStepIndex === null ? null : numericIndex === run.failedStepIndex, + }; +} + +export async function runSteps( + opts: StepsOptions, + deps: TestDeps = {}, +): Promise> { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // When --run-id is supplied, use the authoritative run-scoped endpoint + // `GET /runs/{runId}?includeSteps=true` instead of client-filtering the + // cumulative `/tests/{id}/steps` response. The cumulative FE Portal step + // rows don't reliably carry per-run `runIdIfAvailable`, so the old + // client-side filter always returned empty — even for completed runs. + if (opts.runId !== undefined) { + // 404 → NOT_FOUND (exit 4) — unknown or cross-tenant runId. + // All other errors propagate unchanged (auth, transport, etc.). + const run = await client.getRun(opts.runId, { includeSteps: true }); + + // Restore the implicit test-scoping the old `/tests/{testId}/steps` path + // had: a runId belonging to a DIFFERENT test (same tenant) must not leak + // that other test's steps under `test steps --run-id `. + // Skipped under --dry-run — the canned sample's testId is fixed, not the + // caller's argument, so a real comparison would always (falsely) mismatch. + if (!opts.dryRun && run.testId !== opts.testId) { + throw ApiError.fromEnvelope( + { + code: 'NOT_FOUND', + message: `Run ${opts.runId} does not belong to test ${opts.testId} (it belongs to ${run.testId}). Use 'testsprite test steps ${run.testId} --run-id ${opts.runId}' or drop --run-id.`, + }, + 404, + ); + } + + const rawSteps = run.steps ?? []; + const items: CliTestStep[] = rawSteps.map(s => mapRunStepToCliTestStep(s, run)); + const page: Page = { items, nextToken: null }; + + if (items.length === 0) { + // The run exists but has no recorded step rows yet (e.g. a fast + // code-replay run that finished before any steps were written). + if (opts.output === 'json') { + out.print(page, data => renderStepsText(data as Page)); + } else { + stderrFn( + `[advisory] No step records found for run ${opts.runId}. ` + + `The run may have completed before steps were written, or this run type does not record per-step data. ` + + `For the full failure bundle use: testsprite test artifact get ${opts.runId}`, + ); + } + return page; + } + + out.print(page, data => renderStepsText(data as Page)); + return page; + } + + // Bare `test steps ` (no --run-id): cumulative path — byte-identical to + // the original behavior. Pagination flags are honored here. + const paginationFlags: PaginationFlags = validatePaginationFlags({ + pageSize: opts.pageSize, + startingToken: opts.startingToken, + maxItems: opts.maxItems, + }); + + const useSinglePage = opts.pageSize !== undefined && opts.maxItems === undefined; + const path = `/tests/${encodeURIComponent(opts.testId)}/steps`; + + let page: Page; + if (useSinglePage) { + page = await fetchSinglePage( + client, + path, + paginationFlags.pageSize!, + opts.startingToken, + ); + } else { + page = await paginate( + async ({ pageSize, cursor }) => + client.get>(path, { query: { pageSize, cursor } }), + paginationFlags, + ); + } + + // Bare cumulative path: when the returned items span multiple runIds, + // print a stderr advisory pointing at --run-id so the next invocation + // can scope. Stdout is unchanged (still the full §6.4 wire shape), so + // JSON consumers keep working. + const distinctRunIds = new Set( + page.items.map(s => s.runIdIfAvailable).filter((v): v is string => v !== null), + ); + if (distinctRunIds.size > 1) { + stderrFn( + `[advisory] returned ${page.items.length} steps span ${distinctRunIds.size} distinct runs. ` + + `Pass --run-id to scope to a single run.`, + ); + } + + out.print(page, data => renderStepsText(data as Page)); + return page; +} + +interface ResultOptions extends CommonOptions { + testId: string; + /** + * §6.5.1 (M2.1 piece 3) — when set, the CLI requests the inline + * `analysis` block from the facade and renders it under the result + * summary in text mode. JSON mode prints the wire envelope as + * received (the `analysis` block lives under the `analysis` key). + * Optional with default `false` so pre-M2.1 callers don't have to + * thread the flag through every call site. + */ + includeAnalysis?: boolean; +} + +export async function runResult( + opts: ResultOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const path = `/tests/${encodeURIComponent(opts.testId)}/result`; + const result = await client.get( + path, + opts.includeAnalysis === true ? { query: { includeAnalysis: true } } : undefined, + ); + + // D1 — emit a single advisory to stderr when the backend signals that the + // target URL could not be resolved (source is 'unresolved' or a null that + // was explicitly sent). Text mode only; JSON mode passes the field through. + if ( + opts.output !== 'json' && + (result.targetUrlSource === 'unresolved' || result.targetUrlSource === null) && + result.targetUrl === null + ) { + stderrFn( + '[advisory] target URL unresolved for this run (the stored run row had no target URL); not falling back to the project default.', + ); + } + + // L141 — in JSON mode, annotate the analysis block with truncation + // indicators so programmatic consumers know when the backend cut the + // text. Text mode is unchanged: the renderer already shows the raw + // (possibly truncated) string and adding a `…` hint is redundant. + const printData: CliLatestResult = + opts.output === 'json' && result.analysis !== undefined + ? { ...result, analysis: annotateAnalysisTruncation(result.analysis) } + : result; + + out.print(printData, data => renderResultText(data as CliLatestResult)); + return result; +} + +// --------------------------------------------------------------------------- +// M3.4 piece-5 — `test result --history` (run-history list) +// --------------------------------------------------------------------------- + +/** + * Parse a duration string (`24h`, `7d`) or ISO timestamp to an absolute + * ISO string for use as the `?since=` query parameter. + * + * - `24h` → `now - 24 hours` + * - `7d` → `now - 7 days` + * - Any other value is returned verbatim (assumed to be an ISO timestamp + * or epoch ms string that the server will validate). + * + * Translation is done client-side per piece-5 design decision #3. + */ +export function parseDuration(raw: string, now: Date = new Date()): string { + const hourMatch = /^(\d+)h$/i.exec(raw); + if (hourMatch) { + const hours = Number(hourMatch[1]); + return new Date(now.getTime() - hours * 60 * 60 * 1000).toISOString(); + } + const dayMatch = /^(\d+)d$/i.exec(raw); + if (dayMatch) { + const days = Number(dayMatch[1]); + return new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + } + // Pass-through: ISO timestamp or epoch value — server validates. + return raw; +} + +interface ResultHistoryOptions extends CommonOptions { + testId: string; + /** Filter by trigger source. */ + source?: RunSource; + /** + * Lower bound for `createdAt`. Accepts `24h`, `7d`, or an ISO timestamp. + * Translated client-side to an absolute ISO string before the request. + */ + since?: string; + /** Page size 1–100 (default 20). */ + pageSize?: number; + /** Opaque cursor from a prior page's `nextCursor`. */ + cursor?: string; +} + +/** + * `test result --history [options]` + * + * List a test's prior runs (M3.4 piece-5). Complements the M2 `test result` + * "latest result" mode (unchanged). Branches on `--history` inside the shared + * action handler — the function exposed here is the `--history` branch only; + * bare `test result ` continues to call `runResult`. + */ +export async function runResultHistory( + opts: ResultHistoryOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + + // Validate page size BEFORE makeClient: local validation must win over + // AUTH_REQUIRED so `--page-size 0` exits 5 even with no credentials + // configured (codex round-2), matching validatePaginationFlags ordering + // in `test list` / `project list`. + if (opts.pageSize !== undefined) { + if (!Number.isFinite(opts.pageSize) || opts.pageSize < 1 || opts.pageSize > 100) { + throw localValidationError('page-size', 'must be between 1 and 100'); + } + } + + const client = makeClient(opts, deps); + const pageSize = opts.pageSize ?? 20; + const sinceIso = opts.since !== undefined ? parseDuration(opts.since) : undefined; + + const resp = await client.listTestRuns(opts.testId, { + cursor: opts.cursor, + pageSize, + source: opts.source, + since: sinceIso, + }); + + if (opts.output === 'json') { + out.print({ runs: resp.runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); + return resp; + } + + // Text mode rendering + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // Empty / pre-cutover: print the backend meta.note instead of a blank table. + // EXCEPTION: if nextCursor is non-null this page is empty only because the + // backend filters AFTER limiting rows (limit-before-filter). Matching runs + // may exist on later pages — surface the cursor instead of reporting + // "no history". + if (resp.runs.length === 0) { + if (resp.nextCursor !== null) { + // Filtered-empty page, but more pages exist: prompt user to paginate. + const msg = + `No matching runs on this page (filters skipped all entries), but more history exists.\n` + + `Continue with: --cursor ${resp.nextCursor}`; + out.print(msg, d => d as string); + if (resp.meta.portalUrl) { + stderr(` Portal: ${resp.meta.portalUrl}`); + } + return resp; + } + // Truly empty (nextCursor is null): pre-cutover or genuinely no history. + const note = + resp.meta.note ?? + 'No CLI-tracked history for this test. History is recorded from 2026-05-14 onward.'; + out.print(note, d => d as string); + if (resp.meta.portalUrl) { + stderr(` Portal: ${resp.meta.portalUrl}`); + } + return resp; + } + + const lines: string[] = []; + lines.push(renderRunHistoryTable(resp.runs)); + + // Footer: pointer to per-run detail commands. + lines.push(''); + lines.push('Per-run detail: testsprite test wait '); + lines.push('Failure bundle: testsprite test artifact get '); + + // Pagination hint + if (resp.nextCursor !== null) { + lines.push(''); + lines.push(`Next page: --cursor ${resp.nextCursor}`); + } + + out.print(lines.join('\n'), d => d as string); + + // Short-filtered-page hint: non-null nextCursor even though this page was + // shorter than requested — "none in THIS window" does not mean end-of-history. + if (resp.nextCursor !== null && resp.runs.length < pageSize) { + stderr( + `[hint] Fewer than ${pageSize} rows returned but more may exist — ` + + `source filter skipped some entries. Pass --cursor ${resp.nextCursor} to continue.`, + ); + } + + return resp; +} + +const RUN_HISTORY_TABLE_COL_WIDTHS = { + runId: 36, + status: 10, + source: 18, + rerun: 6, + when: 25, +}; + +/** + * Max width of the `test steps` DESCRIPTION column in text mode. Long / + * multi-line step descriptions are collapsed to one line and truncated to + * this many chars (with an ellipsis) so a single blob can't blow the table + * out (dogfood 2026-06-04). `--output json` carries the full text. + */ +const DESC_COL_MAX = 60; + +/** Max chars to show in the TARGETURL sub-line (excess truncated with …). */ +const HISTORY_TARGET_URL_MAX = 80; + +/** + * Render a compact table of `RunHistoryItem[]` rows, newest-first. + * + * Columns: RUN ID · STATUS · SOURCE · RERUN? · WHEN · DURATION + * `RERUN?` is derived from `isRerun`. + * `WHEN` is the `createdAt` ISO string. + * `DURATION` is wall-clock `finishedAt − (startedAt ?? createdAt)`. The + * `createdAt` fallback keeps the column populated for FE runs, which do + * not record `startedAt` today (dogfood 2026-06-04). + * + * G1b: when `targetUrl` is present and `targetUrlSource` is not + * `'unresolved'`, a sub-line ` targetUrl: ` is printed below each + * run row (truncated to `HISTORY_TARGET_URL_MAX` chars). The table columns + * are left intact to avoid width blow-out on terminals. + */ +function renderRunHistoryTable(runs: RunHistoryItem[]): string { + const cols = RUN_HISTORY_TABLE_COL_WIDTHS; + const header = [ + padEnd('RUN ID', cols.runId), + padEnd('STATUS', cols.status), + padEnd('SOURCE', cols.source), + padEnd('RERUN?', cols.rerun), + padEnd('WHEN', cols.when), + 'DURATION', + ].join(' '); + const sep = '-'.repeat(header.length); + + const rows = runs.flatMap(r => { + // FE runs never populate `startedAt` today — the RUNNING heartbeat + // that would set it doesn't fire on the legacy/sync execution path + // (dogfood 2026-06-04), so without a fallback DURATION was always + // "—" for every FE run. Fall back to `createdAt` so the column shows + // wall-clock from trigger to finish; on sync dev the queue gap is + // ~0, and `--output json` still exposes raw startedAt/finishedAt for + // consumers that need to exclude queue time. + const duration = formatDurationMs(r.startedAt ?? r.createdAt, r.finishedAt); + const mainRow = [ + padEnd(r.runId, cols.runId), + padEnd(r.status, cols.status), + padEnd(r.source, cols.source), + padEnd(r.isRerun ? 'yes' : 'no', cols.rerun), + padEnd(r.createdAt, cols.when), + duration, + ].join(' '); + + // G1b: surface per-run targetUrl as an indented sub-line. + // Render only when truthy (skip null, undefined, empty) and when the + // source is not 'unresolved' (that would mean "backend couldn't resolve + // a URL" — printing "—" is less informative than omitting the line). + const lines: string[] = [mainRow]; + if (r.targetUrl && r.targetUrlSource !== 'unresolved') { + const url = + r.targetUrl.length > HISTORY_TARGET_URL_MAX + ? `${r.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` + : r.targetUrl; + lines.push(` targetUrl: ${url}`); + } else if (r.targetUrlSource === 'unresolved') { + lines.push(` targetUrl: —`); + } + + return lines; + }); + + return [header, sep, ...rows].join('\n'); +} + +function padEnd(s: string, width: number): string { + if (s.length >= width) return s; + return s + ' '.repeat(width - s.length); +} + +function formatDurationMs(startedAt: string | null, finishedAt: string | null): string { + if (!startedAt || !finishedAt) return '—'; + const ms = new Date(finishedAt).getTime() - new Date(startedAt).getTime(); + if (!Number.isFinite(ms) || ms < 0) return '—'; + const totalSec = Math.round(ms / 1000); + const minutes = Math.floor(totalSec / 60); + const seconds = totalSec % 60; + return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`; +} + +interface FailureSummaryOptions extends CommonOptions { + testId: string; +} + +/** + * `test failure summary ` — M2.1 piece 3. + * + * Sibling of `test failure get`. Returns one-screen failure triage + * info (status, failureKind, root-cause hypothesis, suggested fix + * target if the analysis pipeline produced one) without downloading + * video, screenshots, or DOM snapshots. The command an agent should + * reach for first when investigating a reported failure. + * + * 404 NOT_FOUND propagates as exit 4. The facade's `details.reason` + * (`not_found` / `no_failing_run`) reaches the user via the + * `nextAction` template. + */ +export async function runFailureSummary( + opts: FailureSummaryOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + const summary = await client.get( + `/tests/${encodeURIComponent(opts.testId)}/failure/summary`, + ); + out.print(summary, data => renderFailureSummaryText(data as CliFailureSummary)); + return summary; +} + +interface FailureGetOptions extends CommonOptions { + testId: string; + /** + * Directory to write the §7 disk layout into. When unset, the CLI + * prints the wire envelope (`--output json`) or a human summary + * (`--output text`) to stdout — useful for an agent piping straight + * to a vision-aware LLM. When set, stdout is silent on success + * (the on-disk artifact is the contract). + */ + out?: string; + /** §7.4 — keep only the failed step ± 1 in `steps[]` and `evidence[]`. */ + failedOnly: boolean; +} + +export interface FailureGetResult { + /** The wire envelope as returned by the facade. */ + context: CliFailureContext; + /** Set when `--out` was used; otherwise undefined. */ + bundle?: WriteBundleResult; +} + +/** + * `test failure get` — the agent-facing entry point. Fetches the §6.7 + * `FailureContext` for `` and either writes the §7 disk + * layout (when `--out` is set) or prints the wire envelope to stdout + * (default). + * + * Without `--out`: + * - `--output json` (the agent default) — full wire envelope on + * stdout. Presigned URLs left intact for the agent to fetch on + * its own. + * - `--output text` — human summary block (status, failureKind, + * failedStepIndex, hypothesis, fix target, evidence count). + * + * With `--out `: + * - Atomic write under `/.tmp/...` → `rename()`. `meta.json` + * renames last; its presence implies "bundle complete and + * self-consistent." On any failure, `/.partial` is written + * and the CLI exits non-zero with the underlying error code. + * - stdout prints one line per output mode after the bundle is + * written, matching the rest of M2's `--out` ergonomics. + * + * 404 NOT_FOUND propagates as exit 4. The facade's `details.reason` + * (`not_found` / `no_failing_run` / `no_code`) reaches the user via + * the `nextAction` template — the CLI doesn't re-derive its own + * remediation text per §5.4. + */ +export async function runFailureGet( + opts: FailureGetOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + // We resolve the output dir BEFORE the network call so a missing / + // empty path surfaces as VALIDATION_ERROR (exit 5) without spending + // an API call. `writeBundle` re-validates internally; this is the + // fast-fail. + const requestedDir = opts.out; + + const context = await client.get( + `/tests/${encodeURIComponent(opts.testId)}/failure`, + ); + + // Run the §3 atomicity invariants on every path — even when --out is + // absent. An agent piping the JSON envelope into a vision-LLM + // consumer would otherwise be handed stitched data the contract + // guarantees never reaches it. The bundle writer re-runs the check + // internally; this call is the cheap upfront trap. + assertContextIntegrity(context, 'local'); + + if (requestedDir !== undefined) { + // Dry-run: do NOT call writeBundle (which would mkdir, fetch + // presigned URLs, and write files). Print the would-be bundle layout + // to stderr and emit the wire envelope to stdout so the agent sees + // the shape it would parse from disk. + if (opts.dryRun) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const fileNames = plannedBundleFiles(context, opts.failedOnly); + stderr( + `[dry-run] would write bundle to ${requestedDir} (${fileNames.length} files; meta.json renames last)`, + ); + for (const f of fileNames) stderr(`[dry-run] ${f}`); + if (opts.output === 'json') { + out.print({ ok: true, dir: requestedDir, dryRun: true, context }); + } else { + // Use a dry-run-specific renderer: the real success renderer + // says "Bundle written to ..." which would be a lie here. Stdout + // is the success contract automation may parse, so it must not + // imply the bundle was created. + out.print( + { dir: requestedDir, files: fileNames.length, snapshotId: context.snapshotId }, + data => + renderBundleDryRunText(data as { dir: string; files: number; snapshotId: string }), + ); + } + return { context }; + } + + const bundle = await writeBundle(context, { + dir: requestedDir, + failedOnly: opts.failedOnly, + fetchImpl: deps.fetchImpl, + }); + if (opts.output === 'json') { + out.print({ ok: true, dir: bundle.dir, meta: bundle.meta, files: bundle.files }); + } else { + out.print( + { dir: bundle.dir, files: bundle.files.length, snapshotId: bundle.meta.snapshotId }, + data => renderBundleWrittenText(data as { dir: string; files: number; snapshotId: string }), + ); + } + return { context, bundle }; + } + + // --output json (no --out) — print the wire envelope verbatim. This + // is the agent-piping path: the agent's vision-LLM consumer will + // dereference presigned URLs itself. + if (opts.output === 'json') { + out.print(context); + } else { + out.print(context, data => renderFailureContextText(data as CliFailureContext)); + } + return { context }; +} + +// --------------------------------------------------------------------------- +// M3.3 piece-3 — `test run` / `test wait` + create --run chain +// --------------------------------------------------------------------------- + +/** + * Default timeout in seconds for `--wait`. Range 1..3600. + */ +const DEFAULT_RUN_TIMEOUT_SECONDS = 600; +const MAX_RUN_TIMEOUT_SECONDS = 3600; + +interface RunTestRunOptions extends CommonOptions { + testId: string; + targetUrl?: string; + wait: boolean; + timeoutSeconds: number; + /** + * B2(c): true when --timeout was not explicitly set by the user (the default + * is in effect). Used to decide whether to emit the first-run hint. + * Defaults to false (no hint) when not set; only the `test run` / `test + * create --run` command wiring sets this to `cmdOpts.timeout === undefined`. + */ + timeoutIsDefault?: boolean; + idempotencyKey?: string; + /** + * Per codex round-1 P1: when chained from `test create --run`, the caller + * passes the create response here so `runTestRun` can emit a single merged + * envelope `{ ...createContext, run: }` on stdout. Without + * this, `test create --run --output json` produces two JSON objects back-to- + * back and agents cannot `JSON.parse` the result. + */ + createContext?: CliCreateTestResponse | CliCreateFromPlanResponse; + /** + * Optional type hint supplied by the caller when the type is already known + * client-side (e.g. the `test create --run` chain knows `opts.type`). + * Used to derive `isBackend` for the step-summary renderer BEFORE the + * `beFallbackUsed` fallback fires, so fast backend runs that are terminal + * on the first poll still render `steps: n/a (backend)` correctly. + * Leave unset for `test run ` where the type is only discoverable via + * the `resolveAlternate` probe (an extra round-trip we deliberately avoid). + */ + type?: 'frontend' | 'backend'; +} + +interface RunTestWaitOptions extends CommonOptions { + runId: string; + timeoutSeconds: number; +} + +// --------------------------------------------------------------------------- +// M3.4 piece-3 — `test rerun` options +// --------------------------------------------------------------------------- + +interface RunTestRerunOptions extends CommonOptions { + /** One or more testIds to rerun. Empty + all=false → validation error (exit 5). */ + testIds: string[]; + /** --all: resolve all tests in the project and batch-rerun them. */ + all: boolean; + /** --project: used with --all to resolve the project's tests. */ + projectId?: string; + /** --wait: block until terminal (or --timeout). */ + wait: boolean; + /** Polling / overall deadline. Default 600, max 3600. */ + timeoutSeconds: number; + /** + * Auto-heal: defaults true for FE reruns (use --no-auto-heal to opt out). + * Backend ignores the server-side tier gate for CLI callers — Free + paid + * both get auto-heal; charged 0.2 credits per engage when Phase-2 runs. + */ + autoHeal: boolean; + /** + * Whether the user explicitly requested a specific auto-heal state via a + * flag (as opposed to the default-on value). Used to suppress the BE-test + * "ignoring auto-heal" warning when the value was never explicitly set. + * + * With `--no-auto-heal` as the only flag (no `--auto-heal`), this is always + * false — the default-on value is never an explicit user request, so the BE + * warning is suppressed on every default-on rerun. Only future addition of + * an explicit `--auto-heal` flag would set this to true. + */ + autoHealExplicit: boolean; + /** --skip-dependencies: BE only. Don't expand the producer/teardown closure. */ + skipDependencies: boolean; + /** --max-concurrency: bounds the --wait poll fan-out (batch / BE closure). */ + maxConcurrency: number; + /** --idempotency-key: caller-supplied; auto-minted UUID when absent. */ + idempotencyKey?: string; + /** + * --skip-terminal: with --all, exclude tests already in a terminal status + * (passed|failed|blocked|cancelled) before dispatch so an interrupted sweep + * doesn't re-replay finished tests. + */ + skipTerminal?: boolean; + /** + * --status : comma-separated list of public status values. With --all, + * only tests whose status matches one of the listed values are dispatched. + * Reuses the same validated set as `test list --status`. + */ + statusFilter?: string; + /** + * --filter : with --all, only rerun tests whose name contains this + * substring (case-insensitive). Applied after --skip-terminal and --status + * filters. Client-side only. + */ + nameFilter?: string; +} + +/** + * Map a terminal `RunResponse.status` to the CLI exit code. + * `passed` → 0; everything else → 1. + */ +function exitCodeForRunStatus(status: string): number { + return status === 'passed' ? 0 : 1; +} + +// --------------------------------------------------------------------------- +// Backend-test wait fallback (dogfood L1888) +// +// Backend-test run-surface rows are written `queued` then orphaned server-side +// (`RunHistoryService.finalizeRun` is wired on FE terminal paths only), so a +// run-row poll always hits `--timeout` → exit 7 even when the BE test PASSES. +// The verdict DOES reach the test record, readable via `GET /tests/{id}/result`. +// These helpers let `test run --wait` / `test wait` fall back to that +// testId-scoped verdict for backend tests, so a passing BE test exits 0. +// Frontend tests are untouched (their run row finalizes normally). +// --------------------------------------------------------------------------- + +/** Terminal subset of {@link CliPublicStatus} (mirrors TERMINAL_RUN_STATUSES). */ +const TERMINAL_PUBLIC_STATUSES: ReadonlySet = new Set([ + 'passed', + 'failed', + 'blocked', + 'cancelled', +]); + +function isTerminalPublicStatus(status: CliPublicStatus): boolean { + return TERMINAL_PUBLIC_STATUSES.has(status); +} + +/** Minimal client surface the backend-test wait fallback needs. */ +interface ResultReadClient { + get(path: string, options?: { signal?: AbortSignal }): Promise; +} + +/** + * Overlay a terminal testId-scoped {@link CliLatestResult} onto the polled + * (non-terminal) {@link RunResponse} so a backend test whose run-surface row + * never finalizes still renders a complete, correctly-correlated run envelope. + * The real correlation metadata (`runId`, `testId`, `projectId`, `userId`, + * `source`, `createdAt`, `createdFrom`) is preserved from the polled run row; + * only the verdict/result fields are taken from the test record (codex + * round-2: don't fabricate blank correlation fields). + */ +function backendResultToRunResponse(result: CliLatestResult, run: RunResponse): RunResponse { + const total = result.summary.passed + result.summary.failed + result.summary.skipped; + return { + ...run, + status: result.status as RunStatus, + // Drop the polling hint — it's meaningless on a terminal response + // (JSON.stringify omits undefined keys). + retryAfterSeconds: undefined, + startedAt: result.startedAt ?? run.startedAt, + finishedAt: result.finishedAt ?? run.finishedAt, + codeVersion: run.codeVersion || (result.codeVersion ?? ''), + targetUrl: run.targetUrl || (result.targetUrl ?? ''), + // createdFrom / projectId / userId / runId / testId / source / createdAt + // inherited from the polled run row via the spread above. + failedStepIndex: result.failedStepIndex, + failureKind: result.failureKind, + videoUrl: result.videoUrl, + stepSummary: { + total, + completed: result.summary.passed + result.summary.failed, + passedCount: result.summary.passed, + failedCount: result.summary.failed, + }, + }; +} + +/** + * Decide whether a testId-scoped result belongs to THIS run (vs a stale + * verdict from a prior run of the same test). Backend serializes runs per + * testId, so the next terminal verdict after our trigger is ours — but a + * just-finished prior run could still be showing. Gate: + * - result names our runId → accept (strongest signal); + * - result names a different runId → reject (a different run); + * - result has no runId (legacy) → accept iff finishedAt >= notBefore. + */ +export function backendResultIsForThisRun( + result: CliLatestResult, + runId: string, + notBefore: string | undefined, +): boolean { + if (!isTerminalPublicStatus(result.status)) return false; + if (result.runIdIfAvailable) { + return result.runIdIfAvailable === runId; + } + if (!result.finishedAt) return false; + if (!notBefore) return true; + const finished = Date.parse(result.finishedAt); + const floor = Date.parse(notBefore); + if (Number.isNaN(finished) || Number.isNaN(floor)) return true; + return finished >= floor; +} + +/** + * Build a `resolveAlternate` callback for `pollRunUntilTerminal` that falls + * back to the testId-scoped verdict for **backend** tests (dogfood L1888). + * + * - The first non-terminal run tick does a one-time `GET /tests/{id}` to learn + * the test type (cached). Frontend tests → no-op forever, so the FE path is + * byte-identical to before. + * - For backend tests, each later non-terminal tick reads + * `GET /tests/{id}/result`; once that record is terminal AND belongs to this + * run, a synthesized terminal `RunResponse` resolves the wait (exit 0/1) + * instead of timing out (exit 7). + * - Every lookup is best-effort: any error → "keep polling the run row", so + * the fallback can never make either path worse than the prior timeout. + */ +function makeBackendWaitFallback(args: { + client: ResultReadClient; + resolveTestId: (run: RunResponse) => string; + resolveNotBefore: (run: RunResponse) => string | undefined; + onResolved?: (testId: string, status: CliPublicStatus) => void; +}): (run: RunResponse, elapsedMs: number, signal: AbortSignal) => Promise { + let detection: 'pending' | 'frontend' | 'backend' = 'pending'; + return async ( + run: RunResponse, + _elapsedMs: number, + signal: AbortSignal, + ): Promise => { + const testId = args.resolveTestId(run); + if (!testId) return null; + if (detection === 'pending') { + try { + const test = await args.client.get(`/tests/${encodeURIComponent(testId)}`, { + signal, + }); + // Cache ONLY a successful read. A transient probe failure (5xx, + // rate-limit, network blip) must NOT permanently mark a backend test + // as frontend — that would silently re-break the timeout this fallback + // exists to fix (codex round-1). Leave `detection` pending so the next + // non-terminal tick retries; the FE path is unaffected because its run + // row finalizes and `resolveAlternate` stops being called. + detection = test.type === 'backend' ? 'backend' : 'frontend'; + } catch { + return null; // transient — keep polling the run row, retry the probe next tick. + } + } + if (detection !== 'backend') return null; + let result: CliLatestResult; + try { + result = await args.client.get( + `/tests/${encodeURIComponent(testId)}/result`, + { signal }, + ); + } catch { + return null; // not-ready / transient — keep polling. + } + if (!backendResultIsForThisRun(result, run.runId, args.resolveNotBefore(run))) { + return null; + } + args.onResolved?.(testId, result.status); + // Overlay the verdict onto the polled run row (preserves real correlation + // metadata; codex round-2). + return backendResultToRunResponse(result, run); + }; +} + +/** + * Print the trigger/run response, merging the create context when the + * caller is the `test create --run` chain. Per codex round-1 P1: chained + * `--output json` must produce ONE parseable JSON object on stdout, not + * two back-to-back envelopes. With `createContext` set, JSON mode emits + * `{ ...createContext, run: }`; text mode prints the create + * summary first, then the run. + */ +function printRunOrChain( + out: Output, + payload: T, + createContext: CliCreateTestResponse | CliCreateFromPlanResponse | undefined, + textRenderer?: (data: unknown) => string, +): void { + if (!createContext) { + out.print(payload, textRenderer); + return; + } + const merged = { ...createContext, run: payload }; + out.print(merged, () => { + // Text mode of the chain: render the create envelope as a header, + // a blank line, then the run envelope below. Stays readable for + // operators while JSON mode owns the parseable contract. + const createText = renderCreateText(createContext as CliCreateTestResponse); + const runText = textRenderer ? textRenderer(payload) : JSON.stringify(payload, null, 2); + return `${createText}\n\n${runText}`; + }); +} + +/** + * Enrich a terminal RunResponse with a client-synthesized Portal deep link. + * Emitted only when the wire row carries both projectId and testId (the BE + * testId-fallback path synthesizes rows with an empty projectId — those stay + * unenriched) and the API endpoint maps to a known portal host + * (`resolvePortalUrl` returns undefined otherwise). + */ +function withRunDashboardUrl(run: RunResponse, apiUrl: string): RunResponse { + if (!run.projectId || !run.testId) return run; + const dashboardUrl = resolvePortalUrl(apiUrl, run.projectId, run.testId); + return dashboardUrl !== undefined ? { ...run, dashboardUrl } : run; +} + +/** + * Render a `RunResponse` to human-readable text. JSON mode callers get + * the wire envelope via `out.print`. + * + * Pass `isBackend: true` to suppress the per-step breakdown (BE tests are + * atomic — no per-step breakdown exists; showing `0/0 (passed=0, failed=0)` + * reads like a no-op rather than a passing/failing atomic result). + */ +function renderRunResponseText( + run: RunResponse, + { isBackend = false }: { isBackend?: boolean } = {}, +): string { + // P2-9: omit null fields (codeVersion/targetUrl) to match renderResultText + // convention — literal "null" in text output confuses human operators. + const lines: string[] = [ + `runId ${run.runId}`, + `testId ${run.testId}`, + `status ${run.status}`, + ]; + if (run.codeVersion !== null) lines.push(`codeVersion ${run.codeVersion}`); + if (run.targetUrl !== null) lines.push(`targetUrl ${run.targetUrl}`); + lines.push(`createdAt ${run.createdAt}`); + if (run.startedAt) lines.push(`startedAt ${run.startedAt}`); + if (run.finishedAt) lines.push(`finishedAt ${run.finishedAt}`); + if (run.failureKind) lines.push(`failureKind ${run.failureKind}`); + // D5-UX: show the human error string when status is failed/blocked and + // the backend provided it. Truncate long multi-line errors to the first + // line (≤200 chars) so the text output stays readable. JSON mode is + // unaffected — it ships the wire envelope verbatim via out.print. + if ( + (run.status === 'failed' || run.status === 'blocked') && + run.error && + run.error.trim().length > 0 + ) { + const firstLine = run.error.split('\n')[0] ?? run.error; + const truncated = firstLine.length > 200 ? `${firstLine.slice(0, 200)}…` : firstLine; + lines.push(`error ${truncated}`); + } + if (isBackend) { + // BE tests are atomic — no per-step breakdown. Show n/a instead of + // confusing "0/0 (passed=0, failed=0)" which reads like a broken no-op. + lines.push(`steps n/a (backend)`); + } else if (run.stepSummary) { + lines.push( + `steps ${run.stepSummary.completed}/${run.stepSummary.total} (passed=${run.stepSummary.passedCount}, failed=${run.stepSummary.failedCount})`, + ); + } + // Closing pointer: where to inspect this run in the Portal (video, steps, + // screenshots). Present only when withRunDashboardUrl could resolve it. + if (run.dashboardUrl) lines.push(`dashboard ${run.dashboardUrl}`); + return lines.join('\n'); +} + +/** + * Render a `TriggerRunResponse` (no-wait path) to human-readable text. + */ +function renderTriggerRunText(r: TriggerRunResponse): string { + // P2-9: omit null-valued fields to avoid printing literal "null". + const lines: string[] = [ + `runId ${r.runId}`, + `status ${r.status}`, + `enqueuedAt ${r.enqueuedAt}`, + ]; + if (r.codeVersion !== null) lines.push(`codeVersion ${r.codeVersion}`); + if (r.targetUrl !== null) lines.push(`targetUrl ${r.targetUrl}`); + return lines.join('\n'); +} + +/** + * Validate the `--timeout` flag value. Returns a clamped integer in + * range [1, 3600], or the default when absent. + */ +function parseTimeoutFlag(raw: string | undefined, flagName: string): number { + if (raw === undefined) return DEFAULT_RUN_TIMEOUT_SECONDS; + const n = Number(raw); + if (!Number.isFinite(n) || !Number.isInteger(n) || n < 1) { + throw localValidationError( + flagName, + `must be an integer between 1 and ${MAX_RUN_TIMEOUT_SECONDS}`, + ); + } + if (n > MAX_RUN_TIMEOUT_SECONDS) { + throw localValidationError(flagName, `must be at most ${MAX_RUN_TIMEOUT_SECONDS} seconds`); + } + return n; +} + +/** + * `test run ` — M3.3 piece-3. + * + * Triggers a run via `POST /api/cli/v1/tests/{testId}/runs`. With + * `--wait`, polls until terminal status (via `pollRunUntilTerminal`). + * Exit code is 0 on `passed`, 1 on `failed`/`blocked`/`cancelled`, + * 7 on timeout. + */ +export async function runTestRun( + opts: RunTestRunOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + if (opts.targetUrl !== undefined) { + assertNotLocal(opts.targetUrl); + } + + if (opts.dryRun) { + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const idempotencyKey = opts.idempotencyKey ?? `dry-run-${randomUUID()}`; + // P3-14: use the dry-run sample (TriggerRunResponse shape) so `test run + // --dry-run --output json` returns the same shape as a real trigger + // response, matching `test rerun --dry-run` convention. Fall back to + // the HTTP-descriptor envelope only when no sample is registered. + const sampleBody = findSample('POST', `/api/cli/v1/tests/${opts.testId}/runs`)?.body(); + if (sampleBody !== undefined && !opts.wait) { + // Standalone `test run --dry-run` prints just the sample; a chained + // `test create --run --dry-run` routes through printRunOrChain so the + // merged { ...create, run } envelope keeps the created-test fields for + // JSON consumers (codex #128 P2-A). The --wait path falls through to the + // descriptor envelope below so its `thenPoll` hint is preserved. + printRunOrChain(out, sampleBody, opts.createContext, data => + renderTriggerRunText(data as TriggerRunResponse), + ); + void client; + return sampleBody as unknown as TriggerRunResponse; + } + const envelope = { + method: 'POST', + path: `/api/cli/v1/tests/${opts.testId}/runs`, + body: { source: 'cli' as const, ...(opts.targetUrl ? { targetUrl: opts.targetUrl } : {}) }, + idempotencyKey, + ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), + }; + printRunOrChain(out, envelope, opts.createContext); + // Still exercise the client factory so dry-run surfaces credential errors. + void client; + return envelope as unknown as TriggerRunResponse; + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-run-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + // D4: under --wait, raise the per-request timeout to cover --timeout so a + // slow trigger/long-poll under load isn't falsely cut at the 120s default. + const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + let triggerResponse: TriggerRunResponse; + let triggerRequestId: string | undefined; + let resumedFromConflict = false; + try { + const result = await client.triggerRunWithMeta( + opts.testId, + { source: 'cli', ...(opts.targetUrl ? { targetUrl: opts.targetUrl } : {}) }, + { idempotencyKey }, + ); + triggerResponse = result.body; + triggerRequestId = result.requestId; + } catch (err) { + // CONFLICT (409) can arise from two different causes: + // 1. reason === 'run_in_flight' — another run is currently executing for + // this test. Auto-resume polling is valid ONLY for this reason and ONLY + // when --wait is set. Any other reason (snapshot_in_flight, etc.) or + // IDEMPOTENCY_BODY_MISMATCH must propagate to exit 6 so callers can + // decide what to do. + // 2. reason !== 'run_in_flight' — snapshot mid-mutation or body-hash + // mismatch. Always propagate. + if (opts.wait && err instanceof ApiError && err.code === 'CONFLICT') { + const conflictReason = err.getDetail( + 'reason', + (v): v is string => typeof v === 'string' && v.length > 0, + ); + const currentRunId = err.getDetail( + 'currentRunId', + (v): v is string => typeof v === 'string' && v.length > 0, + ); + + // Only the genuine "another run currently executing" race qualifies for + // auto-resume. Other CONFLICT reasons (snapshot_in_flight, etc.) exit 6. + if (conflictReason === 'run_in_flight' && currentRunId !== undefined) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // If the caller supplied --target-url, verify the in-flight run targets + // the same URL. A mismatch means we would be reporting a different + // environment's results as if our requested environment was tested. + if (opts.targetUrl !== undefined) { + const inFlightRun = await client.getRun(currentRunId); + if (inFlightRun.targetUrl !== opts.targetUrl) { + throw new ApiError({ + code: 'CONFLICT', + message: + `Conflict: another run for this test is in flight against a different ` + + `target URL (${inFlightRun.targetUrl}). Your --target-url ${opts.targetUrl} ` + + `cannot attach to that run. Wait for it to finish ` + + `(\`testsprite test wait ${currentRunId}\`) or retry your trigger when ` + + `the test is free.`, + nextAction: `testsprite test wait ${currentRunId}`, + requestId: err.requestId, + details: { + reason: 'run_in_flight', + currentRunId, + inFlightTargetUrl: inFlightRun.targetUrl, + requestedTargetUrl: opts.targetUrl, + }, + }); + } + stderrFn( + `[advisory] Run already in flight (runId: ${currentRunId}, ` + + `target: ${inFlightRun.targetUrl}). ` + + `Attaching to that run's --wait poll instead of creating a new one.`, + ); + triggerResponse = { + runId: currentRunId, + status: 'queued', + enqueuedAt: new Date().toISOString(), + codeVersion: inFlightRun.codeVersion, + targetUrl: inFlightRun.targetUrl, + }; + } else { + // D: No --target-url supplied — fetch the in-flight run so the + // synthesised triggerResponse.targetUrl is the REAL environment being + // tested, not an empty string that would propagate into the timeout + // partial (Finding D). Fall back to null (not '') if the lookup fails. + let inFlightTargetUrl: string | null = null; + let inFlightCodeVersion = ''; + try { + const inFlightRun = await client.getRun(currentRunId); + inFlightTargetUrl = inFlightRun.targetUrl ?? null; + inFlightCodeVersion = inFlightRun.codeVersion ?? ''; + } catch { + // Best-effort — if the lookup fails, proceed with null targetUrl. + } + + // Auto-resume but emit a stronger advisory so the caller is aware + // they are attaching to the project default. + stderrFn( + `[advisory] Run already in flight (runId: ${currentRunId}` + + (inFlightTargetUrl ? `, target: ${inFlightTargetUrl}` : '') + + `). Auto-resuming wait on in-flight run. ` + + `If you needed a specific target URL, cancel with Ctrl-C and ` + + `re-trigger with --target-url.`, + ); + triggerResponse = { + runId: currentRunId, + status: 'queued', + enqueuedAt: new Date().toISOString(), + codeVersion: inFlightCodeVersion, + // Use the real targetUrl from the in-flight run (or null if unknown), + // never '' — the timeout partial inherits this value. + targetUrl: inFlightTargetUrl ?? '', + }; + } + resumedFromConflict = true; + } else { + throw err; + } + } else { + throw err; + } + } + + if (!opts.wait) { + printRunOrChain(out, triggerResponse, opts.createContext, data => + renderTriggerRunText(data as TriggerRunResponse), + ); + if (triggerRequestId && (opts.output === 'json' || opts.verbose || opts.debug)) + stderrFn(`requestId: ${triggerRequestId}`); + return triggerResponse; + } + + // --wait path: poll until terminal. + const startMs = Date.now(); + void resumedFromConflict; // used above; suppress unused-variable lint + const ticker = createTicker( + stderrFn, + opts.output === 'json' ? false : undefined, // disable ticker when --output json + ); + + // B2(c): emit a one-time hint when the user did not explicitly set --timeout + // (i.e. the default is in effect). First runs can take several minutes; + // skipped when --output json (non-interactive consumers don't need the hint). + if (opts.timeoutIsDefault === true && opts.output !== 'json') { + stderrFn( + `[hint] First runs can take several minutes; raise --timeout if this run is cut short.`, + ); + } + + // Backend-test fallback (dogfood L1888): BE run rows never finalize, so + // resolve the verdict from the testId record once it's terminal. + let beFallbackUsed = false; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => opts.testId, + resolveNotBefore: () => triggerResponse.enqueuedAt, + onResolved: testId => { + beFallbackUsed = true; + stderrFn( + `[advisory] Backend run-surface row is not finalized server-side (dogfood L1888); ` + + `resolved the verdict from the test record (testId=${testId}). ` + + `Read full detail with: testsprite test result ${testId}`, + ); + }, + }); + + let finalRun: RunResponse; + try { + finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.update( + `Run ${run.runId} — ${run.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + } catch (err) { + if (err instanceof TimeoutError) { + ticker.finalize(`Run ${triggerResponse.runId} — timed out after ${opts.timeoutSeconds}s`); + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', // exit 7 per errors.md + message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${triggerResponse.runId}.`, + nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}`, + requestId: 'local', + details: { runId: triggerResponse.runId, timeoutSeconds: opts.timeoutSeconds }, + }, + }); + } + // C+D: RequestTimeoutError during polling — emit a partial object to stdout + // routed through printRunOrChain so: + // TEXT mode: renders human-readable (not raw JSON) + // JSON mode: preserves the merged create-chain envelope + // { ...createContext, run: { runId, status, targetUrl } } + // targetUrl is taken from triggerResponse, which is already bound to + // the real in-flight URL (see Finding D fix in the 409 resume path). + if (err instanceof RequestTimeoutError) { + ticker.finalize(`Run ${triggerResponse.runId} — request timed out`); + const partial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, partial, opts.createContext, data => { + const p = data as typeof partial; + const lines = [`runId ${p.runId}`, `status ${p.status} (request timed out)`]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + return lines.join('\n'); + }); + stderrFn( + `Run ${triggerResponse.runId} is still in progress (request timed out). ` + + `Re-attach with: testsprite test wait ${triggerResponse.runId}`, + ); + throw err; + } + ticker.finalize(); + throw err; + } + + const elapsed = Math.round((Date.now() - startMs) / 1000); + const s = finalRun.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.finalize( + `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + + printRunOrChain( + out, + withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), + opts.createContext, + data => + renderRunResponseText(data as RunResponse, { + // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs). + // This ensures fast BE runs terminal on first poll still render n/a. + isBackend: beFallbackUsed || opts.type === 'backend', + }), + ); + + // Surface the trigger requestId under --verbose/--debug or JSON mode so + // operators can trace the full lifecycle (gated since 2026-06-04 dogfood; + // JSON mode always emits to stderr — it never pollutes stdout). + if (triggerRequestId && (opts.output === 'json' || opts.verbose || opts.debug)) + stderrFn(`requestId: ${triggerRequestId}`); + + if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + // BE runs (resolved via the testId fallback) have no run-scoped artifact + // bundle — their failure bundle is addressed by testId, not runId. + stderrFn( + beFallbackUsed + ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` + : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + ); + } + + const exitCode = exitCodeForRunStatus(finalRun.status); + if (exitCode !== 0) { + // Throw a CLIError so index.ts exits with the right code without + // printing an error envelope — the result was already printed above. + throw new CLIError(`Run ${finalRun.runId} finished with status: ${finalRun.status}`, exitCode); + } + + return finalRun; +} + +/** + * `test wait ` — M3.3 piece-3. + * + * Polls `GET /api/cli/v1/runs/{runId}` until terminal status. Exit + * codes match the `--wait` behavior matrix in the spec. + */ +export async function runTestWait( + opts: RunTestWaitOptions, + deps: TestDeps = {}, +): Promise { + if (opts.dryRun) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + emitDryRunBanner(stderrFn); + const out = makeOutput(opts.output, deps); + const envelope = { + method: 'GET', + path: `/api/cli/v1/runs/${opts.runId}?waitSeconds=25`, + timeoutSeconds: opts.timeoutSeconds, + }; + out.print(envelope); + return envelope as unknown as RunResponse; + } + + // D4: `test wait` is always a waiting command (it has no --wait flag — it IS + // the wait), so force wait:true when deriving the per-request timeout. This + // raises the window to cover --timeout so a long-poll under load isn't cut at + // the 120s default. + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const startMs = Date.now(); + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + // Backend-test fallback (dogfood L1888): the run row never finalizes, so + // resolve the verdict from the testId record (discovered from the first + // poll tick) once it's terminal for this run. + let beFallbackUsed = false; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: run => run.testId, + resolveNotBefore: run => run.createdAt, + onResolved: testId => { + beFallbackUsed = true; + stderrFn( + `[advisory] Backend run-surface row is not finalized server-side (dogfood L1888); ` + + `resolved the verdict from the test record (testId=${testId}). ` + + `Read full detail with: testsprite test result ${testId}`, + ); + }, + }); + + let finalRun: RunResponse; + try { + finalRun = await pollRunUntilTerminal(client, opts.runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.update( + `Run ${run.runId} — ${run.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + } catch (err) { + if (err instanceof TimeoutError) { + ticker.finalize(`Run ${opts.runId} — timed out after ${opts.timeoutSeconds}s`); + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', // exit 7 per errors.md + message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, + nextAction: `Resume polling: testsprite test wait ${opts.runId}`, + requestId: 'local', + details: { runId: opts.runId, timeoutSeconds: opts.timeoutSeconds }, + }, + }); + } + // C: RequestTimeoutError during polling — emit a partial object to stdout + // routed through the same render path as the success case (text mode renders + // human-readable; JSON mode produces a parseable envelope — not raw JSON). + if (err instanceof RequestTimeoutError) { + ticker.finalize(`Run ${opts.runId} — request timed out`); + const partial = { runId: opts.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (request timed out)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); + stderrFn( + `Run ${opts.runId} is still in progress (request timed out). ` + + `Re-attach with: testsprite test wait ${opts.runId}`, + ); + throw err; + } + ticker.finalize(); + throw err; + } + + const elapsed = Math.round((Date.now() - startMs) / 1000); + const s = finalRun.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.finalize( + `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => + renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + ); + + if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + // BE runs (resolved via the testId fallback) have no run-scoped artifact + // bundle — their failure bundle is addressed by testId, not runId. + stderrFn( + beFallbackUsed + ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` + : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + ); + } + + const exitCode = exitCodeForRunStatus(finalRun.status); + if (exitCode !== 0) { + throw new CLIError(`Run ${finalRun.runId} finished with status: ${finalRun.status}`, exitCode); + } + + return finalRun; +} + +// --------------------------------------------------------------------------- +// M4 piece-2 — `test run --all --project ` (fresh wave-ordered batch run) +// --------------------------------------------------------------------------- + +interface RunTestRunAllOptions extends CommonOptions { + /** projectId to run all BE tests in. */ + projectId: string; + /** --filter : only run tests whose name contains this substring (case-insensitive). */ + nameFilter?: string; + /** --wait: block until terminal or --timeout. */ + wait: boolean; + /** Polling / overall deadline in seconds. Default 600, max 3600. */ + timeoutSeconds: number; + /** --max-concurrency: bounds the --wait poll fan-out. */ + maxConcurrency: number; + /** Caller-supplied idempotency token; auto-minted if absent. */ + idempotencyKey?: string; +} + +/** + * CLI result shape for a single member of a fresh batch run poll. + */ +interface CliBatchRunFreshResult { + testId: string; + runId: string | undefined; + status: string; + error?: { code: string; message: string; exitCode: number }; + /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ + dashboardUrl?: string; +} + +/** + * `test run --all --project ` — M4 piece-2. + * + * Triggers a fresh wave-ordered batch run via `POST /tests/batch/run`. + * FE tests in the project are skipped by the BE-only engine (advisory). + * With `--wait`, polls each accepted runId. + */ +export async function runTestRunAll( + opts: RunTestRunAllOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + requireProjectId(opts.projectId); + if ( + !Number.isInteger(opts.maxConcurrency) || + opts.maxConcurrency < 1 || + opts.maxConcurrency > MAX_BATCH_CONCURRENCY + ) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + // --- Dry-run path --- + if (opts.dryRun) { + const idempotencyKey = opts.idempotencyKey ?? `dry-run-${randomUUID()}`; + const batchRunSample = findSample('POST', '/api/cli/v1/tests/batch/run')?.body(); + const envelope = { + dryRun: true, + method: 'POST', + path: '/api/cli/v1/tests/batch/run', + body: { + projectId: opts.projectId, + testIds: opts.nameFilter ? [''] : undefined, + source: 'cli' as const, + }, + idempotencyKey, + ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), + }; + out.print(batchRunSample ?? envelope); + return undefined; + } + + // D4: under --wait, raise per-request timeout to cover --timeout. + const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); + + // Portal deep links for batch output: every test in the batch belongs to + // opts.projectId, so per-item dashboardUrl needs no extra wire data. The + // project-level URL closes out text-mode output ("watch the wave here"). + // Both stay undefined for unknown API hosts (resolvePortalBase contract). + const batchApiUrl = resolveApiUrl(opts, deps); + const batchPortalBase = resolvePortalBase(batchApiUrl); + const projectDashboardUrl = + batchPortalBase === undefined + ? undefined + : `${batchPortalBase}/dashboard/tests/${encodeURIComponent(opts.projectId)}`; + const withBatchDashboardUrl = (item: T): T => { + const dashboardUrl = resolvePortalUrl(batchApiUrl, opts.projectId, item.testId); + return dashboardUrl !== undefined ? { ...item, dashboardUrl } : item; + }; + + const idempotencyKey = opts.idempotencyKey ?? `cli-batch-run-fresh-${randomUUID()}`; + if (opts.idempotencyKey === undefined && opts.debug) { + stderrFn(`idempotency-key: ${idempotencyKey}`); + } + if (opts.idempotencyKey === undefined && opts.verbose) { + stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); + } + + // Resolve testIds: fetch all BE tests in the project, apply --filter. + let testIds: string[] | undefined; + if (opts.nameFilter !== undefined && opts.nameFilter !== '') { + // We need to resolve the full test set to apply the name filter. + const allPage = await paginate( + async ({ pageSize, cursor }) => + client.get>('/tests', { + query: { projectId: opts.projectId, pageSize, cursor }, + }), + {}, + ); + const needle = opts.nameFilter.toLowerCase(); + const filtered = allPage.items.filter(t => t.name.toLowerCase().includes(needle)); + const before = allPage.items.length; + const skipped = before - filtered.length; + if (skipped > 0) { + stderrFn( + `--filter: skipped ${skipped} test${skipped !== 1 ? 's' : ''} whose name does not contain "${opts.nameFilter}".`, + ); + } + testIds = filtered.map(t => t.id); + if (testIds.length === 0) { + stderrFn( + `No tests found in project ${opts.projectId} matching --filter "${opts.nameFilter}" — nothing to run.`, + ); + out.print({ + accepted: [], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + } satisfies BatchRunFreshResponse); + return undefined; + } + stderrFn( + `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, + ); + } + // When no --filter, omit testIds → server runs ALL BE tests in the project. + + const batchResp = await client.triggerBatchRunFresh( + { + projectId: opts.projectId, + ...(testIds !== undefined ? { testIds } : {}), + source: 'cli', + }, + { idempotencyKey }, + ); + + // Mutable: D3 deferred-retry loop may append to `accepted`, drain `deferred`, + // and accumulate additional `conflicts` discovered during retries. + let accepted = batchResp.accepted.slice(); + let deferred = batchResp.deferred.slice(); + let conflicts = batchResp.conflicts.slice(); + const { skippedFrontend, skippedIntegration } = batchResp; + + // Print advisory for skipped FE tests. + if (skippedFrontend.length > 0) { + stderrFn( + `[advisory] ${skippedFrontend.length} frontend test${skippedFrontend.length !== 1 ? 's' : ''} skipped — the batch run endpoint uses the BE-only wave engine. ` + + `Use 'testsprite test run ' individually for FE tests.`, + ); + } + if (skippedIntegration.length > 0) { + stderrFn( + `[advisory] ${skippedIntegration.length} assembled integration test${skippedIntegration.length !== 1 ? 's' : ''} skipped — not runnable via the CLI wave path. Run them from the portal.`, + ); + } + if (conflicts.length > 0) { + stderrFn( + `[advisory] ${conflicts.length} test${conflicts.length !== 1 ? 's' : ''} already in flight, skipped: ${conflicts.map(c => c.testId).join(' ')}`, + ); + } + if (deferred.length > 0) { + stderrFn(`Rate-deferred testIds (retry later): ${deferred.map(d => d.testId).join(' ')}`); + } + + stderrFn( + `Dispatched ${accepted.length} test${accepted.length !== 1 ? 's' : ''}` + + `${skippedFrontend.length > 0 ? ` (${skippedFrontend.length} FE skipped)` : ''}` + + `${conflicts.length > 0 ? ` (${conflicts.length} in flight)` : ''}` + + `${deferred.length > 0 ? ` (${deferred.length} rate-deferred)` : ''}.`, + ); + + if (!opts.wait) { + const printResp: BatchRunFreshResponse = { + accepted: accepted.map(withBatchDashboardUrl), + conflicts, + deferred, + skippedFrontend, + skippedIntegration, + }; + out.print(printResp, data => { + const r = data as BatchRunFreshResponse; + const lines: string[] = [`accepted ${r.accepted.length}`]; + if (r.conflicts.length > 0) + lines.push(`conflicts ${r.conflicts.length} (already in flight)`); + if (r.deferred.length > 0) + lines.push(`deferred ${r.deferred.length} (rate-limited — retry)`); + if (r.skippedFrontend.length > 0) { + lines.push(`skippedFE ${r.skippedFrontend.length} (use 'test run ' for FE tests)`); + } + if (r.skippedIntegration.length > 0) { + lines.push( + `skippedIntegr ${r.skippedIntegration.length} (run assembled integration tests via portal)`, + ); + } + for (const a of r.accepted) { + lines.push(` ${a.testId} runId: ${a.runId} enqueuedAt: ${a.enqueuedAt}`); + } + if (projectDashboardUrl !== undefined) { + lines.push(`dashboard ${projectDashboardUrl}`); + } + return lines.join('\n'); + }); + // Rate-deferred tests were NOT dispatched → signal incomplete (exit 7), + // mirroring `test rerun --all`. The user retries with a fresh invocation. + if (deferred.length > 0) { + throw new CLIError( + `Batch run incomplete: ${deferred.length} test${deferred.length !== 1 ? 's' : ''} rate-deferred (per-key run budget). Retry these individually after ~60s: ${deferred.map(d => d.testId).join(' ')}`, + 7, + ); + } + // Nothing queued and everything was an in-flight conflict → surface CONFLICT (exit 6). + if (accepted.length === 0 && conflicts.length > 0) { + throw ApiError.fromEnvelope({ + error: { + code: 'CONFLICT', + message: `Batch run: nothing was queued — ${conflicts.length} test${conflicts.length !== 1 ? 's' : ''} already in flight.`, + nextAction: `Wait for the in-flight runs to complete, then retry, or use: testsprite test wait `, + requestId: 'local', + details: { conflicts: conflicts.map(c => c.testId) }, + }, + }); + } + // [P2] Return post-retry state so programmatic callers and the create-chain + // JSON merge reflect what was actually dispatched, not the stale initial resp. + return { ...batchResp, accepted, deferred, conflicts }; + } + + // D3: bounded deferred-retry loop (only under --wait). + // Up to MAX_DEFERRED_RETRIES attempts to re-dispatch still-deferred tests. + // Each attempt sleeps for Retry-After (if server provided it) or the default + // 61s, clamped to the remaining --timeout budget. Newly-accepted runs are + // merged into `accepted`; if still deferred after all attempts, fall through + // to the existing exit-7 path. + const sleepFn = deps.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const batchDeadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + for (let attempt = 1; attempt <= MAX_DEFERRED_RETRIES && deferred.length > 0; attempt++) { + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + stderrFn( + `[deferred-retry] timeout budget exhausted before attempt ${attempt}/${MAX_DEFERRED_RETRIES} — ${deferred.length} test${deferred.length !== 1 ? 's' : ''} still deferred.`, + ); + break; + } + const sleepMs = Math.min(DEFERRED_RETRY_DEFAULT_SLEEP_MS, remainingMs); + stderrFn( + `[deferred-retry] attempt ${attempt}/${MAX_DEFERRED_RETRIES} — retrying ${deferred.length} deferred test${deferred.length !== 1 ? 's' : ''} in ${Math.round(sleepMs / 1000)}s`, + ); + await sleepFn(sleepMs); + + const remainingAfterSleep = batchDeadlineMs - Date.now(); + if (remainingAfterSleep <= 0) { + stderrFn( + `[deferred-retry] timeout budget exhausted during sleep — ${deferred.length} test${deferred.length !== 1 ? 's' : ''} still deferred.`, + ); + break; + } + + const retryIds = deferred.map(d => d.testId); + // [P2] Bound the derived key to ≤256 chars. Caller-supplied keys may be up + // to 256 chars; appending `:deferred-retryN` (≤16 chars) could push past + // the server's 256-char limit and cause every retry to be rejected. Truncate + // the base key to leave room for the suffix before concatenating. + const retrySuffix = `:deferred-retry${attempt}`; + const retryBase = + idempotencyKey.length + retrySuffix.length > 256 + ? idempotencyKey.slice(0, 256 - retrySuffix.length) + : idempotencyKey; + const retryKey = `${retryBase}${retrySuffix}`; + let retryResp: BatchRunFreshResponse; + try { + retryResp = await client.triggerBatchRunFresh( + { + projectId: opts.projectId, + testIds: retryIds, + source: 'cli', + }, + { idempotencyKey: retryKey }, + ); + } catch (err) { + // If the retry itself errors, surface the error and stop retrying. + stderrFn( + `[deferred-retry] attempt ${attempt} failed with error: ${err instanceof Error ? err.message : String(err)}`, + ); + break; + } + + const newlyAccepted = retryResp.accepted; + const newlyDeferred = retryResp.deferred; + const newlyConflicted = retryResp.conflicts; + + if (newlyAccepted.length > 0) { + stderrFn( + `[deferred-retry] attempt ${attempt}: ${newlyAccepted.length} test${newlyAccepted.length !== 1 ? 's' : ''} now accepted.`, + ); + accepted = accepted.concat(newlyAccepted); + } + if (newlyConflicted.length > 0) { + // [P1] Merge retry-returned conflicts into the running conflicts collection + // so the final summary, stderr output, and exit-code logic reflect them. + // Without this merge, tests deferred-then-conflicted on retry are invisible + // to the final accounting and can cause a false-zero conflicts count. + stderrFn( + `[deferred-retry] attempt ${attempt}: ${newlyConflicted.length} test${newlyConflicted.length !== 1 ? 's' : ''} in-flight (conflict).`, + ); + conflicts = conflicts.concat(newlyConflicted); + } + deferred = newlyDeferred; + if (deferred.length === 0) { + stderrFn(`[deferred-retry] attempt ${attempt}: all previously-deferred tests accepted.`); + } + } + + // --wait: fan-out poll each accepted run by its runId. Every accepted entry + // carries a real runId (the backend routes slot-claim failures to conflicts[]), + // so there is no "draft, no runId" subset to silently skip — polling all of + // them is the only way `--wait` can report a faithful (not false-green) verdict. + const pollable = accepted; + + if (pollable.length === 0) { + // Build final response with potentially-updated accepted/deferred from D3 retry loop. + const finalResp: BatchRunFreshResponse = { + accepted, + conflicts, + deferred, + skippedFrontend, + skippedIntegration, + }; + out.print(finalResp); + // Nothing to poll: surface deferred (rate-limit → exit 7) or all-conflict (exit 6), + // mirroring the non-wait path so `--wait` never silently exits 0 on a no-op batch. + if (deferred.length > 0) { + throw new CLIError( + `Batch run incomplete: ${deferred.length} test${deferred.length !== 1 ? 's' : ''} rate-deferred (per-key run budget) — retry these individually after ~60s: ${deferred.map(d => d.testId).join(' ')}`, + 7, + ); + } + if (conflicts.length > 0) { + throw ApiError.fromEnvelope({ + error: { + code: 'CONFLICT', + message: `Batch run: nothing was queued — ${conflicts.length} test${conflicts.length !== 1 ? 's' : ''} already in flight.`, + nextAction: `Wait for the in-flight runs to complete, then retry, or use: testsprite test wait `, + requestId: 'local', + details: { conflicts: conflicts.map(c => c.testId) }, + }, + }); + } + // [P2] Return post-retry state. + return { ...batchResp, accepted, deferred, conflicts }; + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const concurrencyLimit = opts.maxConcurrency; + const freshRunResults: CliBatchRunFreshResult[] = []; + + // Single deadline shared across the whole fan-out (codex): each queued poll + // gets the time REMAINING against this batch deadline, not a fresh full + // `timeoutSeconds`. Without this, runs that wait behind `--max-concurrency` + // could push total wall-clock to ~ceil(N/concurrency) × timeout instead of + // the documented `--timeout` ceiling. + // NOTE: batchDeadlineMs is set in the D3 deferred-retry loop above and reused here. + + async function pollFreshAccepted(entry: BatchRunFreshAccepted): Promise { + const runId = entry.runId; + const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => entry.testId, + resolveNotBefore: () => entry.enqueuedAt, + onResolved: () => undefined, + }); + try { + const finalRun = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.update( + `Run ${run.runId} (${entry.testId}) — ${run.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + return { testId: entry.testId, runId, status: finalRun.status }; + } catch (err) { + if (err instanceof TimeoutError) { + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + if (err instanceof ApiError) { + // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, + // RATE_LIMITED=11, …) instead of flattening every member failure to 1 + // (codex) — an operator/agent needs the actionable code, not a generic 1. + return { + testId: entry.testId, + runId, + status: 'error', + error: { code: err.code, message: err.message, exitCode: err.exitCode }, + }; + } + throw err; + } + } + + // Bounded concurrency fan-out + let pollIdx = 0; + let inFlight = 0; + + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && pollIdx < pollable.length) { + const entry = pollable[pollIdx++]!; + inFlight++; + pollFreshAccepted(entry) + .then(result => { + freshRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && pollIdx >= pollable.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (pollable.length === 0) resolve(); + }); + + ticker.finalize(); + + const passed = freshRunResults.filter(r => r.status === 'passed').length; + const failed = freshRunResults.filter( + r => r.status !== 'passed' && r.status !== 'timeout', + ).length; + const timedOut = freshRunResults.filter(r => r.status === 'timeout').length; + + stderrFn( + `Batch run complete: ${passed}/${pollable.length} passed, ${failed} failed/blocked, ${timedOut} timed out`, + ); + if (projectDashboardUrl !== undefined) { + stderrFn(`Dashboard: ${projectDashboardUrl}`); + } + + const jsonPayload = { + accepted: freshRunResults.map(withBatchDashboardUrl), + conflicts, + deferred, + skippedFrontend, + skippedIntegration, + summary: { + passed, + failed, + timedOut, + deferred: deferred.length, + conflicts: conflicts.length, + total: pollable.length, + }, + }; + out.print(jsonPayload); + + // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), + // mirroring `test rerun --all`. Checked before the failed-run throw so the + // operator learns to retry the deferred set. + if (deferred.length > 0) { + throw new CLIError( + `Batch run incomplete: ${deferred.length} test${deferred.length !== 1 ? 's' : ''} rate-deferred (per-key run budget) — retry these individually after ~60s: ${deferred.map(d => d.testId).join(' ')}`, + 7, + ); + } + + if (timedOut > 0) { + const timedOutRunIds = freshRunResults + .filter(r => r.status === 'timeout') + .map(r => r.runId) + .filter(Boolean) as string[]; + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', + message: `${timedOut} run${timedOut !== 1 ? 's' : ''} timed out.`, + nextAction: timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`).join('\n'), + requestId: 'local', + details: { timedOutRunIds, timeoutSeconds: opts.timeoutSeconds }, + }, + }); + } + + if (failed > 0) { + // An auth failure on any member is a batch-wide condition (the credential is + // bad, not the test) — propagate exit 3 so the operator fixes auth rather + // than chasing a "test failed" (exit 1). Other operational codes stay folded + // into the generic batch-failure exit 1; their per-member envelope is in JSON. + const authErr = freshRunResults.find(r => r.error?.exitCode === 3); + if (authErr) { + throw new CLIError( + `${failed} run${failed !== 1 ? 's' : ''} failed — auth error (${authErr.error?.code}): ${authErr.error?.message}`, + 3, + ); + } + throw new CLIError(`${failed} run${failed !== 1 ? 's' : ''} failed.`, 1); + } + + // [P2] Return object reconstructed from post-retry mutable state (accepted, + // deferred, conflicts) so the caller always sees what was actually dispatched, + // not the stale initial batchResp. accepted here still holds the original + // BatchRunFreshAccepted entries (with runId + enqueuedAt); the freshRunResults + // fan-out is for exit-code logic only and is not part of the returned type. + return { ...batchResp, accepted, deferred, conflicts }; +} + +// --------------------------------------------------------------------------- +// M3.4 piece-3 — `test rerun` (single + batch) +// --------------------------------------------------------------------------- + +/** + * CLI result shape for a single rerun in a batch fan-out poll. + * Mirrors `CliBatchRunResult` but keyed on the rerun's runId. + */ +interface CliRerunResult { + testId: string; + runId: string; + /** Terminal status, or 'timeout' for per-run deadline exceeded. */ + status: string; + /** Set when the test is a closure member (not the user's named test). */ + role?: string; + /** Structured error for non-passing runs. */ + error?: { + code: string; + message: string; + exitCode: number; + }; +} + +/** + * `test rerun` — M3.4 piece-3. + * + * FE: `POST /tests/{id}/runs/rerun` → verbatim replay (no credit). With + * `--wait`, polls `GET /runs/{runId}` until terminal. + * + * BE: same route → closure + per-member runIds. With `--wait`, polls every + * closure-member runId; exits on the named test's verdict; failed closure + * members surface as warnings + `closureFailures[]` in JSON. + * + * Batch / `--all`: `POST /tests/batch/rerun` → per-test runIds. With + * `--wait`, fan-out poll under `--max-concurrency`. `deferred[]` → exit 7. + */ +export async function runTestRerun( + opts: RunTestRerunOptions, + deps: TestDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + // ------------------------------------------------------------------------- + // Input validation + // ------------------------------------------------------------------------- + if (opts.testIds.length === 0 && !opts.all) { + throw localValidationError( + 'test-ids', + 'provide at least one , or use --all to rerun all tests in the project', + ); + } + if (opts.all && !opts.projectId) { + throw localValidationError( + 'project', + '--all requires a project context — pass --project or configure a default', + ); + } + // --filter is an --all-only narrowing filter (applied to the fetched project + // test set). Without --all it would be SILENTLY ignored while explicit ids + // still get reran — defeating the user's narrowing intent and burning + // rerun/auto-heal credits (codex). Reject early. (Mirrors delete-batch's + // --status guard.) + if (opts.nameFilter !== undefined && opts.nameFilter !== '' && !opts.all) { + throw localValidationError( + 'filter', + '--filter only applies with --all (it narrows which project tests get reran). ' + + 'Remove --filter, or add --all --project .', + ); + } + if ( + !Number.isInteger(opts.maxConcurrency) || + opts.maxConcurrency < 1 || + opts.maxConcurrency > MAX_BATCH_CONCURRENCY + ) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + + const isSingle = !opts.all && opts.testIds.length === 1; + + // ------------------------------------------------------------------------- + // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) + // ------------------------------------------------------------------------- + let effectiveAutoHeal = opts.autoHeal; + + if (opts.dryRun) { + const client = makeClient(opts, deps); + const idempotencyKey = opts.idempotencyKey ?? `dry-run-${randomUUID()}`; + if (isSingle) { + const testId = opts.testIds[0]!; + const envelope = { + dryRun: true, + method: 'POST', + path: `/api/cli/v1/tests/${testId}/runs/rerun`, + body: { + source: 'cli' as const, + autoHeal: effectiveAutoHeal, + skipDependencies: opts.skipDependencies, + }, + idempotencyKey, + ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), + }; + out.print(findSample('POST', `/api/cli/v1/tests/${testId}/runs/rerun`)?.body() ?? envelope); + } else { + const testIds = opts.all ? [''] : opts.testIds; + const envelope = { + dryRun: true, + method: 'POST', + path: `/api/cli/v1/tests/batch/rerun`, + body: { + source: 'cli' as const, + testIds, + autoHeal: effectiveAutoHeal, + skipDependencies: opts.skipDependencies, + }, + idempotencyKey, + ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), + }; + out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); + } + void client; + return undefined; + } + + // D4: under --wait, raise the per-request timeout to cover --timeout so a + // slow rerun trigger / long-poll under load isn't cut at the 120s default. + const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); + const idempotencyKey = opts.idempotencyKey ?? `cli-rerun-${randomUUID()}`; + if (opts.idempotencyKey === undefined && opts.debug) { + stderrFn(`idempotency-key: ${idempotencyKey}`); + } + if (opts.idempotencyKey === undefined && opts.verbose) { + stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); + } + + // ------------------------------------------------------------------------- + // Single rerun path + // ------------------------------------------------------------------------- + if (isSingle) { + const testId = opts.testIds[0]!; + + // Pre-flight: check if BE test with auto-heal (best-effort). + // Only emit the "ignoring auto-heal" advisory when the user EXPLICITLY + // requested auto-heal via a flag (autoHealExplicit=true). With the current + // default-on design (`--no-auto-heal` is the only flag), autoHealExplicit + // is always false — there is no `--auto-heal` flag to set. Suppressing the + // warning on default-on prevents every BE rerun from printing noisy advice + // about a feature the user never asked for. A future explicit `--auto-heal` + // flag would set autoHealExplicit=true and restore the warning. + if (opts.autoHeal) { + try { + const test = await client.get(`/tests/${encodeURIComponent(testId)}`); + if (test.type === 'backend') { + if (opts.autoHealExplicit) { + stderrFn( + `[advisory] auto-heal applies to frontend tests only; ignoring for backend test ${testId}`, + ); + } + effectiveAutoHeal = false; + } + } catch { + // Best-effort: don't fail on a lookup error; server will gate anyway. + } + } + + let rerunResp: RerunResponse; + try { + rerunResp = await client.triggerRerun( + testId, + { + source: 'cli', + ...(effectiveAutoHeal ? { autoHeal: true } : {}), + ...(opts.skipDependencies ? { skipDependencies: true } : {}), + }, + { idempotencyKey }, + ); + } catch (err) { + if (err instanceof ApiError && err.code === 'CONFLICT') { + const currentRunId = err.getDetail( + 'currentRunId', + (v): v is string => typeof v === 'string' && v.length > 0, + ); + throw ApiError.fromEnvelope({ + error: { + code: 'CONFLICT', + message: `Test ${testId} already has a run in flight. Wait for it to finish before rerunning.`, + nextAction: currentRunId + ? `testsprite test wait ${currentRunId}` + : `testsprite test result ${testId}`, + requestId: err.requestId ?? 'local', + details: { testId, currentRunId }, + }, + }); + } + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + // D2 (dogfood): a rerun replays a SAVED run/script. A test that has + // never completed a clean run (or an unknown/cross-tenant id) has + // nothing to replay → NOT_FOUND. Point the user at a fresh run, which + // requires no prior result. (Without this hint the bare exit-4 gives no + // clue that `test run` is the fallback.) + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Test ${testId} has no replayable run to rerun (unknown/cross-tenant id, or it has never completed a clean run).`, + nextAction: `For a first run (no prior result to replay), trigger a fresh run: testsprite test run ${testId}`, + requestId: err.requestId ?? 'local', + details: { testId, reason: 'no_replayable_run' }, + }, + }); + } + throw err; + } + + // Print auto-heal advisory. + // CLI path: auto-heal is default-on for FE reruns (--no-auto-heal to opt + // out). Free and paid CLI callers both get auto-heal; backend no longer + // tier-gates for source='cli'. Cost: 0.2 credits per engage (charged only + // when Phase-2 heal actually runs; verbatim replay passes are free). + // + // Defensive branch: if the server still echoes autoHeal:false after we sent + // autoHeal:true, the server did not apply it (unexpected; may happen on + // very old portal backends or if the CLI's claim was rejected for another + // reason). We keep this branch but reword it — do NOT claim "requires Pro + // plan" since the CLI path has no paid gate. + // + // Use effectiveAutoHeal (not opts.autoHeal) so BE reruns — where + // effectiveAutoHeal was set to false earlier — do NOT trigger the + // "not applied" advisory on every run (opts.autoHeal is still the + // default-on `true`, so opts.autoHeal && !rerunResp.autoHeal would + // fire spuriously for every BE rerun). + if (effectiveAutoHeal && !rerunResp.autoHeal) { + // Env-correct billing link (dev/prod portals differ); route-only when + // the API host is unknown. + const advisoryPortalBase = resolvePortalBase(resolveApiUrl(opts, deps)); + stderrFn( + `[advisory] auto-heal was not applied by the server (verbatim replay).` + + ` If this was unexpected, check your balance at ${ + advisoryPortalBase !== undefined + ? `${advisoryPortalBase}/dashboard/settings/billing` + : 'the portal Billing page (/dashboard/settings/billing)' + }.`, + ); + } else if (rerunResp.autoHeal) { + stderrFn( + `[advisory] auto-heal on (FE rerun default). If a step has drifted, healing runs and costs 0.2 credit. Disable with --no-auto-heal.`, + ); + } + + const isBERerun = !!rerunResp.closure; + + if (isBERerun && rerunResp.closure) { + const totalCount = rerunResp.closure.members.length; + // G1d: split producers and teardowns into separate parts so the + // summary accurately labels each role. Example outputs: + // "Reran 5 tests: 1 selected + 2 producer(s) + 2 teardown(s)" + // "Reran 3 tests: 1 selected + 2 producer(s)" + // "Reran 2 tests: 1 selected + 1 teardown(s)" + // "Reran 1 test: 1 selected" + const parts: string[] = ['1 selected']; + const nProducers = rerunResp.closure.addedProducers.length; + const nTeardowns = rerunResp.closure.addedTeardowns.length; + if (nProducers > 0) parts.push(`${nProducers} producer${nProducers !== 1 ? 's' : ''}`); + if (nTeardowns > 0) parts.push(`${nTeardowns} teardown${nTeardowns !== 1 ? 's' : ''}`); + stderrFn(`Reran ${totalCount} test${totalCount !== 1 ? 's' : ''}: ${parts.join(' + ')}`); + // B4 (dogfood): backend reruns do not set `createdFrom`, so run-history + // can't distinguish a rerun from a fresh run for BE tests. Tell the user + // up front so they don't trust `test result --history` to flag reruns. + stderrFn( + `[advisory] backend rerun history does not distinguish reruns — ` + + `'test result --history' shows isRerun:false / createdFrom:null for backend rows by design. ` + + `Rerun-ness lives in the audit trail only (command=test.rerun).`, + ); + } + + if (!opts.wait) { + out.print(rerunResp, data => { + const r = data as RerunResponse; + const lines = [ + `runId ${r.runId}`, + `status ${r.status}`, + `enqueuedAt ${r.enqueuedAt}`, + `codeVersion ${r.codeVersion}`, + `autoHeal ${r.autoHeal}`, + ]; + if (r.closure) { + lines.push( + `closure ${r.closure.members.length} members (${r.closure.addedProducers.length} producers added)`, + ); + } + return lines.join('\n'); + }); + return rerunResp; + } + + // --wait path for single rerun + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + if (isBERerun && rerunResp.closure && rerunResp.closure.members.length > 1) { + // BE rerun: poll every closure-member runId, exit on named test's verdict. + const namedRunId = rerunResp.runId; + const closureMembers = rerunResp.closure.members; + const closureFailures: Array<{ testId: string; runId: string; status: string }> = []; + + const pollMember = async (member: RerunClosureMember): Promise => { + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => member.testId, + resolveNotBefore: run => run.createdAt, + onResolved: () => undefined, + }); + try { + return await pollRunUntilTerminal(client, member.runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { + total: 0, + completed: 0, + passedCount: 0, + failedCount: 0, + }; + ticker.update( + `Run ${run.runId} [${member.role}] — ${run.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + } catch (err) { + if (err instanceof TimeoutError) { + return null; + } + throw err; + } + }; + + // Fan-out poll with concurrency limit + const members = closureMembers; + const memberResults = new Map(); + const concurrencyLimit = opts.maxConcurrency; + let inFlight = 0; + let memberIdx = 0; + + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && memberIdx < members.length) { + const member = members[memberIdx++]!; + inFlight++; + pollMember(member) + .then(result => { + memberResults.set(member.runId, result); + if (member.runId !== namedRunId) { + if (result === null) { + // Timed-out closure member: treat as incomplete/failed so + // the exit-code path fires exit 7 rather than silently + // succeeding with an unobserved member. + closureFailures.push({ + testId: member.testId, + runId: member.runId, + status: 'timeout', + }); + stderrFn( + `⚠ closure member ${member.testId} (runId: ${member.runId}) timed out — rerun did not reach terminal within --timeout`, + ); + } else if (result.status !== 'passed') { + closureFailures.push({ + testId: member.testId, + runId: member.runId, + status: result.status, + }); + stderrFn( + `⚠ closure member ${member.testId} (runId: ${member.runId}) finished with status: ${result.status}`, + ); + } + } + inFlight--; + startNext(); + if (inFlight === 0 && memberIdx >= members.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (members.length === 0) resolve(); + }); + } catch (fanOutErr) { + // D4 (closure fan-out): a RequestTimeoutError from any member's poll + // propagates through .catch(reject) and rejects the fan-out promise + // before any stdout is written — leaving a redirected stdout empty. + // Emit a partial object for every dispatched run so the caller always + // has something parseable on stdout, then re-throw (exit 7). + if (fanOutErr instanceof RequestTimeoutError) { + ticker.finalize(`Closure fan-out — request timed out`); + const dispatchedRunIds = closureMembers.map(m => ({ + runId: m.runId, + testId: m.testId, + role: m.role, + status: 'running' as const, + })); + out.print({ runId: namedRunId, status: 'running', closure: dispatchedRunIds }, () => + dispatchedRunIds + .map(m => `${m.role.padEnd(9)} ${m.testId} (runId: ${m.runId}) — running`) + .join('\n'), + ); + const reattachHints = closureMembers + .map(m => `testsprite test wait ${m.runId}`) + .join('\n'); + stderrFn( + `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}`, + ); + throw fanOutErr; + } + throw fanOutErr; + } + + ticker.finalize(); + + // Find named test's result + const namedMember = closureMembers.find(m => m.runId === namedRunId); + const namedResult = namedMember ? memberResults.get(namedRunId) : null; + + const jsonPayload: Record = { + runId: namedRunId, + testId, + autoHeal: rerunResp.autoHeal, + closure: rerunResp.closure, + namedStatus: namedResult?.status ?? 'timeout', + ...(closureFailures.length > 0 ? { closureFailures } : {}), + }; + + out.print(jsonPayload, () => { + const lines = [ + `runId ${namedRunId}`, + `testId ${testId}`, + `status ${namedResult?.status ?? 'timeout (exceeded --timeout)'}`, + `autoHeal ${rerunResp.autoHeal}`, + ]; + if (closureFailures.length > 0) { + lines.push(`⚠ closureFailures:`); + for (const f of closureFailures) lines.push(` ${f.testId} (${f.runId}): ${f.status}`); + } + return lines.join('\n'); + }); + + if (!namedResult) { + // timeout + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${namedRunId}.`, + nextAction: `Resume polling: testsprite test wait ${namedRunId}`, + requestId: 'local', + details: { runId: namedRunId, timeoutSeconds: opts.timeoutSeconds }, + }, + }); + } + + if (exitCodeForRunStatus(namedResult.status) !== 0) { + stderrFn( + `Run finished with status: ${namedResult.status}. Use 'testsprite test artifact get ${namedRunId}' to download the failure bundle.`, + ); + throw new CLIError(`Run ${namedRunId} finished with status: ${namedResult.status}`, 1); + } + + // Fix B: timed-out closure members (non-named) are recorded in + // closureFailures with status 'timeout'. Even when the named run passes, + // we must exit 7 so --wait does not silently succeed when the closure + // as a whole was never observed to reach terminal. + const timedOutMembers = closureFailures.filter(f => f.status === 'timeout'); + if (timedOutMembers.length > 0) { + const resumeHints = timedOutMembers.map(f => `testsprite test wait ${f.runId}`).join('\n'); + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', + message: `${timedOutMembers.length} closure member${timedOutMembers.length !== 1 ? 's' : ''} timed out before reaching terminal status.`, + nextAction: resumeHints, + requestId: 'local', + details: { + timedOutRunIds: timedOutMembers.map(f => f.runId), + timeoutSeconds: opts.timeoutSeconds, + }, + }, + }); + } + + return rerunResp; + } + + // Single FE rerun (or BE without closure) — poll the single runId + let beFallbackUsed = false; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => testId, + resolveNotBefore: () => rerunResp.enqueuedAt, + onResolved: tid => { + beFallbackUsed = true; + stderrFn( + `[advisory] Backend run-surface row is not finalized server-side; ` + + `resolved the verdict from the test record (testId=${tid}).`, + ); + }, + }); + + let finalRun: RunResponse; + try { + finalRun = await pollRunUntilTerminal(client, rerunResp.runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.update( + `Run ${run.runId} — ${run.status} (${s.completed}/${s.total} steps replay elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + } catch (err) { + if (err instanceof TimeoutError) { + ticker.finalize(`Run ${rerunResp.runId} — timed out after ${opts.timeoutSeconds}s`); + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${rerunResp.runId}.`, + nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}`, + requestId: 'local', + details: { runId: rerunResp.runId, timeoutSeconds: opts.timeoutSeconds }, + }, + }); + } + // C: RequestTimeoutError during polling — emit partial through the same + // render path (text mode: human-readable, JSON mode: parseable envelope). + if (err instanceof RequestTimeoutError) { + ticker.finalize(`Run ${rerunResp.runId} — request timed out`); + const partial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (request timed out)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); + stderrFn( + `Run ${rerunResp.runId} is still in progress (request timed out). ` + + `Re-attach with: testsprite test wait ${rerunResp.runId}`, + ); + throw err; + } + ticker.finalize(); + throw err; + } + + const elapsed = Math.round(Date.now() / 1000); + void elapsed; + const s = finalRun.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.finalize( + `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps replay)`, + ); + + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => + renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + ); + + if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + stderrFn( + `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + ); + } + + const exitCode = exitCodeForRunStatus(finalRun.status); + if (exitCode !== 0) { + throw new CLIError( + `Run ${finalRun.runId} finished with status: ${finalRun.status}`, + exitCode, + ); + } + + return rerunResp; + } + + // ------------------------------------------------------------------------- + // Batch / --all rerun path + // ------------------------------------------------------------------------- + let testIds = opts.testIds; + + if (opts.all) { + // Validate --status filter before any network call. + if (opts.statusFilter !== undefined) { + validateStatusFilter(opts.statusFilter); + } + + // Resolve all tests in the project — follow nextToken until exhausted so + // projects with >1 service page (>25 tests) are fully covered. + const allPage = await paginate( + async ({ pageSize, cursor }) => + client.get>('/tests', { + query: { projectId: opts.projectId!, pageSize, cursor }, + }), + {}, + ); + let allTests = allPage.items; + + // --skip-terminal: exclude tests already in a terminal status so an + // interrupted sweep doesn't re-replay finished tests. + if (opts.skipTerminal) { + const before = allTests.length; + allTests = allTests.filter(t => !TERMINAL_PUBLIC_STATUSES.has(t.status)); + const skipped = before - allTests.length; + if (skipped > 0) { + stderrFn( + `--skip-terminal: skipped ${skipped} already-terminal test${skipped !== 1 ? 's' : ''} (passed|failed|blocked|cancelled).`, + ); + } + } + + // --status : only dispatch tests whose status matches one of the + // listed values. Tokens are already validated above. + if (opts.statusFilter !== undefined && opts.statusFilter !== '') { + const allowed = new Set( + opts.statusFilter + .split(',') + .map(s => s.trim()) + .filter(s => s.length > 0), + ); + const before = allTests.length; + allTests = allTests.filter(t => allowed.has(t.status)); + const skipped = before - allTests.length; + if (skipped > 0) { + stderrFn( + `--status filter: skipped ${skipped} test${skipped !== 1 ? 's' : ''} not matching status=${opts.statusFilter}.`, + ); + } + } + + // --filter : only dispatch tests whose name contains the + // substring (case-insensitive). Applied after --skip-terminal and --status. + if (opts.nameFilter !== undefined && opts.nameFilter !== '') { + const needle = opts.nameFilter.toLowerCase(); + const before = allTests.length; + allTests = allTests.filter(t => t.name.toLowerCase().includes(needle)); + const skipped = before - allTests.length; + if (skipped > 0) { + stderrFn( + `--filter: skipped ${skipped} test${skipped !== 1 ? 's' : ''} whose name does not contain "${opts.nameFilter}".`, + ); + } + } + + testIds = allTests.map(t => t.id); + if (testIds.length === 0) { + stderrFn(`No tests found in project ${opts.projectId} matching filters — nothing to rerun.`); + out.print({ accepted: [], deferred: [], conflicts: [], closure: { byProject: [] } }); + return undefined; + } + stderrFn( + `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch rerun.`, + ); + } + + // Fix D: chunk testIds to stay within the MAX_BATCH_RERUN_IDS (50) cap on + // POST /tests/batch/rerun. When --all resolves >50 tests we issue one + // request per chunk (distinct idempotency-key per chunk so retries are + // safe) and aggregate accepted/deferred/conflicts/closure into a single + // synthetic BatchRerunResponse that downstream --wait / exit-code logic + // can treat as one result. + const chunks: string[][] = []; + for (let i = 0; i < testIds.length; i += MAX_BATCH_RERUN_IDS) { + chunks.push(testIds.slice(i, i + MAX_BATCH_RERUN_IDS)); + } + if (chunks.length === 0) chunks.push([]); // defensive: empty list handled above + + let chunkResponses: BatchRerunResponse[]; + try { + chunkResponses = await Promise.all( + chunks.map((chunk, idx) => { + const chunkKey = chunks.length === 1 ? idempotencyKey : `${idempotencyKey}:chunk${idx}`; + return client.triggerBatchRerun( + { + source: 'cli', + testIds: chunk, + ...(effectiveAutoHeal ? { autoHeal: true } : {}), + ...(opts.skipDependencies ? { skipDependencies: true } : {}), + }, + { idempotencyKey: chunkKey }, + ); + }), + ); + } catch (err) { + // D2 (dogfood): the batch endpoint rejects the WHOLE request when any id is + // unresolvable (unknown, cross-tenant, or never ran cleanly), so one bad id + // aborts the batch with NOT_FOUND. Replace the bare exit-4 with an + // actionable hint. (Server-side partial-accept of unknown ids — a + // `notFound[]` in the batch response so good ids still run — is a tracked + // backend follow-up.) + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Batch rerun aborted: one or more of the ${testIds.length} requested test${testIds.length !== 1 ? 's' : ''} has no replayable run (unknown/cross-tenant id, or never completed a clean run). The batch endpoint rejects the whole request when any id is unresolvable.`, + nextAction: `Verify the test ids and drop any that have never run, or trigger fresh runs individually: testsprite test run `, + requestId: err.requestId ?? 'local', + details: { reason: 'batch_contains_unreplayable', testIds }, + }, + }); + } + throw err; + } + + // Aggregate chunk responses into a single synthetic BatchRerunResponse. + const batchResp: BatchRerunResponse = { + accepted: chunkResponses.flatMap(r => r.accepted), + deferred: chunkResponses.flatMap(r => r.deferred), + conflicts: chunkResponses.flatMap(r => r.conflicts), + closure: { + byProject: chunkResponses.flatMap(r => r.closure.byProject), + }, + notFound: chunkResponses.flatMap(r => r.notFound ?? []), + }; + + // Print dispatch summary + // Mutable: D3 deferred-retry loop may append to `accepted`/`conflicts` and + // drain `deferred` under --wait. + let accepted = batchResp.accepted.slice(); + let deferred = batchResp.deferred.slice(); + let conflicts = batchResp.conflicts.slice(); + // [P2] `notFound` is mutable: a deferred test may become un-replayable during + // the retry window; the retry response's notFound[] is merged into this set so + // the test is never reported as "resolved" when it actually vanished. + let notFound = (batchResp.notFound ?? []).slice(); + const closureByProject = batchResp.closure.byProject; + const addedProducersTotal = closureByProject.reduce((n, p) => n + p.addedProducers.length, 0); + + const summaryParts: string[] = [ + `Reran ${accepted.length} test${accepted.length !== 1 ? 's' : ''}`, + ]; + if (addedProducersTotal > 0) { + summaryParts[0] += ` (${addedProducersTotal} BE producer${addedProducersTotal !== 1 ? 's' : ''} auto-added)`; + } + if (conflicts.length > 0) { + summaryParts.push(`${conflicts.length} already in flight, skipped`); + } + if (deferred.length > 0) { + summaryParts.push(`${deferred.length} rate-deferred`); + } + if (notFound.length > 0) { + summaryParts.push(`${notFound.length} not found, skipped`); + } + stderrFn(summaryParts.join('; ')); + + // D2-CLI: warn about notFound ids so the operator knows which tests were + // skipped while the remaining accepted ids were still dispatched. Mirror + // the style of the deferred warning block above. + if (notFound.length > 0) { + stderrFn( + `[warn] ${notFound.length} test id${notFound.length !== 1 ? 's' : ''} skipped (unknown/cross-tenant id, or test never completed a clean run):`, + ); + for (const id of notFound) stderrFn(` ${id}`); + stderrFn( + ` Skipped ids have no replayable run. Use 'testsprite test run ' for a first (fresh) run.`, + ); + } + + if (deferred.length > 0) { + stderrFn(`Rate-deferred testIds (retry later):`); + for (const d of deferred) stderrFn(` ${d.testId} (reason: ${d.reason})`); + const deferredIds = deferred.map(d => d.testId).join(' '); + stderrFn(`nextAction: testsprite test rerun ${deferredIds}`); + } + + // D3: bounded deferred-retry loop for rerun --all (only under --wait). + // Up to MAX_DEFERRED_RETRIES attempts to re-dispatch still-deferred tests. + // Each attempt sleeps for Retry-After (if server provided it) or the default + // 61s, clamped to the remaining --timeout budget. Newly-accepted runs are + // merged into `accepted`; if still deferred after all attempts, fall through + // to the existing exit-7 path. + const sleepFn = deps.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const batchDeadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + if (opts.wait) { + for (let attempt = 1; attempt <= MAX_DEFERRED_RETRIES && deferred.length > 0; attempt++) { + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + stderrFn( + `[deferred-retry] timeout budget exhausted before attempt ${attempt}/${MAX_DEFERRED_RETRIES} — ${deferred.length} test${deferred.length !== 1 ? 's' : ''} still deferred.`, + ); + break; + } + const sleepMs = Math.min(DEFERRED_RETRY_DEFAULT_SLEEP_MS, remainingMs); + stderrFn( + `[deferred-retry] attempt ${attempt}/${MAX_DEFERRED_RETRIES} — retrying ${deferred.length} deferred test${deferred.length !== 1 ? 's' : ''} in ${Math.round(sleepMs / 1000)}s`, + ); + await sleepFn(sleepMs); + + const remainingAfterSleep = batchDeadlineMs - Date.now(); + if (remainingAfterSleep <= 0) { + stderrFn( + `[deferred-retry] timeout budget exhausted during sleep — ${deferred.length} test${deferred.length !== 1 ? 's' : ''} still deferred.`, + ); + break; + } + + // Chunk the retry ids to stay within MAX_BATCH_RERUN_IDS cap. + const retryIds = deferred.map(d => d.testId); + const retryChunks: string[][] = []; + for (let i = 0; i < retryIds.length; i += MAX_BATCH_RERUN_IDS) { + retryChunks.push(retryIds.slice(i, i + MAX_BATCH_RERUN_IDS)); + } + + let retryChunkResponses: BatchRerunResponse[]; + try { + retryChunkResponses = await Promise.all( + retryChunks.map((chunk, idx) => { + // [P2] Bound the derived key to ≤256 chars. Caller-supplied keys may + // be up to 256 chars; appending the suffix could exceed the server + // limit and cause every retry to be rejected. Truncate the base key + // to leave room for the longest possible suffix before concatenating. + const retrySuffix = + retryChunks.length === 1 + ? `:deferred-retry${attempt}` + : `:deferred-retry${attempt}:chunk${idx}`; + const retryBase = + idempotencyKey.length + retrySuffix.length > 256 + ? idempotencyKey.slice(0, 256 - retrySuffix.length) + : idempotencyKey; + const retryKey = `${retryBase}${retrySuffix}`; + return client.triggerBatchRerun( + { + source: 'cli', + testIds: chunk, + ...(effectiveAutoHeal ? { autoHeal: true } : {}), + ...(opts.skipDependencies ? { skipDependencies: true } : {}), + }, + { idempotencyKey: retryKey }, + ); + }), + ); + } catch (err) { + stderrFn( + `[deferred-retry] attempt ${attempt} failed with error: ${err instanceof Error ? err.message : String(err)}`, + ); + break; + } + + const newlyAccepted = retryChunkResponses.flatMap(r => r.accepted); + const newlyDeferred = retryChunkResponses.flatMap(r => r.deferred); + const newlyConflicted = retryChunkResponses.flatMap(r => r.conflicts); + // [P2] Collect notFound[] from the retry response. A deferred test may be + // un-replayable by the time we retry (e.g. the test was deleted). Merge + // into the running notFound set and remove from deferred so it isn't + // reported as "resolved" in the final output. + const newlyNotFound = retryChunkResponses.flatMap(r => r.notFound ?? []); + + if (newlyAccepted.length > 0) { + stderrFn( + `[deferred-retry] attempt ${attempt}: ${newlyAccepted.length} test${newlyAccepted.length !== 1 ? 's' : ''} now accepted.`, + ); + accepted = accepted.concat(newlyAccepted); + } + if (newlyConflicted.length > 0) { + // [P1] Merge retry-returned conflicts into the running conflicts collection + // so the final summary, stderr output, and exit-code logic reflect them. + stderrFn( + `[deferred-retry] attempt ${attempt}: ${newlyConflicted.length} test${newlyConflicted.length !== 1 ? 's' : ''} in-flight (conflict).`, + ); + conflicts = conflicts.concat(newlyConflicted); + } + if (newlyNotFound.length > 0) { + // [P2] Merge retry-discovered notFound ids and warn so the operator knows + // which tests vanished. Remove the now-un-replayable ids from `deferred` + // (newlyDeferred is the authoritative post-retry deferred set — it won't + // include these ids — but be explicit so the logic is clear). + stderrFn( + `[deferred-retry] attempt ${attempt}: ${newlyNotFound.length} test id${newlyNotFound.length !== 1 ? 's' : ''} not found on retry (deleted or never ran cleanly): ${newlyNotFound.join(' ')}`, + ); + notFound = notFound.concat(newlyNotFound); + // Warn via the standard notFound stderr block (mirrors the initial dispatch). + for (const id of newlyNotFound) stderrFn(` ${id}`); + stderrFn( + ` Skipped ids have no replayable run. Use 'testsprite test run ' for a first (fresh) run.`, + ); + } + deferred = newlyDeferred; + if (deferred.length === 0) { + stderrFn(`[deferred-retry] attempt ${attempt}: all previously-deferred tests accepted.`); + } + } + } + + if (!opts.wait) { + // [P2] Build output from post-retry mutable state so deferred/conflicts/notFound + // reflect what the D3 loop discovered, not just the initial batchResp. + out.print({ ...batchResp, accepted, deferred, conflicts, notFound }); + if (deferred.length > 0) { + throw new CLIError( + `Batch rerun incomplete: ${deferred.length} test${deferred.length !== 1 ? 's' : ''} were rate-deferred. Retry with: testsprite test rerun ${deferred.map(d => d.testId).join(' ')}`, + 7, + ); + } + // Fix C: all-conflict no-op (no --wait path) + if (accepted.length === 0 && conflicts.length > 0) { + // codex P2: don't claim "all in flight" when some ids were also notFound — + // a mixed conflicts+notFound response with no accepted runs must report + // both causes accurately and surface the notFound ids in details. + throw ApiError.fromEnvelope({ + error: { + code: 'CONFLICT', + message: `Batch rerun: nothing was queued — ${conflicts.length} test${conflicts.length !== 1 ? 's' : ''} already in flight${notFound.length > 0 ? `, ${notFound.length} not found` : ''}.`, + nextAction: `Wait for the in-flight runs to complete, then retry, or use: testsprite test wait `, + requestId: 'local', + details: { + conflicts: conflicts.map(c => ({ testId: c.testId, currentRunId: c.currentRunId })), + ...(notFound.length > 0 ? { notFound } : {}), + }, + }, + }); + } + // [P2] Return post-retry state including merged notFound. + return { ...batchResp, accepted, deferred, conflicts, notFound }; + } + + // --wait: fan-out poll each accepted run by its runId + if (accepted.length === 0) { + // [P2] Build output from post-retry mutable state including merged notFound. + out.print({ ...batchResp, accepted, deferred, conflicts, notFound }); + if (deferred.length > 0) { + throw new CLIError( + `Batch rerun: no tests were accepted (${deferred.length} deferred). ` + + `Retry with: testsprite test rerun ${deferred.map(d => d.testId).join(' ')}`, + 7, + ); + } + // Fix C: all-conflict no-op (--wait path) + if (conflicts.length > 0) { + // codex P2: mixed conflicts+notFound (no accepted runs) must not be + // reported as "all in flight"; surface the notFound ids in details too. + throw ApiError.fromEnvelope({ + error: { + code: 'CONFLICT', + message: `Batch rerun: nothing was queued — ${conflicts.length} test${conflicts.length !== 1 ? 's' : ''} already in flight${notFound.length > 0 ? `, ${notFound.length} not found` : ''}.`, + nextAction: `Wait for the in-flight runs to complete, then retry, or use: testsprite test wait `, + requestId: 'local', + details: { + conflicts: conflicts.map(c => ({ testId: c.testId, currentRunId: c.currentRunId })), + ...(notFound.length > 0 ? { notFound } : {}), + }, + }, + }); + } + // [P2] Return post-retry state including merged notFound. + return { ...batchResp, accepted, deferred, conflicts, notFound }; + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const concurrencyLimit = opts.maxConcurrency; + const rerunResults: CliRerunResult[] = []; + // sleepFn is declared above in the D3 deferred-retry section (shared by fan-out). + + async function pollAccepted(entry: BatchRerunAccepted): Promise { + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => entry.testId, + resolveNotBefore: () => entry.enqueuedAt, + onResolved: () => undefined, + }); + try { + // [P2] Use remaining time against the shared batch deadline rather than the + // full opts.timeoutSeconds. Without this, a run that starts polling after + // up to ~183s of retry sleeps still gets the full --timeout budget, so total + // wall time can exceed the documented --timeout ceiling. Mirror the same + // pattern used by pollFreshAccepted in runTestRunAll. + const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const finalRun = await pollRunUntilTerminal(client, entry.runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + const s = run.stepSummary ?? { total: 0, completed: 0, passedCount: 0, failedCount: 0 }; + ticker.update( + `Run ${run.runId} (${entry.testId}) — ${run.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, + ); + }, + resolveAlternate, + }); + return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + } catch (err) { + if (err instanceof TimeoutError) { + return { + testId: entry.testId, + runId: entry.runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + if (err instanceof ApiError) { + return { + testId: entry.testId, + runId: entry.runId, + status: 'error', + error: { code: err.code, message: err.message, exitCode: 1 }, + }; + } + throw err; + } + } + + // Bounded concurrency fan-out + let acceptedIdx = 0; + let inFlight = 0; + + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { + const entry = accepted[acceptedIdx++]!; + inFlight++; + pollAccepted(entry) + .then(result => { + rerunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (accepted.length === 0) resolve(); + }); + + ticker.finalize(); + + const passed = rerunResults.filter(r => r.status === 'passed').length; + const failed = rerunResults.filter(r => r.status !== 'passed' && r.status !== 'timeout').length; + const timedOut = rerunResults.filter(r => r.status === 'timeout').length; + + stderrFn( + `Batch rerun complete: ${passed}/${accepted.length} passed, ${failed} failed/blocked, ${timedOut} timed out`, + ); + + const jsonPayload = { + accepted: rerunResults, + // [P2] Use post-retry mutable vars, not the stale initial batchResp fields. + // batchResp.deferred/conflicts reflect only the INITIAL response; after D3 + // retries drain deferred and may accumulate conflicts, the mutable `deferred` + // and `conflicts` vars are the authoritative post-retry state. + deferred, + conflicts, + // D2-CLI (codex P2): the --wait path builds its own jsonPayload, so it must + // carry `notFound` too — otherwise a partial batch with at least one + // accepted run drops the skipped ids from JSON output and a consumer would + // report the partial run as fully successful. Mirrors the non-wait + // `out.print(batchResp)` path. + notFound, + closure: batchResp.closure, + summary: { + passed, + failed, + timedOut, + // D3 (dogfood): surface deferred + conflicts + notFound in the summary so + // a JSON consumer reading `summary` alone can't silently undercount — + // `total` counts dispatched (accepted) runs only. requested = total + + // deferred + conflicts + notFound. + deferred: deferred.length, + conflicts: conflicts.length, + notFound: notFound.length, + total: accepted.length, + }, + }; + out.print(jsonPayload); + + // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 + if (deferred.length > 0 || timedOut > 0) { + const stillRunning = + timedOut > 0 ? rerunResults.filter(r => r.status === 'timeout').map(r => r.runId) : []; + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', + message: [ + deferred.length > 0 ? `${deferred.length} test(s) were rate-deferred.` : '', + timedOut > 0 ? `${timedOut} run(s) timed out.` : '', + ] + .filter(Boolean) + .join(' '), + nextAction: [ + deferred.length > 0 + ? `testsprite test rerun ${deferred.map(d => d.testId).join(' ')}` + : '', + // `test wait` accepts exactly one run id — emit one command per + // timed-out run so the hint is always valid. + ...(timedOut > 0 ? stillRunning.map(rid => `Resume: testsprite test wait ${rid}`) : []), + ] + .filter(Boolean) + .join('\n'), + requestId: 'local', + details: { deferredTestIds: deferred.map(d => d.testId), timedOutRunIds: stillRunning }, + }, + }); + } + + if (failed > 0) { + throw new CLIError(`${failed} rerun${failed !== 1 ? 's' : ''} failed.`, 1); + } + + // [P2] Return post-retry state including merged notFound so callers see the + // final accounting (accepted = original BatchRerunAccepted[] dispatch list + // as required by the BatchRerunResponse type; rerunResults is the polled + // outcome printed to stdout and is not part of the returned shape). + return { ...batchResp, accepted, deferred, conflicts, notFound }; +} + +// --------------------------------------------------------------------------- +// M3.3 piece-4 — `test artifact get ` +// --------------------------------------------------------------------------- + +export interface ArtifactGetOptions extends CommonOptions { + runId: string; + /** + * Directory to write the §7 disk layout into. When absent, defaults to + * `./.testsprite/runs//` (computed at action time from `process.cwd()`). + * The default is intentionally not stored here to ensure it is computed freshly + * at action time; pass the resolved path when you want an explicit directory. + */ + out?: string; + /** §7.4 — keep only the failed step ± 1 in `steps[]` and `evidence[]`. */ + failedOnly: boolean; +} + +export interface ArtifactGetResult { + /** The wire envelope as returned by the facade. */ + context: CliFailureContext; + /** Set when bundle was written to disk. */ + bundle?: WriteBundleResult; +} + +/** + * Validate that the parent directory of `resolvedDir` exists and is a + * directory. Surfaces `VALIDATION_ERROR` (exit 5) — matches the convention + * from `closeOutputFile` for single-file `--out` flags (P4 D4 convention). + * + * The bundle directory itself (`resolvedDir`) may or may not exist; + * `writeBundle` creates it if absent. + */ +export async function assertOutDirParentExists(resolvedDir: string): Promise { + const parent = dirname(resolvedDir); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('out', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('out', `parent path is not a directory: ${parent}`); + } + // Also guard against --out pointing at an existing FILE (not a dir). + let targetStat; + try { + targetStat = await stat(resolvedDir); + } catch { + // Does not exist yet — fine; writeBundle will create it. + return; + } + if (!targetStat.isDirectory()) { + throw localValidationError('out', `must point to a directory, not a file: ${resolvedDir}`); + } +} + +/** + * `test artifact get ` — run-scoped failure-bundle download. + * + * Downloads `GET /api/cli/v1/runs/{runId}/failure` and either: + * - Writes the §7 disk layout under `` (default `./.testsprite/runs//`) + * - Or prints the wire envelope / human summary to stdout when `--out` is absent. + * + * Differences from M2 `test failure get`: + * - Addresses the bundle by `runId` (exact run) not `testId` (latest). + * - Enforces `meta.runId === ` as a cross-check against backend bugs. + * - Passes `{ requireRunId: true }` to `assertContextIntegrity`. + */ +export async function runArtifactGet( + opts: ArtifactGetOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + const { runId } = opts; + + // Resolve output dir: explicit --out or the default .testsprite/runs// + const resolvedDir = + opts.out !== undefined + ? resolveBundleDir(opts.out) + : join(process.cwd(), '.testsprite', 'runs', runId); + + // --dry-run: no network, no disk write. + // The client (makeClient) is already wired with createDryRunFetch() when + // dryRun: true, so a real call to client.get() would return the canned + // sample. We replicate that here without touching credentials, the + // network, or the filesystem. + if (opts.dryRun) { + const sample = findSample('GET', `/api/cli/v1/runs/${encodeURIComponent(runId)}/failure`); + const cannedCtx = (sample?.body() ?? {}) as CliFailureContext; + const cannedMeta = buildMeta(cannedCtx, new Date()); + + if (opts.output === 'json') { + // Emit the same schema shape as the real success path so automation + // that learns the surface via dry-run sees the correct keys. + out.print({ + out: resolvedDir, + snapshotId: cannedMeta.snapshotId, + meta: { + runId: cannedCtx.result?.runIdIfAvailable ?? null, + testId: cannedMeta.testId, + projectId: cannedMeta.projectId, + codeVersion: cannedMeta.codeVersion, + targetUrl: cannedMeta.targetUrl, + failedStepIndex: cannedMeta.failedStepIndex, + failureKind: cannedMeta.failureKind, + capturedAt: cannedMeta.capturedAt, + fetchedAt: cannedMeta.fetchedAt, + }, + }); + } else { + out.print({ dir: resolvedDir, files: 0, snapshotId: cannedMeta.snapshotId, runId }, data => + renderArtifactGetDryRunText( + data as { dir: string; files: number; snapshotId: string; runId: string }, + ), + ); + } + return { context: cannedCtx }; + } + + // Parent-dir validation for explicit --out only. The default path + // (.testsprite/runs//) is always under cwd — mkdir will create it. + if (opts.out !== undefined) { + await assertOutDirParentExists(resolvedDir); + } + + // Fetch the run-scoped failure bundle. + const { body: context, requestId: fetchRequestId } = await client.getWithMeta( + `/runs/${encodeURIComponent(runId)}/failure`, + ); + + // §3 atomicity invariants — run-scoped path requires runId to be present. + assertContextIntegrity(context, 'local', { requireRunId: true }); + + // Verify the backend returned the exact runId we asked for. + // A mismatch is a backend bug; refuse rather than silently writing the wrong bundle. + if (context.result.runIdIfAvailable !== runId) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Bundle integrity check failed.', + nextAction: + 'The server returned a bundle for a different runId. ' + + 'Report the requestId to support@testsprite.com.', + requestId: 'local', + details: { + field: 'meta.runId', + reason: 'mismatch', + expected: runId, + received: context.result.runIdIfAvailable, + }, + }, + }); + } + + // Write bundle to disk. + const bundle = await writeBundle(context, { + dir: resolvedDir, + failedOnly: opts.failedOnly, + fetchImpl: deps.fetchImpl, + }); + + if (opts.output === 'json') { + out.print({ + out: bundle.dir, + snapshotId: bundle.meta.snapshotId, + requestId: fetchRequestId, + meta: { + runId: context.result.runIdIfAvailable, + testId: bundle.meta.testId, + projectId: bundle.meta.projectId, + codeVersion: bundle.meta.codeVersion, + targetUrl: bundle.meta.targetUrl, + failedStepIndex: bundle.meta.failedStepIndex, + failureKind: bundle.meta.failureKind, + capturedAt: bundle.meta.capturedAt, + fetchedAt: bundle.meta.fetchedAt, + }, + }); + } else { + out.print( + { dir: bundle.dir, files: bundle.files.length, snapshotId: bundle.meta.snapshotId, runId }, + data => + renderArtifactGetWrittenText( + data as { dir: string; files: number; snapshotId: string; runId: string }, + ), + ); + if (opts.verbose || opts.debug) { + const stderrWriter = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrWriter(`requestId: ${fetchRequestId}`); + } + } + return { context, bundle }; +} + +function renderArtifactGetDryRunText(data: { + dir: string; + files: number; + snapshotId: string; + runId: string; +}): string { + return [ + `[dry-run] no network call made`, + `method: GET`, + `path: /api/cli/v1/runs/${data.runId}/failure`, + `writeTo: ${data.dir}`, + `snapshotId: ${data.snapshotId}`, + ].join('\n'); +} + +function renderArtifactGetWrittenText(data: { + dir: string; + files: number; + snapshotId: string; + runId: string; +}): string { + return [ + `Bundle written to ${data.dir}`, + `runId: ${data.runId}`, + `snapshotId: ${data.snapshotId}`, + `files: ${data.files}`, + ].join('\n'); +} + +export function createTestCommand(deps: TestDeps = {}): Command { + const test = new Command('test').description('Inspect TestSprite tests'); + + test + .command('list') + .description('List tests in a project') + // Intentionally NOT `.requiredOption` — Commander's missing-required-option + // path throws a plain Error and `index.ts` maps it to exit 1, which would + // bypass the typed `VALIDATION_ERROR` (exit 5) envelope contract from + // the CLI error spec §2 ("missing required field"). `requireProjectId` + // below raises `ApiError(VALIDATION_ERROR)` so JSON consumers can read + // `error.code` and the exit code matches the catalog. + .option('--project ', 'project id (returned by `testsprite project list`)') + .option('--type ', 'filter by test type (frontend|backend)') + .option('--created-from ', 'filter by where the test was authored (portal|mcp|cli)') + .option( + '--status ', + 'filter by normalized status (comma-separated). One of: draft, ready, queued, running, passed, failed, blocked, cancelled, unknown — M2.1', + ) + .option('--page-size ', 'service page-size hint (1-100, default 25)') + .option('--starting-token ', 'opaque cursor from a previous list response') + .option( + '--cursor ', + 'alias for --starting-token; accepted for parity with `test result --history`', + ) + .option('--max-items ', 'stop after this many items across auto-paged pages') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ListFlagOpts, command: Command) => { + // Same parser strategy as `project list`: skip Commander's number + // parser so a non-numeric --page-size surfaces as a typed + // VALIDATION_ERROR (exit 5) rather than Commander's plain + // exception (exit 1). Enum filters validate locally too. + // + // --cursor is an alias for --starting-token (vocabulary parity with + // `test result --history`). --starting-token takes precedence if + // both are supplied (prevents accidental override). + await runList( + { + ...resolveCommonOptions(command), + projectId: cmdOpts.project, + type: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES), + createdFrom: parseEnumFlag(cmdOpts.createdFrom, 'created-from', CREATED_FROMS), + status: cmdOpts.status, + pageSize: parseNumericFlag(cmdOpts.pageSize, 'page-size'), + startingToken: cmdOpts.startingToken ?? cmdOpts.cursor, + maxItems: parseNumericFlag(cmdOpts.maxItems, 'max-items'), + }, + deps, + ); + }); + + test + .command('get ') + .description('Get a test by id') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, _cmdOpts, command: Command) => { + await runGet({ ...resolveCommonOptions(command), testId }, deps); + }); + + test + .command('create') + .description( + 'Create a test from saved code (--code-file) or an agent-supplied plan (--plan-from, FE-only, M3.2 piece-5)', + ) + .option('--project ', 'project id (returned by `testsprite project list`)') + .option('--type ', 'frontend|backend') + .option('--name ', 'human-readable test name (becomes `title` in storage)') + .option('--description ', 'optional human description (≤ 2000 chars)') + .option('--priority ', 'optional priority — one of: p0, p1, p2, p3') + .option('--code-file ', 'file containing the test code (≤ 350 KB)') + .option( + '--plan-from ', + 'JSON file with the full FE test definition — projectId, type, name, planSteps[] all live in the file ' + + '(≤ 256 KB; mutually exclusive with --code-file). In this mode --project/--type/--name/--description/--priority are ignored.', + ) + .option( + '--run', + 'after create, trigger the test. Combine with --wait to block until terminal.', + false, + ) + .option('--wait', 'with --run, poll until terminal status', false) + .option('--timeout ', 'with --run --wait, max seconds to wait') + .option('--target-url ', 'with --run, override the project default env URL') + .option( + '--idempotency-key ', + 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .option( + '--produces ', + 'BE only: variable name this test captures (repeatable). Drives dependency-aware wave ordering on `test rerun` and `test run --all`.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--needs ', + 'BE only: variable name this test consumes (repeatable). Use to declare upstream producer dependencies.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--category ', + "BE only: test category. Use 'teardown' or 'cleanup' to mark a final-wave cleanup test.", + ) + .addHelpText( + 'after', + '\nBE dependency authoring (M4):\n' + + ' --produces/--needs drive wave ordering on `test rerun` + `test run --all`.\n' + + ' --category teardown marks a final-wave cleanup test.\n' + + ' These flags are backend-only; supplying with --type frontend is an error (exit 5).', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: CreateFlagOpts, command: Command) => { + // --plan-from and --code-file are mutually exclusive. Dispatch + // here so each `run*` function stays single-purpose. If neither + // is set, the existing runCreate path enforces --code-file. + if (cmdOpts.planFrom !== undefined && cmdOpts.codeFile !== undefined) { + throw localValidationError( + 'plan-from', + 'is mutually exclusive with --code-file; pass one or the other', + ); + } + if (cmdOpts.planFrom !== undefined) { + // BE dependency flags are backend-only (they drive the BE wave engine). + // --plan-from creates FE plan-steps tests, which have no wave model — so + // supplying --produces/--needs/--category here is a contradiction. Reject + // loudly (exit 5) rather than silently dropping the requested metadata, + // matching the `--type frontend` + dep-flags guard in runCreate (codex). + if ( + (cmdOpts.produces && cmdOpts.produces.length > 0) || + (cmdOpts.needs && cmdOpts.needs.length > 0) || + cmdOpts.category !== undefined + ) { + throw localValidationError( + 'produces', + '--produces/--needs/--category are backend-only and cannot be used with --plan-from (plan-steps tests are FE and have no dependency/wave model). Use --code-file --type backend to author a dependency-aware BE test.', + ); + } + // On the --plan-from path the test definition lives entirely + // inside the JSON file (projectId, type, name, description, + // priority, planSteps). Any of those flags supplied alongside + // --plan-from is silently dropped — collect them so runCreateFromPlan + // can warn the user AFTER the plan validates. Emitting the advisory + // here (before validation) made a missing-projectId failure look like + // the ignored --project flag was the cause (dogfood L1778); deferring + // it means a malformed plan fails fast with a clear `projectId` field + // error and no misleading warning lands first. + const ignored: string[] = []; + if (cmdOpts.project !== undefined) ignored.push('--project'); + if (cmdOpts.type !== undefined) ignored.push('--type'); + if (cmdOpts.name !== undefined) ignored.push('--name'); + if (cmdOpts.description !== undefined) ignored.push('--description'); + if (cmdOpts.priority !== undefined) ignored.push('--priority'); + await runCreateFromPlan( + { + ...resolveCommonOptions(command), + planFrom: cmdOpts.planFrom, + run: cmdOpts.run === true, + wait: cmdOpts.wait === true, + timeout: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + // B2(c): capture before parseTimeoutFlag converts undefined → default. + timeoutIsDefault: cmdOpts.timeout === undefined, + targetUrl: cmdOpts.targetUrl, + idempotencyKey: cmdOpts.idempotencyKey, + ignoredFlags: ignored, + }, + deps, + ); + return; + } + await runCreate( + { + ...resolveCommonOptions(command), + projectId: cmdOpts.project, + type: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) as 'frontend' | 'backend', + name: cmdOpts.name, + description: cmdOpts.description, + priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as + | CliCreatePriority + | undefined, + codeFile: cmdOpts.codeFile, + idempotencyKey: cmdOpts.idempotencyKey, + // M3.3 chain flags: + run: cmdOpts.run === true, + wait: cmdOpts.wait === true, + timeout: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + // B2(c): capture before parseTimeoutFlag converts undefined → default. + timeoutIsDefault: cmdOpts.timeout === undefined, + targetUrl: cmdOpts.targetUrl, + // M4 piece-2: BE dependency authoring flags. + // Commander variadic collectors initialise to [] — treat empty array as undefined + // so we don't send an empty array on the wire when no flags were passed. + produces: cmdOpts.produces && cmdOpts.produces.length > 0 ? cmdOpts.produces : undefined, + needs: cmdOpts.needs && cmdOpts.needs.length > 0 ? cmdOpts.needs : undefined, + category: cmdOpts.category, + }, + deps, + ); + }); + + test + .command('create-batch') + .description('Create multiple FE tests from a JSONL of plan specs (FE-only)') + .option('--plans ', 'JSONL file with one plan-from spec per line (≤ 50 specs, ≤ 5 MB)') + .option( + '--plan-from-dir ', + 'directory of *.json plan files — each file is one plan spec (≤ 50 files, ≤ 5 MB total). Sorted by filename for determinism. Mutually exclusive with --plans.', + ) + .option('--run', 'after create, trigger each created test as a run', false) + .option( + '--max-concurrency ', + 'with --run, max in-flight triggers at once (1-100, default: 50). The server caps run-triggers at 60/min/key; the CLI throttles to 50/min and auto-retries RATE_LIMITED responses client-side — raising this value does not bypass the server cap.', + ) + .option('--wait', 'with --run, poll each run until terminal status', false) + .option('--timeout ', 'with --run --wait, per-run max seconds to wait (1-3600, default 600)') + .option('--target-url ', 'with --run, override the project default env URL for each run') + .option( + '--idempotency-key ', + 'opaque idempotency token for the batch create (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: CreateBatchFlagOpts, command: Command) => { + await runCreateBatch( + { + ...resolveCommonOptions(command), + plans: cmdOpts.plans, + planFromDir: cmdOpts.planFromDir, + run: cmdOpts.run === true, + maxConcurrency: parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency'), + wait: cmdOpts.wait === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + targetUrl: cmdOpts.targetUrl, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + test + .command('steps ') + .description( + 'List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run)', + ) + .option('--page-size ', 'service page size hint (1-100, default 25)') + .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--starting-token ', 'opaque cursor from a previous response') + .option( + '--run-id ', + "Filter steps to those belonging to the specified runId. Useful for tests that have been run multiple times — by default 'test steps' returns the cumulative log across every run. Note: legacy step records (pre-M3.1) with null runIdIfAvailable are excluded when this flag is set.", + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: StepsFlagOpts, command: Command) => { + await runSteps( + { + ...resolveCommonOptions(command), + testId, + pageSize: parseNumericFlag(cmdOpts.pageSize, 'page-size'), + startingToken: cmdOpts.startingToken, + maxItems: parseNumericFlag(cmdOpts.maxItems, 'max-items'), + runId: cmdOpts.runId, + }, + deps, + ); + }); + + test + .command('result ') + .description( + 'Get the latest result for a test (default) or list prior runs (--history).\n' + + '\n--output json shape differs by mode:\n' + + ' (default) single CliLatestResult object\n' + + ' --history { runs: RunHistoryItem[], nextCursor: string|null }\n' + + '\nPer-run detail: testsprite test wait \n' + + 'Failure bundle: testsprite test artifact get ', + ) + .option( + '--include-analysis', + 'attach the inline `analysis` block (rootCauseHypothesis, recommendedFixTarget, failureKind, snapshotId) — M2.1', + false, + ) + .option('--history', 'list prior runs for this test instead of showing the latest result') + .option('--source ', `with --history: filter by trigger source (${RUN_SOURCES.join('|')})`) + .option( + '--since ', + 'with --history: lower bound on createdAt — 24h, 7d, or ISO timestamp (client-side translated)', + ) + .option('--page-size ', 'with --history: number of runs per page (1–100, default 20)') + .option('--cursor ', 'with --history: opaque cursor from a prior page') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: ResultFlagOpts, command: Command) => { + if (cmdOpts.history) { + // M3.4 piece-5: --history mode — list prior runs. + await runResultHistory( + { + ...resolveCommonOptions(command), + testId, + source: parseEnumFlag(cmdOpts.source, 'source', RUN_SOURCES) as RunSource | undefined, + since: cmdOpts.since, + pageSize: + cmdOpts.pageSize !== undefined + ? parseNumericFlag(cmdOpts.pageSize, 'page-size') + : undefined, + cursor: cmdOpts.cursor, + }, + deps, + ); + } else { + // M2 mode: latest result (byte-identical to pre-piece-5 behavior). + await runResult( + { + ...resolveCommonOptions(command), + testId, + includeAnalysis: cmdOpts.includeAnalysis === true, + }, + deps, + ); + } + }); + + test + .command('update ') + .description('Update test metadata — name, description, priority') + .option('--name ', 'new human-readable test name') + .option('--description ', 'new human description (≤ 2000 chars)') + .option('--priority ', 'new priority — one of: p0, p1, p2, p3') + .option( + '--idempotency-key ', + 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: UpdateFlagOpts, command: Command) => { + await runUpdate( + { + ...resolveCommonOptions(command), + testId, + name: cmdOpts.name, + description: cmdOpts.description, + priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as + | CliCreatePriority + | undefined, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + test + .command('delete ') + .description('Permanently delete a test. Requires --confirm. (M3.2 piece-3)') + .option('--confirm', 'required: explicit confirmation for the destructive operation', false) + .option( + '--idempotency-key ', + 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: DeleteFlagOpts, command: Command) => { + await runDelete( + { + ...resolveCommonOptions(command), + testId, + confirm: cmdOpts.confirm === true, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + // ------------------------------------------------------------------------- + // dogfood L1800 — `test delete-batch` (bulk soft-delete) + // ------------------------------------------------------------------------- + + test + .command('delete-batch [test-ids...]') + .description( + 'Permanently delete multiple tests in one command. Requires --confirm.\n' + + 'Use --all --project to delete all tests in a project (optionally filtered by --status).\n' + + '\nPrints a per-test summary (Deleted N, Skipped M, Failed K) to stdout.\n' + + '\nExit codes:\n' + + ' 0 all targeted tests deleted (or --dry-run)\n' + + ' 1 one or more deletions failed (server error)\n' + + ' 5 validation error (missing --confirm, missing --project with --all, etc.)\n' + + '\nNote: a 404 "not found" response is counted as skipped in the summary, not an error.', + ) + .option('--confirm', 'required: explicit confirmation for the destructive operation', false) + .option('--all', 'delete all tests in the resolved project (requires --project)', false) + .option( + '--project ', + 'project id (required with --all; returned by `testsprite project list`)', + ) + .option( + '--status ', + `with --all: only delete tests whose status matches these values (comma-separated; accepted: ${PUBLIC_STATUSES.join('|')})`, + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testIdsArg: string[], cmdOpts: DeleteBatchFlagOpts, command: Command) => { + await runDeleteBatch( + { + ...resolveCommonOptions(command), + testIds: testIdsArg ?? [], + all: cmdOpts.all === true, + projectId: cmdOpts.project, + statusFilter: cmdOpts.status, + confirm: cmdOpts.confirm === true, + }, + deps, + ); + }); + + // ------------------------------------------------------------------------- + // M3.3 piece-3 — `test run` and `test wait` + // ------------------------------------------------------------------------- + + test + .command('run [test-id]') + .description( + 'Trigger a test run. With --wait, polls until terminal status.\n' + + 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + + '\nExit codes:\n' + + ' 0 passed (or queued without --wait)\n' + + ' 1 failed / blocked / cancelled\n' + + ' 3 auth error\n' + + ' 4 test not found\n' + + ' 5 validation error (e.g., bad --target-url, or positional + --all both set)\n' + + ' 6 conflict (already running — see nextAction for the active runId)\n' + + ' 7 timeout — resume with: testsprite test wait \n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + + ' 11 rate limited — honor Retry-After\n' + + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + ) + .option( + '--target-url ', + 'override the project default env URL for this run (http/https only, no localhost/private IPs)', + ) + .option('--wait', 'poll until terminal status or --timeout elapses', false) + .option( + '--timeout ', + `with --wait, max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .option( + '--idempotency-key ', + 'opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated.', + ) + .option( + '--all', + 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + false, + ) + .option( + '--project ', + 'project id (required with --all; returned by `testsprite project list`)', + ) + .option( + '--filter ', + 'with --all: only run tests whose name contains this substring (case-insensitive)', + ) + .option( + '--max-concurrency ', + `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, + ) + .addHelpText( + 'after', + '\nDependency-aware fresh run (M4):\n' + + ' testsprite test run --all --project run all BE tests in wave order\n' + + ' testsprite test run --all --project --filter name-glob subset\n' + + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + + '(see `testsprite test create --help` for details).', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { + const isAll = cmdOpts.all === true; + + // Mutual-exclusion: exactly one of positional vs --all must be set. + if (testIdArg !== undefined && isAll) { + throw localValidationError( + 'test-id', + 'positional and --all are mutually exclusive; use one or the other', + ); + } + if (testIdArg === undefined && !isAll) { + throw localValidationError( + 'test-id', + 'provide a , or use --all --project to run all BE tests in a project', + ); + } + // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). + // Without --all it would be SILENTLY ignored while the explicit + // still runs — defeating the caller's narrowing intent. Reject early. + if (cmdOpts.filter !== undefined && cmdOpts.filter !== '' && !isAll) { + throw localValidationError( + 'filter', + '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', + ); + } + + if (isAll) { + // --all path: wave-ordered fresh batch run. + if (!cmdOpts.project) { + throw localValidationError( + 'project', + '--all requires a project id — pass --project ', + ); + } + // --target-url has no effect on the --all batch path: it is BE-only + // (FE tests are skipped server-side) and a BE test's base URL is baked + // into its code. Silently dropping it could run the suite against an + // unintended environment in the caller's mind — reject loudly instead. + if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { + throw localValidationError( + 'target-url', + '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', + ); + } + await runTestRunAll( + { + ...resolveCommonOptions(command), + projectId: cmdOpts.project, + nameFilter: cmdOpts.filter, + wait: cmdOpts.wait === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + maxConcurrency: + parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? + DEFAULT_BATCH_RUN_CONCURRENCY, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + return; + } + + // Single test-id path (unchanged M3.3 behavior). + await runTestRun( + { + ...resolveCommonOptions(command), + testId: testIdArg!, + targetUrl: cmdOpts.targetUrl, + wait: cmdOpts.wait === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + // B2(c): tell runTestRun whether --timeout was explicitly provided. + timeoutIsDefault: cmdOpts.timeout === undefined, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + test + .command('wait ') + .description( + 'Wait for a run to reach a terminal status.\n' + + '\nExit codes:\n' + + ' 0 passed\n' + + ' 1 failed / blocked / cancelled\n' + + ' 3 auth error\n' + + ' 4 run not found\n' + + ' 7 timeout — resume with: testsprite test wait \n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + ) + .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { + await runTestWait( + { + ...resolveCommonOptions(command), + runId, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + }); + + // ------------------------------------------------------------------------- + // M3.4 piece-3 — `test rerun` + // ------------------------------------------------------------------------- + + test + .command('rerun [test-ids...]') + .description( + 'Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure.\n' + + '\nExit codes:\n' + + ' 0 passed (or queued without --wait)\n' + + ' 1 failed / blocked / cancelled\n' + + ' 3 auth error\n' + + ' 4 test not found\n' + + ' 5 validation error\n' + + ' 6 conflict (already running — see nextAction for the active runId)\n' + + ' 7 timeout or deferred — resume with: testsprite test wait \n' + + ' 11 rate limited — honor Retry-After\n' + + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + ) + .option('--all', 'rerun all tests in the resolved project (requires --project)', false) + .option( + '--project ', + 'project id (required with --all; returned by `testsprite project list`)', + ) + .option( + '--skip-terminal', + 'with --all: skip tests already in a terminal status (passed|failed|blocked|cancelled)', + false, + ) + .option( + '--status ', + `with --all: only dispatch tests whose status matches one of these values (comma-separated; accepted: ${PUBLIC_STATUSES.join('|')})`, + ) + .option( + '--filter ', + 'with --all: only rerun tests whose name contains this substring (case-insensitive)', + ) + .option('--wait', 'block until terminal status or --timeout elapses', false) + .option( + '--timeout ', + `with --wait, max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .option( + '--no-auto-heal', + 'opt out of AI heal-on-drift for this FE rerun (default: auto-heal is ON). Costs 0.2 credits per engage when a step has drifted. Ignored for backend tests.', + ) + .option( + '--skip-dependencies', + 'BE only: rerun only the named test without expanding the producer/teardown closure', + false, + ) + .option( + '--max-concurrency ', + `with --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, + ) + .option( + '--idempotency-key ', + 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', + ) + .addHelpText( + 'after', + '\nNotes:\n' + + ' • rerun replays a saved run/script and is MORE LENIENT than a fresh `test run`\n' + + ' (auto-heal can pass steps that have drifted) — for strict scoring/regression,\n' + + ' prefer `test run`. The two are not interchangeable for pass-rate measurement.\n' + + ' • Under --wait the per-request HTTP timeout is auto-raised to cover --timeout so a\n' + + ' slow trigger/poll under load is not cut at the 120s default (see --request-timeout).\n' + + ' • Batch --wait: rate-deferred tests appear in `deferred[]` and `summary.deferred`,\n' + + ' and force a non-zero exit — they are NOT counted in `summary.total` (dispatched only).', + ) + .addHelpText( + 'after', + '\nDry-run shape notes:\n' + + ' • --dry-run shows the BE rerun wire shape (includes `closure{}`); FE rerun responses\n' + + ' omit `closure` (or return it as null) since there is no dependency expansion.\n' + + ' • `autoHeal` defaults true for FE reruns; BE reruns ignore the field entirely.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testIdsArg: string[], cmdOpts: RerunFlagOpts, command: Command) => { + // Commander's `--no-auto-heal` pattern makes `cmdOpts.autoHeal` default + // `true` (unchanged by the user) and `false` when the user passes + // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so + // autoHealExplicit is always false in this design — the default-on value + // is never a deliberate user choice to opt in. + await runTestRerun( + { + ...resolveCommonOptions(command), + testIds: testIdsArg ?? [], + all: cmdOpts.all === true, + projectId: cmdOpts.project, + skipTerminal: cmdOpts.skipTerminal === true, + statusFilter: cmdOpts.status, + nameFilter: cmdOpts.filter, + wait: cmdOpts.wait === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + autoHeal: cmdOpts.autoHeal !== false, + autoHealExplicit: false, + skipDependencies: cmdOpts.skipDependencies === true, + maxConcurrency: + parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? + DEFAULT_BATCH_RUN_CONCURRENCY, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + test.addCommand(createTestCodeCommand(deps)); + test.addCommand(createTestPlanCommand(deps)); + test.addCommand(createTestFailureCommand(deps)); + test.addCommand(createTestArtifactCommand(deps)); + + return test; +} + +interface RunFlagOpts { + targetUrl?: string; + wait?: boolean; + timeout?: string; + idempotencyKey?: string; + /** M4 piece-2: batch fresh run flags. */ + all?: boolean; + project?: string; + filter?: string; + maxConcurrency?: string; +} + +interface WaitFlagOpts { + timeout?: string; +} + +interface RerunFlagOpts { + all?: boolean; + project?: string; + skipTerminal?: boolean; + status?: string; + filter?: string; + wait?: boolean; + timeout?: string; + autoHeal?: boolean; + skipDependencies?: boolean; + maxConcurrency?: string; + idempotencyKey?: string; +} + +interface UpdateFlagOpts { + name?: string; + description?: string; + priority?: string; + idempotencyKey?: string; +} + +interface DeleteFlagOpts { + confirm?: boolean; + idempotencyKey?: string; +} + +interface DeleteBatchFlagOpts { + confirm?: boolean; + all?: boolean; + project?: string; + status?: string; +} + +interface ResultFlagOpts { + includeAnalysis?: boolean; + /** M3.4 piece-5: switch to run-history mode. */ + history?: boolean; + /** Filter history by trigger source. */ + source?: string; + /** Filter history by lower bound on createdAt: 24h, 7d, or ISO timestamp. */ + since?: string; + /** History page size (1–100, default 20). */ + pageSize?: string; + /** Opaque pagination cursor from a prior page's nextCursor. */ + cursor?: string; +} + +interface CreateFlagOpts { + project: string; + type: string; + name: string; + description?: string; + planFrom?: string; + run?: boolean; + wait?: boolean; + timeout?: string; + targetUrl?: string; + priority?: string; + codeFile: string; + idempotencyKey?: string; + /** M4 piece-2: BE dependency authoring flags. */ + produces?: string[]; + needs?: string[]; + category?: string; +} + +interface CreateBatchFlagOpts { + plans: string; + planFromDir?: string; + run?: boolean; + maxConcurrency?: string; + wait?: boolean; + timeout?: string; + targetUrl?: string; + idempotencyKey?: string; +} + +interface ListFlagOpts { + project: string; + type?: string; + createdFrom?: string; + status?: string; + pageSize?: string; + startingToken?: string; + /** + * Alias for `--starting-token` accepted for vocabulary parity with + * `test result --history` which uses `--cursor`. Both flags are + * forwarded to the same pagination field; `--starting-token` takes + * precedence when both are supplied (unlikely in practice). + */ + cursor?: string; + maxItems?: string; +} + +interface StepsFlagOpts { + pageSize?: string; + startingToken?: string; + maxItems?: string; + runId?: string; +} + +function requireProjectId(projectId: string): void { + if (typeof projectId !== 'string' || projectId.length === 0) { + throw localValidationError('project', 'is required'); + } +} + +/** + * §6.6 / M2.1 piece 2 — validate the `--status ` flag client + * side. Empty / undefined → no filter. Each token must be a public + * status value; unknown tokens fail with VALIDATION_ERROR (exit 5) + * before the request hits the wire so a typo like `--status fail` + * gets a pointed error including the accepted set. + */ +function validateStatusFilter(raw: string | undefined): void { + if (raw === undefined || raw === '') return; + for (const token of raw.split(',')) { + const trimmed = token.trim(); + if (trimmed === '') continue; + if (!PUBLIC_STATUSES.includes(trimmed as CliPublicStatus)) { + throw localValidationError( + 'status', + `must be one of: ${PUBLIC_STATUSES.join(', ')} (comma-separated for multiple)`, + [...PUBLIC_STATUSES], + ); + } + } +} + +function parseEnumFlag( + raw: string | undefined, + flagName: string, + accepted: ReadonlyArray, +): T | undefined { + if (raw === undefined) return undefined; + if (!accepted.includes(raw as T)) { + throw localValidationError(flagName, `must be one of: ${accepted.join(', ')}`, [...accepted]); + } + return raw as T; +} + +function parseNumericFlag(raw: string | undefined, flagName: string): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n)) { + throw localValidationError(flagName, 'must be an integer'); + } + return n; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + // P2-8: validate --output before allowing silent fallback to 'text'. + // An invalid value (e.g. `--output yaml`) must exit 5 with a clear error + // rather than silently treating the request as text mode. + const rawOutput = globals.output; + if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { + throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); + } + return { + profile: globals.profile ?? 'default', + output: (globals.output as OutputMode | undefined) ?? 'text', + dryRun: globals.dryRun ?? false, + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * Returns `undefined` when the flag was not supplied (factory falls back to + * the env var / default). Silently clamps out-of-range values. + */ +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return undefined; + return Math.round(n * 1000); // seconds → milliseconds +} + +/** D4: headroom added on top of `--timeout` when deriving the per-request window under `--wait`. */ +const WAIT_REQUEST_TIMEOUT_CUSHION_MS = 5_000; + +/** + * D4 (dogfood CoderCup 2026-06-05): under `--wait` the user opts into a + * long-running operation bounded by `--timeout`. The default 120s per-request + * timeout can falsely cut a single trigger or long-poll request when the + * backend is slow under load (e.g. a large concurrent batch) — failing the + * command even though `--timeout` is large and the run finishes fine + * server-side. Raise the per-request window to cover `--timeout` (capped at + * {@link REQUEST_TIMEOUT_MAX_MS}, floored at the resolved default so we never + * lower it). The poll loop's own deadline-aware `AbortSignal` (poll.ts) still + * bounds the TOTAL wait to `--timeout`; non-wait callers are unchanged. + */ +export function resolveWaitRequestTimeoutMs(opts: { + wait?: boolean; + timeoutSeconds?: number; + requestTimeoutMs?: number; +}): number | undefined { + if (opts.wait !== true || opts.timeoutSeconds === undefined) return opts.requestTimeoutMs; + const base = opts.requestTimeoutMs ?? REQUEST_TIMEOUT_DEFAULT_MS; + const cover = Math.min( + opts.timeoutSeconds * 1000 + WAIT_REQUEST_TIMEOUT_CUSHION_MS, + REQUEST_TIMEOUT_MAX_MS, + ); + return Math.max(base, cover); +} + +function makeClient(opts: CommonOptions, deps: TestDeps): HttpClient { + return makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); +} + +function makeOutput(mode: OutputMode, deps: TestDeps): Output { + return new Output(mode, { + stdout: deps.stdout, + stderr: deps.stderr, + rawStdout: deps.rawStdout, + }); +} + +/** + * Internal handle for `--out ` writes. Wraps a Node WriteStream + * with a tracked `error` field so `closeOutputFile` can re-raise an + * async stream error (EACCES on a write, ENOSPC mid-stream, etc.) that + * was emitted between writes. + */ +interface FileSink { + readonly stream: WriteStream; + readonly path: string; + error: Error | null; +} + +/** + * Open the `--out` target before any network I/O so a permission/dir + * error fails fast. Synchronous open via `createWriteStream` doesn't + * actually open the descriptor until first write, so we don't surface + * EACCES/ENOENT here — instead the stream emits `'error'`, which we + * remember on the sink and re-throw at close time. The benefit of + * opening early is still real: invalid path strings (empty, `/dev/null` + * on a sandboxed fs, etc.) are caught before the API request goes out. + */ +function openOutputFile(rawPath: string): FileSink { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('out', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + // Defensive: reject obviously-bad paths up front (a directory string + // would fail later with EISDIR; that's a clearer 5/VALIDATION_ERROR + // surface than letting it crash mid-write with TransportError). + if (resolved.endsWith('/')) { + throw localValidationError('out', 'must point to a file, not a directory'); + } + // Validate the parent dir synchronously so a missing or non-directory + // parent surfaces as exit 5 / VALIDATION_ERROR rather than exit 1 / + // TRANSPORT_ERROR. Without this, an ENOENT/ENOTDIR fires asynchronously + // on first write and gets re-raised through `closeOutputFile` as a + // TransportError — an exit-code mismatch with the rest of `--out`'s + // input validation. + const parent = dirname(resolved); + let parentStat; + try { + parentStat = statSync(parent); + } catch { + throw localValidationError('out', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('out', `parent path is not a directory: ${parent}`); + } + const stream = createWriteStream(resolved, { encoding: 'utf8' }); + const sink: FileSink = { stream, path: resolved, error: null }; + stream.on('error', err => { + sink.error = err instanceof Error ? err : new Error(String(err)); + }); + return sink; +} + +/** + * Adapter that turns a `FileSink` into the `Output` writer set. Both + * `print` (line-oriented JSON) and `writeChunk` (raw bytes) flow into + * the same stream; backpressure is preserved on the chunk path by + * resolving the returned promise on `'drain'` when the kernel buffer + * is full, mirroring the stdout writer in `output.ts`. + */ +function makeFileOutput(mode: OutputMode, sink: FileSink): Output { + return new Output(mode, { + stdout: line => { + sink.stream.write(`${line}\n`); + }, + rawStdout: text => { + if (sink.error) throw sink.error; + if (sink.stream.write(text)) return; + return new Promise(resolve => { + sink.stream.once('drain', () => resolve()); + }); + }, + }); +} + +/** + * Flush + close the file sink. Called on the success path after the + * last write and on the error path inside a `.catch(() => undefined)` + * so the original error isn't masked by a teardown failure. + * + * Re-raises any async stream error captured by the `'error'` listener. + * Without this re-raise, an EACCES on first write would leave a + * zero-byte file behind and exit 0 — a false-success surface that is + * exactly the failure mode `--out` exists to avoid. + */ +function closeOutputFile(sink: FileSink): Promise { + return new Promise((resolve, reject) => { + sink.stream.end(() => { + if (sink.error) { + reject(new TransportError(`Failed to write --out ${sink.path}: ${sink.error.message}`)); + return; + } + resolve(); + }); + }); +} + +/** A presigned `code` body is any `https://` URL — never anything else. */ +export function isPresignedCodeUrl(code: string): boolean { + return code.startsWith('https://'); +} + +/** + * Stream a presigned URL into the Output's chunk writer using the + * deps-provided fetch impl, with no API-key headers — presigned URLs + * carry their own authority. Three failure shapes the caller might + * see, all routed through the typed envelope so `index.ts` produces + * the documented exit code: + * + * - The fetch itself rejects (DNS, TLS reset, offline) → + * `TransportError` (UNAVAILABLE / exit 10) per the CLI error spec + * §7. + * - The fetch resolves with a non-2xx → `UNAVAILABLE` envelope with + * the HTTP status in `details`. Same exit code as transport, since + * a presigned URL that returns 4xx/5xx is functionally indistinct + * from a network failure (the URL is short-lived; the answer is + * "re-run"). + * - The body stream errors mid-read → wrapped as `TransportError` + * so partial output to stdout never silently truncates without a + * non-zero exit. + * + * Streaming via `response.body.getReader()` keeps memory bounded for + * multi-MB code bodies and starts emitting bytes to stdout the moment + * the first chunk arrives — important for `> file.ts` piping where + * the reader may want to start indexing before the download finishes. + */ +async function streamPresignedBody(url: string, out: Output, deps: TestDeps): Promise { + const fetchImpl = deps.fetchImpl ?? globalThis.fetch.bind(globalThis); + let response: Response; + try { + response = await fetchImpl(url); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new TransportError(`Failed to download presigned code body: ${message}`); + } + if (!response.ok) { + throw ApiError.fromEnvelope({ + error: { + code: 'UNAVAILABLE', + message: `Failed to download presigned code body (HTTP ${response.status}).`, + nextAction: + 'Re-run `testsprite test code get`. Presigned URLs expire after a short window.', + requestId: 'local', + details: { status: response.status, url }, + }, + }); + } + if (!response.body) { + // No streamable body (some test runtimes / fetch polyfills). Fall + // back to text() — same correctness, just no streaming benefit. + await out.writeChunk(await response.text()); + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value && value.byteLength > 0) { + // `await` is load-bearing: the default rawStdout writer + // resolves on `'drain'` when stdout's kernel buffer is full, + // which pauses this loop so we don't pull more chunks from + // the network than the consumer can absorb. + await out.writeChunk(decoder.decode(value, { stream: true })); + } + } + // Flush any remaining buffered bytes from a multi-byte UTF-8 codepoint + // straddling a chunk boundary. Without this the last code point of a + // file ending in (say) a Chinese character would be silently dropped. + const tail = decoder.decode(); + if (tail.length > 0) await out.writeChunk(tail); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new TransportError(`Failed mid-download of presigned code body: ${message}`); + } +} + +function renderTestListText(page: Page): string { + if (page.items.length === 0) { + return page.nextToken ? `No tests on this page.\nnextToken: ${page.nextToken}` : 'No tests.'; + } + const idWidth = Math.max(2, ...page.items.map(t => t.id.length)); + const nameWidth = Math.max(4, ...page.items.map(t => t.name.length)); + const typeWidth = 8; + const fromWidth = 6; + const statusWidth = 9; + + const header = + pad('ID', idWidth) + + ' ' + + pad('NAME', nameWidth) + + ' ' + + pad('TYPE', typeWidth) + + ' ' + + pad('FROM', fromWidth) + + ' ' + + pad('STATUS', statusWidth) + + ' ' + + 'UPDATED'; + + const rows = page.items.map( + t => + pad(t.id, idWidth) + + ' ' + + pad(t.name, nameWidth) + + ' ' + + pad(t.type, typeWidth) + + ' ' + + pad(t.createdFrom, fromWidth) + + ' ' + + pad(t.status, statusWidth) + + ' ' + + t.updatedAt, + ); + + const lines = [header, ...rows]; + if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); + return lines.join('\n'); +} + +function renderTestText(t: CliTest): string { + // M2.1 piece 4: when the facade ships `projectName`, lead with a + // human-friendly `project: ()` line so an operator + // skimming `test get` sees the project label without a second + // command. Pre-M2.1 facades that don't emit the field still render + // — we fall back to `projectId:` only. + const projectLine = + t.projectName != null && t.projectName.length > 0 + ? `project: ${t.projectName} (${t.projectId})` + : `projectId: ${t.projectId}`; + const lines = [ + `id: ${t.id}`, + projectLine, + `name: ${t.name}`, + `type: ${t.type}`, + `createdFrom: ${t.createdFrom}`, + `status: ${t.status}`, + ]; + // G1a: surface priority when the backend ships it and it is non-null. + if (t.priority) { + lines.push(`priority: ${t.priority}`); + } + // M3.4: surface plan-step count when the facade ships it (FE tests). + // Lets an operator read the current count — e.g. to recover after a + // `test plan put --expected-step-count` 412 — without a JSON round-trip. + if (typeof t.planStepCount === 'number') { + lines.push(`planSteps: ${t.planStepCount}`); + } + lines.push(`createdAt: ${t.createdAt}`, `updatedAt: ${t.updatedAt}`); + return lines.join('\n'); +} + +function renderStepsText(page: Page): string { + if (page.items.length === 0) { + return page.nextToken ? `No steps on this page.\nnextToken: ${page.nextToken}` : 'No steps.'; + } + + // M2.1 piece 4: prefix every row with a marker column. `*` flags + // steps the facade marked as contributing to the test failure + // (synthetic terminal "assertion" rows always get the marker; + // pre-M2.1 callers see no markers because the field is absent or + // null). Two-character column ("* " or " ") so alignment stays + // stable across pages. + // Collapse newlines / runs of whitespace to single spaces and cap the + // column width. Without this, one long or multi-line description — e.g. + // a synthetic "TEST BLOCKED\n\n…" assertion blob — set the + // column to its full length, padding every short row with hundreds of + // trailing spaces and shoving UPDATED far off-screen; embedded newlines + // broke alignment outright (dogfood 2026-06-04). Full untruncated text + // is still available via `--output json`. + const descOf = (s: CliTestStep): string => { + const isSynthetic = + s.action === 'assertion' && s.htmlSnapshotUrl === null && s.screenshotUrl === null; + const base = s.description.length > 0 ? s.description : '—'; + // Truncate the base FIRST, then append the synthetic hint, so the + // "(synthetic assertion failure)" marker always survives truncation + // (it explains why an `assertion` row exists when the code had none). + const oneLine = base.replace(/\s+/g, ' ').trim(); + const clamped = + oneLine.length > DESC_COL_MAX ? `${oneLine.slice(0, DESC_COL_MAX - 1)}…` : oneLine; + return isSynthetic ? `${clamped} (synthetic assertion failure)` : clamped; + }; + + const indexWidth = Math.max(5, ...page.items.map(s => String(s.stepIndex).length)); + const actionWidth = Math.max(6, ...page.items.map(s => s.action.length)); + const statusWidth = 6; // "passed" / "failed" / "—" + const descWidth = Math.max(11, ...page.items.map(s => descOf(s).length)); + + const header = + pad(' ', 2) + + pad('INDEX', indexWidth) + + ' ' + + pad('ACTION', actionWidth) + + ' ' + + pad('STATUS', statusWidth) + + ' ' + + pad('DESCRIPTION', descWidth) + + ' ' + + 'UPDATED'; + + const rows = page.items.map(s => { + const marker = s.outcomeContributesToFailure === true ? '* ' : ' '; + return [ + marker, + pad(String(s.stepIndex), indexWidth), + pad(s.action, actionWidth), + pad(s.status ?? '—', statusWidth), + pad(descOf(s), descWidth), + s.updatedAt, + ].join(' '); + }); + + const lines: string[] = [header, ...rows, '']; + + // §6.4: all steps in one response share `runIdIfAvailable` and + // `codeVersion` when non-null. Render them once at the bottom — the + // agent gets them per-step in JSON, but humans don't need a column + // repeated 50 times. + const sharedRunId = uniqueNonNull(page.items.map(s => s.runIdIfAvailable)); + const sharedCodeVersion = uniqueNonNull(page.items.map(s => s.codeVersion)); + if (sharedRunId !== undefined) lines.push(`runId: ${sharedRunId}`); + if (sharedCodeVersion !== undefined) lines.push(`codeVersion: ${sharedCodeVersion}`); + if (page.nextToken) lines.push(`nextToken: ${page.nextToken}`); + + return lines.join('\n').replace(/\n+$/, ''); +} + +/** + * Human summary block for `test failure get` (no `--out`). Headlines + * the routing-relevant bits (status / failureKind / failedStepIndex) + * and folds the `failure` sub-block in plain text. Intentionally + * compact — JSON mode is the automation contract; this is for an + * engineer running it interactively. + */ +function renderFailureContextText(ctx: CliFailureContext): string { + const lines: string[] = []; + lines.push(`status: ${ctx.result.status}`); + lines.push(`testId: ${ctx.testId}`); + lines.push(`projectId: ${ctx.projectId}`); + if (ctx.result.failureKind !== null) lines.push(`failureKind: ${ctx.result.failureKind}`); + if (ctx.result.failedStepIndex !== null) + lines.push(`failedStepIndex: ${ctx.result.failedStepIndex}`); + lines.push(`snapshotId: ${ctx.snapshotId}`); + if (ctx.result.runIdIfAvailable !== null) + lines.push(`runId: ${ctx.result.runIdIfAvailable}`); + if (ctx.result.codeVersion !== null) lines.push(`codeVersion: ${ctx.result.codeVersion}`); + if (ctx.result.targetUrl !== null) lines.push(`targetUrl: ${ctx.result.targetUrl}`); + lines.push(''); + if (ctx.failure.rootCauseHypothesis !== null) { + lines.push(`rootCause: ${ctx.failure.rootCauseHypothesis}`); + } else { + lines.push('rootCause: — (analysis pipeline produced none)'); + } + // M2.1 piece 3: `recommendedFixTarget` may be `null` when every + // field is unfilled. Use the shared helper so this surface, the + // /result `--include-analysis` block, and `failure summary` all + // format the field identically. + appendFixTargetLines(lines, ctx.failure.recommendedFixTarget, 'recommendedFix: '); + // Evidence: count + per-kind breakdown. + if (ctx.failure.evidence.length === 0) { + lines.push('evidence: (empty — bundle ships result + code only)'); + } else { + const counts = new Map(); + for (const e of ctx.failure.evidence) counts.set(e.kind, (counts.get(e.kind) ?? 0) + 1); + const breakdown = [...counts.entries()].map(([k, n]) => `${k}×${n}`).join(', '); + lines.push(`evidence: ${ctx.failure.evidence.length} items (${breakdown})`); + } + if (ctx.result.videoUrl !== null) lines.push(`videoUrl: ${ctx.result.videoUrl}`); + return lines.join('\n'); +} + +function renderBundleWrittenText(data: { dir: string; files: number; snapshotId: string }): string { + return `Bundle written to ${data.dir}\n ${data.files} files, snapshotId=${data.snapshotId}`; +} + +function renderBundleDryRunText(data: { dir: string; files: number; snapshotId: string }): string { + return `(dry-run) would write bundle to ${data.dir}\n ${data.files} files, snapshotId=${data.snapshotId}`; +} + +/** + * Approximate the file list `writeBundle` would produce for the given + * context, so dry-run can communicate it to the user without touching + * disk. Mirrors `bundle.ts` write order (result/failure/code/video, + * per-step screenshot+snapshot, meta last) at a coarse level — sidecar + * evidence files (log/network/console) are summarized rather than + * predicted by exact name. Honors `--failed-only` by trimming step rows + * to the failure window the bundle writer would keep. + */ +function plannedBundleFiles(ctx: CliFailureContext, failedOnly: boolean): string[] { + const files: string[] = []; + files.push('result.json'); + files.push('failure.json'); + files.push(`code.${pickCodeExtension(ctx.code.language, ctx.code.framework)}`); + if (ctx.result.videoUrl) files.push('video.mp4'); + + const stepsToInclude = failedOnly + ? ctx.steps.filter(s => { + if (ctx.result.failedStepIndex === null) return false; + const target = ctx.result.failedStepIndex; + return s.stepIndex >= target - 1 && s.stepIndex <= target + 1; + }) + : ctx.steps; + + for (const step of stepsToInclude) { + const prefix = stepFilenamePrefix(step.stepIndex); + if (step.screenshotUrl) files.push(`steps/${prefix}-screenshot.png`); + if (step.htmlSnapshotUrl) files.push(`steps/${prefix}-snapshot.html`); + } + + // Sidecar evidence (log/network/console): the count varies and depends + // on the bundle writer's per-step grouping. Surface a rolled-up count + // rather than predict per-file names — agents reading the dry-run see + // there's "N more sidecar files" without us re-implementing the + // grouping logic. + const sidecar = ctx.failure.evidence.filter( + e => e.kind !== 'screenshot' && e.kind !== 'snapshot', + ); + if (sidecar.length > 0) { + files.push(`steps/-evidence.json (×${sidecar.length} sidecar entries)`); + } + + files.push('meta.json'); + return files; +} + +/** + * L141 — detect server-side truncation: returns `true` when the string + * ends with U+2026 (`…`) AND is long enough that the ellipsis is likely a + * truncation sentinel rather than intentional punctuation in short text. + * + * The backend historically appended `…` when truncating analysis text at ~600 + * chars. The backend is in the process of removing that hard cap, so this + * indicator is transitional/defensive. We apply a length heuristic (≥ 500 + * chars) to avoid flagging short strings that legitimately end in `…` as + * truncated — e.g. "Check the failing element…" (22 chars) is not truncated. + * + * Only used on JSON output to add sibling indicator fields; text-mode + * rendering is unchanged. + * + * Note: the CLI cannot un-truncate data it never received. Full text + * requires backend support. + */ +const TRUNCATION_MIN_LENGTH = 500; + +function isServerTruncated(value: string | null | undefined): boolean { + return typeof value === 'string' && value.length >= TRUNCATION_MIN_LENGTH && value.endsWith('…'); +} + +/** + * L141 — annotate `CliAnalysisBlock` with truncation indicator fields + * for programmatic consumers. Returns a shallow copy (never mutates the + * original). Only called on the JSON output path; text mode is unchanged. + */ +function annotateAnalysisTruncation(block: CliAnalysisBlock): CliAnalysisBlock { + const annotated: CliAnalysisBlock = { ...block }; + if (isServerTruncated(block.rootCauseHypothesis)) { + annotated.rootCauseHypothesisTruncated = true; + } + if ( + block.recommendedFixTarget !== null && + isServerTruncated(block.recommendedFixTarget.rationale) + ) { + annotated.recommendedFixRationaleTruncated = true; + } + return annotated; +} + +function renderResultText(r: CliLatestResult): string { + // §12.7: failureKind + failedStepIndex highlighted "when present". + // Position is the highlight — failure-relevant fields lead the block + // when the run failed; passing/running runs render in chronological + // order so a glance reads like a timeline. + const lines: string[] = []; + lines.push(`status: ${r.status}`); + lines.push(`testId: ${r.testId}`); + if (r.failureKind !== null) lines.push(`failureKind: ${r.failureKind}`); + if (r.failedStepIndex !== null) lines.push(`failedStepIndex: ${r.failedStepIndex}`); + if (r.startedAt !== null) lines.push(`startedAt: ${r.startedAt}`); + if (r.finishedAt !== null) lines.push(`finishedAt: ${r.finishedAt}`); + lines.push(`snapshotId: ${r.snapshotId}`); + if (r.runIdIfAvailable !== null) lines.push(`runId: ${r.runIdIfAvailable}`); + if (r.codeVersion !== null) lines.push(`codeVersion: ${r.codeVersion}`); + if (r.targetUrl !== null) lines.push(`targetUrl: ${r.targetUrl}`); + lines.push( + `summary: passed=${r.summary.passed} failed=${r.summary.failed} skipped=${r.summary.skipped}`, + ); + if (r.videoUrl !== null) lines.push(`videoUrl: ${r.videoUrl}`); + if (r.failureAnalysisUrl !== null) lines.push(`failureAnalysisUrl: ${r.failureAnalysisUrl}`); + if (r.analysis !== undefined) { + // §6.5.1 (M2.1 piece 3) — render the inline analysis block under + // the result summary. Only fires when the caller passed + // `--include-analysis`. JSON mode bypasses this renderer + // (`out.print(result)` ships the wire envelope verbatim). + lines.push(''); + lines.push(`rootCause: ${r.analysis.rootCauseHypothesis ?? '— (none)'}`); + appendFixTargetLines(lines, r.analysis.recommendedFixTarget, 'recommendedFix: '); + } + return lines.join('\n'); +} + +/** + * Render a `recommendedFixTarget` value in text mode. Shared by + * `renderResultText` (under `--include-analysis`), + * `renderFailureContextText`, and `renderFailureSummaryText` so the + * three surfaces format the field identically. + * + * `null` (M2.1 visibility policy) renders as "— (analysis pipeline + * did not propose one)" — the user-facing equivalent of the wire- + * level null. Non-null wrappers render `kind=...` plus an optional + * reference and indented rationale. + */ +function appendFixTargetLines(lines: string[], fix: CliFixTarget | null, label: string): void { + if (fix === null) { + lines.push(`${label}— (analysis pipeline did not propose one)`); + return; + } + const ref = fix.reference ? ` reference=${fix.reference}` : ''; + lines.push(`${label}kind=${fix.kind}${ref}`); + if (fix.rationale !== null) { + // Indent the rationale to the column under the value. The exact + // column doesn't have to match the label width across surfaces; + // a fixed two-space hanging indent keeps the rendering local to + // this helper so callers don't pad themselves. + lines.push(` ${fix.rationale}`); + } +} + +/** + * §5.2 / M2.1 piece 3 — text renderer for `test failure summary`. + * One-screen agent-readable triage card, no bundle. Mirrors + * `renderFailureContextText`'s top section minus the bundle metadata + * (no run id, no codeVersion, no evidence count, no videoUrl) so a + * caller running `failure summary` after `failure get` doesn't see a + * confusing partial bundle. + */ +function renderFailureSummaryText(s: CliFailureSummary): string { + const lines: string[] = []; + lines.push(`testId: ${s.testId}`); + lines.push(`status: ${s.status}`); + if (s.failureKind !== null) lines.push(`failureKind: ${s.failureKind}`); + lines.push(`snapshotId: ${s.snapshotId}`); + lines.push( + `rootCauseHypothesis: ${s.rootCauseHypothesis ?? '— (analysis pipeline produced none)'}`, + ); + appendFixTargetLines(lines, s.recommendedFixTarget, 'recommendedFixTarget: '); + return lines.join('\n'); +} + +/** + * If every non-null entry shares the same value, return it; otherwise + * undefined. Used by step rendering to surface a per-response shared + * `runId` / `codeVersion` once instead of in every row. + */ +function uniqueNonNull(values: Array): string | undefined { + const filtered = values.filter((v): v is string => v !== null); + if (filtered.length === 0) return undefined; + const first = filtered[0]!; + return filtered.every(v => v === first) ? first : undefined; +} + +function pad(s: string, width: number): string { + if (s.length >= width) return s; + return s + ' '.repeat(width - s.length); +} + +function createTestCodeCommand(deps: TestDeps): Command { + const code = new Command('code').description('Inspect and edit generated test code'); + code + .command('get ') + .description('Print the generated test code') + .option( + '--out ', + 'Write the response to this file instead of stdout (text mode: source body; json mode: wire envelope)', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: { out?: string }, command: Command) => { + await runCodeGet({ ...resolveCommonOptions(command), testId, out: cmdOpts.out }, deps); + }); + code + .command('put ') + .description('Replace test code with etag-guarded optimistic concurrency') + .option('--code-file ', 'file containing the new test code (≤ 350 KB)') + .option( + '--expected-version ', + 'expected current codeVersion (e.g. v3); sent as `If-Match`. Mutually exclusive with --force.', + ) + .option( + '--force', + 'send `If-Match: *` to skip the etag check (audit-logged with force: true). Mutually exclusive with --expected-version.', + false, + ) + .option( + '--language ', + 'override the stored language (typescript|javascript|python). Defaults to the existing language.', + ) + .option( + '--idempotency-key ', + 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .option( + '--dry-run-simulate-error ', + 'With --dry-run: synthesise an error envelope to preview the error path. Supported: PRECONDITION_FAILED (412).', + ) + .addHelpText( + 'after', + '\nDry-run note: --dry-run always returns the happy response shape.\n' + + 'To preview the 412 retry-hint path, combine with --dry-run-simulate-error PRECONDITION_FAILED.\n' + + '\n' + + GLOBAL_OPTS_HINT, + ) + .action(async (testId: string, cmdOpts: CodePutFlagOpts, command: Command) => { + const simulateError = cmdOpts.dryRunSimulateError; + if (simulateError !== undefined && simulateError !== 'PRECONDITION_FAILED') { + throw localValidationError( + 'dry-run-simulate-error', + `unsupported value "${simulateError}"; only PRECONDITION_FAILED is supported`, + ['PRECONDITION_FAILED'], + ); + } + await runCodePut( + { + ...resolveCommonOptions(command), + testId, + codeFile: cmdOpts.codeFile, + expectedVersion: cmdOpts.expectedVersion, + force: cmdOpts.force === true, + language: parseEnumFlag(cmdOpts.language, 'language', CODE_PUT_LANGUAGES) as + | CodePutLanguage + | undefined, + idempotencyKey: cmdOpts.idempotencyKey, + dryRunSimulateError: + simulateError === 'PRECONDITION_FAILED' ? 'PRECONDITION_FAILED' : undefined, + }, + deps, + ); + }); + return code; +} + +function createTestPlanCommand(deps: TestDeps): Command { + const plan = new Command('plan').description('Manage FE test plan-steps (FE-only)'); + plan + .command('put ') + .description("Replace an FE test's planSteps[] (BE tests return 400 → use 'test code put')") + .option( + '--steps ', + 'JSON file with { planSteps: [...] } (FE-only, ≤ 200 steps, ≤ 256 KB)', + ) + .option( + '--expected-step-count ', + 'optional defensive concurrency check; server rejects with 412 when the current length differs', + ) + .option( + '--idempotency-key ', + 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', + ) + .option( + '--dry-run-simulate-error ', + 'With --dry-run: synthesise an error envelope to preview the error path. Supported: PRECONDITION_FAILED (412).', + ) + .addHelpText( + 'after', + '\nDry-run note: --dry-run always returns the happy response shape.\n' + + 'To preview the 412 retry-hint path, combine with --dry-run-simulate-error PRECONDITION_FAILED.\n' + + '\n' + + GLOBAL_OPTS_HINT, + ) + .action(async (testId: string, cmdOpts: PlanPutFlagOpts, command: Command) => { + const simulateError = cmdOpts.dryRunSimulateError; + if (simulateError !== undefined && simulateError !== 'PRECONDITION_FAILED') { + throw localValidationError( + 'dry-run-simulate-error', + `unsupported value "${simulateError}"; only PRECONDITION_FAILED is supported`, + ['PRECONDITION_FAILED'], + ); + } + await runPlanPut( + { + ...resolveCommonOptions(command), + testId, + stepsFile: cmdOpts.steps, + expectedStepCount: parseNumericFlag(cmdOpts.expectedStepCount, 'expected-step-count'), + idempotencyKey: cmdOpts.idempotencyKey, + dryRunSimulateError: + simulateError === 'PRECONDITION_FAILED' ? 'PRECONDITION_FAILED' : undefined, + }, + deps, + ); + }); + return plan; +} + +interface PlanPutFlagOpts { + steps: string; + expectedStepCount?: string; + idempotencyKey?: string; + dryRunSimulateError?: string; +} + +interface CodePutFlagOpts { + codeFile: string; + expectedVersion?: string; + force?: boolean; + language?: string; + idempotencyKey?: string; + dryRunSimulateError?: string; +} + +export function createTestArtifactCommand(deps: TestDeps): Command { + const artifact = new Command('artifact').description( + 'Download run-scoped artifact bundles (M3.3 piece-4)', + ); + artifact + .command('get ') + .description( + [ + 'Download the §7 failure-context bundle for a specific run.', + '', + 'Default : ./.testsprite/runs//', + '', + 'Exit codes:', + ' 0 bundle written successfully', + ' 3 authentication error (AUTH_* scope)', + ' 4 run not found / not ready / no failure / cancelled', + ' 5 validation error (bad --out, meta.runId mismatch)', + ' 6 conflict — snapshot in flight (retried once)', + ' 10 transport failure (.partial left on disk)', + ].join('\n'), + ) + .option( + '--out ', + [ + 'Directory to write the §7 disk layout (default: ./.testsprite/runs//).', + 'Parent must exist. The bundle dir itself is created if absent.', + ].join(' '), + ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { + await runArtifactGet( + { + ...resolveCommonOptions(command), + runId, + out: cmdOpts.out, + failedOnly: Boolean(cmdOpts.failedOnly), + }, + deps, + ); + }, + ); + return artifact; +} + +function createTestFailureCommand(deps: TestDeps): Command { + const failure = new Command('failure').description('Export the latest-failure agent bundle'); + failure + .command('get ') + .description("Write a self-contained failure-context bundle for a test's latest failing run") + .option( + '--out ', + 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', + ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { + await runFailureGet( + { + ...resolveCommonOptions(command), + testId, + out: cmdOpts.out, + failedOnly: Boolean(cmdOpts.failedOnly), + }, + deps, + ); + }, + ); + failure + .command('summary ') + .description( + 'Print a one-screen summary of the latest failing run (status, failureKind, hypothesis, fix target — M2.1)', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, _cmdOpts, command: Command) => { + await runFailureSummary({ ...resolveCommonOptions(command), testId }, deps); + }); + return failure; +} diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts new file mode 100644 index 0000000..68a4c99 --- /dev/null +++ b/src/commands/test.wait.spec.ts @@ -0,0 +1,1244 @@ +/** + * Unit tests for `test wait ` — M3.3 piece-3. + * + * Behavior matrix is identical to `--wait` path of `test run` except + * there is no trigger step. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import type { RunResponse } from '../lib/runs.types.js'; +import { runTestWait } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13502', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-m33-wait-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +function makeRun(status: RunResponse['status']): RunResponse { + return { + runId: 'run_abc', + testId: 'test_xyz', + projectId: 'project_1', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: status !== 'queued' ? '2026-05-15T10:00:01.000Z' : null, + finishedAt: status !== 'queued' && status !== 'running' ? '2026-05-15T10:00:30.000Z' : null, + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }; +} + +function errorBody(code: string, details: Record = {}) { + const statusMap: Record = { + AUTH_REQUIRED: 401, + AUTH_FORBIDDEN: 403, + NOT_FOUND: 404, + VALIDATION_ERROR: 400, + CONFLICT: 409, + RATE_LIMITED: 429, + INTERNAL: 500, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +const instantSleep = () => Promise.resolve(); + +// --------------------------------------------------------------------------- +// Happy paths +// --------------------------------------------------------------------------- + +describe('runTestWait — happy paths', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('returns passed RunResponse, exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + const stdout: string[] = []; + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + expect(result.status).toBe('passed'); + expect(result.runId).toBe('run_abc'); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.status).toBe('passed'); + }); + + it('polls multiple times before reaching terminal', async () => { + const { credentialsPath } = makeCreds(); + let getCount = 0; + const fetchImpl = makeFetch(() => { + getCount++; + if (getCount < 3) return { body: makeRun('running') }; + return { body: makeRun('passed') }; + }); + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ); + expect(result.status).toBe('passed'); + expect(getCount).toBe(3); + }); + + it('sends GET to /runs/{runId}', async () => { + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: makeRun('passed') }; + }); + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ); + expect(seenUrls.some(u => u.includes('/runs/run_abc'))).toBe(true); + }); + + it('uses long-poll (waitSeconds param)', async () => { + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: makeRun('passed') }; + }); + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ); + expect(seenUrls.some(u => u.includes('waitSeconds'))).toBe(true); + }); + + it('--output json prints JSON RunResponse on stdout', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const parsed = JSON.parse(stdout.join('')) as Record; + expect(parsed.runId).toBe('run_abc'); + expect(parsed.status).toBe('passed'); + }); +}); + +// --------------------------------------------------------------------------- +// Non-passed terminal states +// --------------------------------------------------------------------------- + +describe('runTestWait — non-passed terminal states', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('failed status → CLIError exit 1', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('failed') })); + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + expect(err.name).toBe('CLIError'); + }); + + it('blocked status → CLIError exit 1', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('blocked') })); + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + }); + + it('cancelled status → CLIError exit 1', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('cancelled') })); + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + }); + + it('non-passed prints artifact hint to stderr', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('failed') })); + const stderrLines: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(() => {}); + expect(stderrLines.some(l => l.includes('artifact'))).toBe(true); + expect(stderrLines.some(l => l.includes('run_abc'))).toBe(true); + }); + + // B3 — cancelled must NOT emit the artifact hint (no artifacts were captured) + it('B3 — cancelled status → CLIError exit 1 with NO artifact hint', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('cancelled') })); + const stderrLines: string[] = []; + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + // Exit 1, but no artifact hint — cancelled runs have no failure bundle + expect(err.exitCode).toBe(1); + expect(stderrLines.some(l => l.includes('artifact'))).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Error scenarios +// --------------------------------------------------------------------------- + +describe('runTestWait — error scenarios', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('404 NOT_FOUND → ApiError exit 4 (cross-tenant runId)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND', { reason: 'not_found' })); + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + expect((err as ApiError).exitCode).toBe(4); + }); + + it('timeout → UNSUPPORTED error exit 7 with nextAction containing run-id', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('running') })); + + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('UNSUPPORTED'); + expect((err as ApiError).exitCode).toBe(7); + expect((err as ApiError).nextAction).toContain('run_abc'); + } finally { + Date.now = realDateNow; + } + }); + + it('403 AUTH_FORBIDDEN → ApiError exit 3', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('AUTH_FORBIDDEN')); + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: () => {}, sleep: instantSleep }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(3); + }); +}); + +// --------------------------------------------------------------------------- +// Dry-run +// --------------------------------------------------------------------------- + +describe('runTestWait — dry-run', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + resetDryRunBannerForTesting(); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('dry-run: no network call; prints envelope with method=GET and run-id in path', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + const envelope = JSON.parse(stdout.join('')) as Record; + expect(envelope.method).toBe('GET'); + expect(envelope.path as string).toContain('run_abc'); + expect(envelope.path as string).toContain('waitSeconds=25'); + }); + + it('dry-run: timeoutSeconds in envelope', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runId: 'run_abc', + timeoutSeconds: 300, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + const envelope = JSON.parse(stdout.join('')) as Record; + expect(envelope.timeoutSeconds).toBe(300); + }); + + it('dry-run: exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: () => {}, + sleep: instantSleep, + }, + ), + ).resolves.toBeDefined(); + }); + + // defect-2 fix: dry-run getRun sample must return status: "passed". + // Prior to fix, a duplicate failed-shape getRun entry appeared before + // the passed-shape entry in samples.ts; findSample's first-match-wins + // always returned status: "failed" for `test wait --dry-run`. + // + // In dry-run mode, runTestWait emits a describe-only envelope + // { method, path, timeoutSeconds } and exits 0. The canned sample is + // not actually fetched — the important invariant is that the command + // completes without throwing (exit 0) even though no real run is + // resolved. The getRun sample correctness is asserted separately in + // samples.test.ts ("GET /runs/{runId} resolves to the passed-shape…"). + it('dry-run: command exits 0 and envelope contains GET path for run-id', async () => { + const { credentialsPath } = makeCreds(); + const stdout: string[] = []; + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runId: 'run_xyz', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdout.push(line), + sleep: instantSleep, + }, + ); + // runTestWait in dry-run mode returns the describe envelope — not a real RunResponse + const envelope = result as unknown as Record; + expect(envelope.method).toBe('GET'); + expect(envelope.path as string).toContain('run_xyz'); + // The command must not throw (exit 0 — verifies defect-2 regression: + // prior to fix, a duplicate failed getRun sample caused `test wait --dry-run` + // to emit the wrong happy-path response) + expect(stdout.length).toBeGreaterThan(0); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.method).toBe('GET'); + }); + + // L1782 banner fix: test wait --dry-run must write the [dry-run] banner + // to stderr (like test run --dry-run and test artifact get --dry-run do), + // while the canned envelope stays on stdout only. + it('dry-run: emits [dry-run] banner to stderr and canned envelope to stdout, exit 0', async () => { + const { credentialsPath } = makeCreds(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runId: 'run_banner', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: {} })), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + // Banner must appear on stderr + expect(stderrLines).toContain(DRY_RUN_BANNER); + // Envelope must appear on stdout and be parseable + const envelope = JSON.parse(stdoutLines.join('')) as Record; + expect(envelope.method).toBe('GET'); + expect(envelope.path as string).toContain('run_banner'); + // Return value must be defined (exit 0 — no throw) + expect(result).toBeDefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Backoff fallback path (VALIDATION_ERROR on waitSeconds) +// --------------------------------------------------------------------------- + +describe('runTestWait — backoff fallback', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('400 VALIDATION_ERROR on waitSeconds → switches to backoff path and eventually succeeds', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl = makeFetch(url => { + callCount++; + // First call with waitSeconds → returns VALIDATION_ERROR + if (callCount === 1 && url.includes('waitSeconds')) { + return errorBody('VALIDATION_ERROR'); + } + return { body: makeRun('passed') }; + }); + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + }); +}); + +// --------------------------------------------------------------------------- +// Backend testId fallback (dogfood L1888) +// +// Backend run-surface rows never finalize server-side, so the run-row poll +// would always hit --timeout (exit 7) even on a passing BE test. `test wait` +// falls back to the testId-scoped verdict (GET /tests/{id}/result) once it is +// terminal for this run. +// --------------------------------------------------------------------------- + +interface BeResult { + testId: string; + status: string; + startedAt: string | null; + finishedAt: string | null; + videoUrl: string | null; + failureAnalysisUrl: string | null; + snapshotId: string; + runIdIfAvailable: string | null; + codeVersion: string | null; + targetUrl: string | null; + failedStepIndex: number | null; + failureKind: string | null; + summary: { passed: number; failed: number; skipped: number }; +} + +function makeBeResult(overrides: Partial = {}): BeResult { + return { + testId: 'test_xyz', + status: 'passed', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + videoUrl: null, + failureAnalysisUrl: null, + snapshotId: 'snap_1', + runIdIfAvailable: 'run_abc', + codeVersion: 'v1', + targetUrl: 'https://example.com', + failedStepIndex: null, + failureKind: null, + summary: { passed: 1, failed: 0, skipped: 0 }, + ...overrides, + }; +} + +function beWaitRouter(args: { + type?: string; + runStatus?: RunResponse['status'] | (() => RunResponse['status']); + result: () => BeResult; +}) { + const counts = { type: 0, result: 0, run: 0 }; + const handler = (url: string) => { + if (url.includes('/tests/test_xyz/result')) { + counts.result += 1; + return { body: args.result() }; + } + if (url.includes('/tests/test_xyz')) { + counts.type += 1; + return { + body: { + id: 'test_xyz', + projectId: 'p1', + name: 'BE test', + type: args.type ?? 'backend', + createdFrom: 'mcp', + status: 'running', + createdAt: '2026-05-15T10:00:00.000Z', + updatedAt: '2026-05-15T10:00:00.000Z', + }, + }; + } + counts.run += 1; + const rs = + typeof args.runStatus === 'function' ? args.runStatus() : (args.runStatus ?? 'running'); + return { body: makeRun(rs) }; + }; + return { handler, counts }; +} + +describe('runTestWait — backend testId fallback (L1888)', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('passing backend test resolves via testId result (exit 0) + emits advisory', async () => { + const { credentialsPath } = makeCreds(); + const router = beWaitRouter({ result: () => makeBeResult({ status: 'passed' }) }); + const stderr: string[] = []; + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: l => stderr.push(l), + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + expect(result.testId).toBe('test_xyz'); + expect(router.counts.type).toBeGreaterThan(0); + expect(router.counts.result).toBeGreaterThan(0); + expect(stderr.join(' ')).toContain('test record'); + }); + + it('failing backend test resolves via fallback (CLIError exit 1) + testId artifact hint', async () => { + const { credentialsPath } = makeCreds(); + const router = beWaitRouter({ + result: () => + makeBeResult({ + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 1, + summary: { passed: 0, failed: 1, skipped: 0 }, + }), + }); + const stderr: string[] = []; + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: l => stderr.push(l), + sleep: instantSleep, + }, + ).catch(e => e); + expect(err.exitCode).toBe(1); + expect(err.name).toBe('CLIError'); + expect(stderr.join(' ')).toContain('test failure get test_xyz'); + }); + + it('frontend test is untouched — terminal run row resolves with zero testId lookups', async () => { + const { credentialsPath } = makeCreds(); + const router = beWaitRouter({ + type: 'frontend', + runStatus: 'passed', + result: () => makeBeResult(), + }); + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + expect(router.counts.type).toBe(0); + expect(router.counts.result).toBe(0); + }); + + it('frontend test with a non-terminal tick resolves via the run row, never the result endpoint', async () => { + const { credentialsPath } = makeCreds(); + let runCall = 0; + const router = beWaitRouter({ + type: 'frontend', + runStatus: () => (++runCall >= 2 ? 'passed' : 'running'), + result: () => makeBeResult(), + }); + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + // The fallback probed the type once (FE) but never read the result endpoint. + expect(router.counts.type).toBe(1); + expect(router.counts.result).toBe(0); + }); + + it('a transient type-probe error does not permanently disable the fallback (retries next tick)', async () => { + const { credentialsPath } = makeCreds(); + let typeCalls = 0; + const handler = (url: string) => { + if (url.includes('/tests/test_xyz/result')) { + return { body: makeBeResult({ status: 'passed' }) }; + } + if (url.includes('/tests/test_xyz')) { + typeCalls += 1; + // First probe 500s (transient); second succeeds as backend. + if (typeCalls === 1) return { status: 500, body: { error: { code: 'INTERNAL' } } }; + return { + body: { + id: 'test_xyz', + projectId: 'p1', + name: 'BE', + type: 'backend', + createdFrom: 'mcp', + status: 'running', + createdAt: '2026-05-15T10:00:00.000Z', + updatedAt: '2026-05-15T10:00:00.000Z', + }, + }; + } + return { body: makeRun('running') }; + }; + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(handler), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + // Despite the first probe failing, a later tick re-probed → backend → resolved. + expect(result.status).toBe('passed'); + expect(typeCalls).toBeGreaterThanOrEqual(2); + }); + + it('rejects a stale prior-run verdict, then accepts the matching one', async () => { + const { credentialsPath } = makeCreds(); + let call = 0; + const router = beWaitRouter({ + result: () => { + call += 1; + // First result names a DIFFERENT run (stale) → must be rejected; the + // next names this run → accepted. + return call === 1 + ? makeBeResult({ status: 'passed', runIdIfAvailable: 'run_OLD' }) + : makeBeResult({ status: 'passed', runIdIfAvailable: 'run_abc' }); + }, + }); + const result = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(router.handler), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + expect(router.counts.result).toBeGreaterThanOrEqual(2); + }); +}); + +// --------------------------------------------------------------------------- +// Fix 3 — D4: RequestTimeoutError under test wait emits partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when poll throws RequestTimeoutError', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl: typeof globalThis.fetch = async () => { + throw new RequestTimeoutError(120000, 'req_timeout_wait_test'); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + // Stdout must contain the partial object with runId + status:"running" + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + // Stderr should mention the runId and suggest test wait + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('run_abc'); + expect(stderrBlock).toContain('test wait'); + }); +}); + +// --------------------------------------------------------------------------- +// FIX 4 — D5-UX: text mode shows the `error` string for failed/blocked runs +// --------------------------------------------------------------------------- +describe('[fix-4-ux] runTestWait — text mode shows error string for failed/blocked runs', () => { + it('failed run with error string renders "error " line in text mode', async () => { + const { credentialsPath } = makeCreds(); + const run: RunResponse = { + ...makeRun('failed'), + failureKind: 'assertion', + error: 'Element not found: .checkout-button', + }; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(() => { + // exit 1 (failed run) — we only care about stdout + }); + + const stdoutBlock = stdoutLines.join('\n'); + // Must include the error line + expect(stdoutBlock).toContain('error'); + expect(stdoutBlock).toContain('Element not found: .checkout-button'); + }); + + it('failed run with multi-line error shows first line truncated to 200 chars', async () => { + const { credentialsPath } = makeCreds(); + const longFirstLine = 'A'.repeat(250); + const run: RunResponse = { + ...makeRun('failed'), + failureKind: 'assertion', + error: `${longFirstLine}\nsecond line`, + }; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(() => { + // exit 1 + }); + + const stdoutBlock = stdoutLines.join('\n'); + // First line truncated to 200 chars + ellipsis + expect(stdoutBlock).toContain('error'); + expect(stdoutBlock).toContain('A'.repeat(200)); + expect(stdoutBlock).toContain('…'); + // Second line should NOT appear + expect(stdoutBlock).not.toContain('second line'); + }); + + it('passed run with null error does NOT render error line', async () => { + const { credentialsPath } = makeCreds(); + const run: RunResponse = { ...makeRun('passed'), error: null }; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + + const stdoutBlock = stdoutLines.join('\n'); + // Must NOT include an "error" line for a passing run + expect(stdoutBlock).not.toMatch(/^error\s+/m); + }); + + it('JSON mode does NOT change: error field passes through wire envelope unchanged', async () => { + const { credentialsPath } = makeCreds(); + const run: RunResponse = { + ...makeRun('failed'), + failureKind: 'assertion', + error: 'Something went wrong', + }; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(() => { + // exit 1 + }); + + // JSON mode: parse stdout and verify error field is present + const parsed = JSON.parse(stdoutLines.join('')) as RunResponse; + expect(parsed.error).toBe('Something went wrong'); + }); +}); + +// --------------------------------------------------------------------------- +// dashboardUrl on terminal output (colleague feedback 2026-06-10) — the +// GET /runs/{runId} wire row carries projectId+testId, so `test wait` +// closes with a Portal deep link on known API hosts. +// --------------------------------------------------------------------------- + +describe('runTestWait — dashboardUrl on terminal output', () => { + it('JSON mode (prod endpoint): terminal envelope includes dashboardUrl', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/project_1/test/test_xyz', + ); + }); + + it('text mode (TESTSPRITE_PORTAL_URL override): card ends with the overridden dashboard line', async () => { + vi.stubEnv('TESTSPRITE_PORTAL_URL', 'https://portal.internal.example.com'); + try { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.example.com:8443'); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + expect(stdout.join('\n')).toContain( + 'dashboard https://portal.internal.example.com/dashboard/tests/project_1/test/test_xyz', + ); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('unknown API host (localhost): no dashboardUrl field', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBeUndefined(); + }); +}); diff --git a/src/commands/usage.test.ts b/src/commands/usage.test.ts new file mode 100644 index 0000000..28bdef3 --- /dev/null +++ b/src/commands/usage.test.ts @@ -0,0 +1,313 @@ +/** + * Unit tests for `testsprite usage` / `testsprite credits`. + * + * Backend follow-up: the `/me` endpoint must add `credits` + `subPlan` projection. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { writeProfile } from '../lib/credentials.js'; +import type { UsageDeps, UsageResponse } from './usage.js'; +import { DRY_RUN_USAGE_SAMPLE, createUsageCommand, runUsage } from './usage.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { + capture: CapturedOutput; + deps: Pick; +} { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +/** Minimal MeResponse without credits/subPlan (backend current state) */ +const meWithoutCredits = { + userId: 'u-abc', + keyId: 'k-abc', + scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], + env: 'development' as const, +}; + +/** Extended MeResponse WITH credits + subPlan (backend future state) */ +const meWithCredits: UsageResponse = { + ...meWithoutCredits, + credits: 100, + subPlan: 'Standard', + creditsPerRun: 2, +}; + +function makeFetch(body: unknown, status = 200): UsageDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as UsageDeps['fetchImpl']; +} + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-usage-')), 'credentials'); +}); + +describe('runUsage — dry-run', () => { + it('emits the dry-run banner + note about missing backend data', async () => { + const { capture, deps } = makeCapture(); + const result = await runUsage( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + deps, + ); + const stderr = capture.stderr.join('\n'); + // Banner must be present. + expect(stderr).toContain('dry-run'); + // Must note that credits require a backend update. + expect(stderr).toContain('backend'); + // Must return the canned sample. + expect(result).toEqual(DRY_RUN_USAGE_SAMPLE); + }); + + it('dry-run sample contains credits, subPlan, and creditsPerRun', () => { + expect(DRY_RUN_USAGE_SAMPLE.credits).toBeGreaterThan(0); + expect(DRY_RUN_USAGE_SAMPLE.subPlan).toBeTruthy(); + expect(DRY_RUN_USAGE_SAMPLE.creditsPerRun).toBeGreaterThan(0); + }); + + it('dry-run JSON output contains the sample fields', async () => { + const { capture, deps } = makeCapture(); + await runUsage({ profile: 'default', output: 'json', debug: false, dryRun: true }, deps); + const parsed = JSON.parse(capture.stdout.join('')) as UsageResponse; + expect(parsed.credits).toBe(DRY_RUN_USAGE_SAMPLE.credits); + expect(parsed.subPlan).toBe(DRY_RUN_USAGE_SAMPLE.subPlan); + expect(parsed.creditsPerRun).toBe(DRY_RUN_USAGE_SAMPLE.creditsPerRun); + }); +}); + +describe('runUsage — real path without credits (current backend)', () => { + it('returns the /me response and emits a note about missing balance', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const result = await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithoutCredits), + }, + ); + expect(result.userId).toBe('u-abc'); + expect(result.credits).toBeUndefined(); + // Must emit the note pointing at the billing URL. + const stderr = capture.stderr.join('\n'); + expect(stderr).toContain('billing'); + expect(stderr).toContain('testsprite.com'); + }); + + it('text output includes identity fields even without credits', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithoutCredits), + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('userId:'); + expect(out).toContain('u-abc'); + }); + + it('JSON output passes the raw /me response through (no credits key present)', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'json', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithoutCredits), + }, + ); + const parsed = JSON.parse(capture.stdout.join('')) as UsageResponse; + expect(parsed.userId).toBe('u-abc'); + expect(parsed.credits).toBeUndefined(); + }); +}); + +describe('runUsage — real path with credits (future backend)', () => { + it('renders balance block when credits + subPlan are present', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithCredits), + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('credits:'); + expect(out).toContain('100'); + expect(out).toContain('plan:'); + expect(out).toContain('Standard'); + expect(out).toContain('cost per frontend run:'); + // Should show max runs estimate. + expect(out).toContain('can trigger:'); + // 100 / 2 = 50 runs + expect(out).toContain('50'); + }); + + it('does NOT emit the missing-balance note when credits are present', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithCredits), + }, + ); + const stderr = capture.stderr.join('\n'); + // No missing-balance note when data is present. + expect(stderr).not.toContain('not available'); + }); + + it('emits low-balance warning when credits < creditsPerRun', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const lowBalance: UsageResponse = { ...meWithCredits, credits: 1, creditsPerRun: 2 }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(lowBalance), + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('warning'); + expect(out).toContain('billing'); + }); + + it('emits free-plan upgrade hint when subPlan is Free', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const freePlan: UsageResponse = { ...meWithCredits, subPlan: 'Free', credits: 10 }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(freePlan), + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('Free plan'); + expect(out).toContain('pricing'); + }); + + it('JSON output passes credits and subPlan through verbatim', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'json', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(meWithCredits), + }, + ); + const parsed = JSON.parse(capture.stdout.join('')) as UsageResponse; + expect(parsed.credits).toBe(100); + expect(parsed.subPlan).toBe('Standard'); + expect(parsed.creditsPerRun).toBe(2); + }); +}); + +describe('runUsage — error handling', () => { + it('throws AUTH_REQUIRED when no profile is configured', async () => { + const { deps } = makeCapture(); + await expect( + runUsage({ profile: 'default', output: 'text', debug: false }, { ...deps, credentialsPath }), + ).rejects.toMatchObject({ code: 'AUTH_REQUIRED' }); + }); + + it('forwards server AUTH_INVALID with exit code 3', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const errorBody = { + error: { + code: 'AUTH_INVALID', + message: 'Bad key.', + nextAction: 'rotate it', + requestId: 'req_x', + details: {}, + }, + }; + await expect( + runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(errorBody, 401), + }, + ), + ).rejects.toMatchObject({ code: 'AUTH_INVALID', exitCode: 3 }); + }); + + it('re-maps INSUFFICIENT_CREDITS (rate_limited with credits sub-case) to exit 12', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const creditError = { + error: { + code: 'RATE_LIMITED', + message: 'Insufficient credits: 2 credit(s) required.', + nextAction: 'Top up at billing.', + requestId: 'req_y', + details: { required: 2, userId: 'u-abc' }, + }, + }; + await expect( + runUsage( + { profile: 'default', output: 'text', debug: false }, + { + ...deps, + credentialsPath, + fetchImpl: makeFetch(creditError, 429), + }, + ), + ).rejects.toMatchObject({ code: 'INSUFFICIENT_CREDITS', exitCode: 12 }); + }); +}); + +describe('createUsageCommand wiring', () => { + it('exposes the expected command name and credits alias', () => { + const cmd = createUsageCommand(); + expect(cmd.name()).toBe('usage'); + expect(cmd.alias()).toBe('credits'); + }); + + it('--help includes the expected command description', () => { + const cmd = createUsageCommand(); + const helpText = cmd.helpInformation(); + // Commander's helpInformation() includes the command description. + expect(helpText).toContain('credit balance'); + }); +}); diff --git a/src/commands/usage.ts b/src/commands/usage.ts new file mode 100644 index 0000000..d8a0f21 --- /dev/null +++ b/src/commands/usage.ts @@ -0,0 +1,237 @@ +/** + * `testsprite usage` — show the account's credit balance and plan/entitlement + * info as a proactive pre-flight before a large `test run` fan-out. + * + * Backend note: the `GET /me` endpoint does NOT currently return credit balance + * or plan info. This command calls `/me` for auth-identity fields and surfaces + * the `credits` / `subPlan` fields when and only when the backend supplies them + * (forward-compat / absent-safe). A dedicated backend endpoint is a required + * follow-up. + * + * BACKEND FOLLOW-UP REQUIRED: + * Add `credits`, `subPlan` to the `/me` response, or add a dedicated + * `GET /api/cli/v1/usage` endpoint returning `{ credits, subPlan, creditsPerRun }`. + */ + +import { Command } from 'commander'; +import { + emitDryRunBanner, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { loadConfig } from '../lib/config.js'; +import { resolvePortalBase } from '../lib/facade.js'; +import type { FetchImpl } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; + +/** + * Usage/balance response from `/me` (when the backend supplies it) or a future + * `/usage` endpoint. + * + * All fields except `userId`/`keyId`/`env` are forward-compat: the backend + * does not return them today. They are rendered only when present. + */ +export interface UsageResponse { + userId: string; + keyId: string; + env: 'development' | 'staging' | 'production'; + /** + * Remaining credit balance. Present only when the backend /me (or /usage) + * includes the User.credits projection. BACKEND FOLLOW-UP: me.controller.ts. + */ + credits?: number; + /** + * Subscription plan name (e.g. "Free", "Standard", "Pro"). Present only when + * the backend /me (or /usage) includes the User.subPlan projection. + * BACKEND FOLLOW-UP: me.controller.ts. + */ + subPlan?: string; + /** + * Credit cost per test run trigger (informational). Present only when the + * backend supplies it. + */ + creditsPerRun?: number; +} + +export interface UsageDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; +} + +type CommonOptions = FactoryCommonOptions; + +/** Dry-run canned response — matches what the real /me + User lookup would return. */ +export const DRY_RUN_USAGE_SAMPLE: UsageResponse = { + userId: '11111111-1111-4111-8111-111111111111', + keyId: 'key_dryrun_2026', + env: 'development', + credits: 42, + subPlan: 'Standard', + creditsPerRun: 2, +}; + +/** + * Run the `usage` command. Calls `GET /me`, surfaces identity + any + * credits/plan fields the backend supplies. Absent fields are silently + * omitted (forward-compat until the backend adds the projection). + */ +export async function runUsage(opts: CommonOptions, deps: UsageDeps = {}): Promise { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (opts.dryRun) { + emitDryRunBanner(stderr); + stderr('[note] credit balance requires a backend update — showing dry-run sample values'); + out.print(DRY_RUN_USAGE_SAMPLE, data => renderUsage(data as UsageResponse)); + return DRY_RUN_USAGE_SAMPLE; + } + + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + + // Environment-correct portal origin for billing/upgrade links (dev and prod + // portals live on different domains — never hardcode). Resolved from the + // same credentials the client was just built from; undefined for unknown + // hosts (links then render as routes only). + const portalBase = resolvePortalBase( + loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env: deps.env, + credentialsPath: deps.credentialsPath, + }).apiUrl, + ); + const billingUrl = + portalBase !== undefined + ? `${portalBase}/dashboard/settings/billing` + : 'the portal Billing page (/dashboard/settings/billing)'; + + // /me is the only available source of credits/plan today. + // When the backend adds credits/subPlan to MeResponse (or adds /usage), + // this single get call is sufficient — no code change needed in the CLI. + const me = await client.get('/me'); + + out.print(me, data => renderUsage(data as UsageResponse, portalBase)); + + // In text mode, emit a backend-gap note when credits are missing so the + // user knows why the balance isn't shown (instead of assuming zero or error). + if (opts.output === 'text' && me.credits === undefined) { + stderr( + '[note] credit balance not available — backend does not yet expose credits on /me.' + + ` Check ${billingUrl} for your current balance.`, + ); + } + + return me; +} + +function renderUsage(u: UsageResponse, portalBase?: string): string { + const lines: string[] = []; + + // Identity block (always present) + lines.push(`userId: ${u.userId}`); + lines.push(`keyId: ${u.keyId}`); + lines.push(`env: ${u.env}`); + + // Balance block — shown only when the backend supplies it + const hasBalanceData = u.credits !== undefined || u.subPlan !== undefined; + if (hasBalanceData) { + lines.push(''); + lines.push('--- credits & plan ---'); + if (u.subPlan !== undefined) { + lines.push(`plan: ${u.subPlan}`); + } + if (u.credits !== undefined) { + lines.push(`credits: ${u.credits}`); + } + if (u.creditsPerRun !== undefined) { + lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); + lines.push( + `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, + ); + } + + // Pre-flight hint: how many runs the current balance can fund + if (u.credits !== undefined && u.creditsPerRun !== undefined && u.creditsPerRun > 0) { + const maxRuns = Math.floor(u.credits / u.creditsPerRun); + lines.push(`can trigger: ~${maxRuns} run(s) at current balance`); + } + } + + // Actionable upgrade line for Free or low-balance keys + const isLowBalance = + u.credits !== undefined && u.creditsPerRun !== undefined && u.credits < u.creditsPerRun; + const isFree = u.subPlan?.toLowerCase() === 'free'; + + if (isLowBalance) { + lines.push(''); + lines.push( + 'warning: credit balance is below the per-run cost. Top up at:' + + ` ${portalBase !== undefined ? `${portalBase}/dashboard/settings/billing` : 'the portal Billing page (/dashboard/settings/billing)'}`, + ); + } else if (isFree) { + lines.push(''); + lines.push( + 'note: on Free plan — upgrade for more credits and higher run limits:' + + ` ${portalBase !== undefined ? `${portalBase}/pricing` : 'the portal Pricing page (/pricing)'}`, + ); + } + + return lines.join('\n'); +} + +export function createUsageCommand(deps: UsageDeps = {}): Command { + const cmd = new Command('usage') + .alias('credits') + .description( + 'Show credit balance and plan/entitlement info (proactive pre-flight before a large test run)', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite usage # show balance + plan\n' + + ' testsprite usage --output json # machine-readable balance\n' + + ' testsprite credits # alias for usage\n' + + '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + + " check your portal's Billing page (/dashboard/settings/billing) for your balance.", + ) + .action(async (_cmdOpts, command: Command) => { + await runUsage(resolveCommonOptions(command), deps); + }); + + return cmd; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return undefined; + return Math.round(n * 1000); +} + +function makeOutput(mode: OutputMode, deps: UsageDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..38b669e --- /dev/null +++ b/src/index.ts @@ -0,0 +1,160 @@ +#!/usr/bin/env node +import { Command, CommanderError } from 'commander'; +import { createAgentCommand } from './commands/agent.js'; +import { createAuthCommand } from './commands/auth.js'; +import { createInitCommand } from './commands/init.js'; +import { createProjectCommand } from './commands/project.js'; +import { createTestCommand } from './commands/test.js'; +import { createUsageCommand } from './commands/usage.js'; +import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { Output, isOutputMode } from './lib/output.js'; +import { rephraseUnknownOption } from './lib/render-error.js'; +import { VERSION } from './version.js'; + +const program = new Command(); + +// exitOverride() causes Commander to throw CommanderError instead of calling +// process.exit() directly, giving our catch block a chance to remap error +// exit codes (e.g. missing-argument → exit 5 per taxonomy). +program.exitOverride(); + +program + .name('testsprite') + .description('Official TestSprite command-line interface') + .version(VERSION) + .option('--output ', 'Output format (json|text)', 'text') + .option('--profile ', 'Configuration profile to use') + .option('--endpoint-url ', 'Override the API endpoint host') + .option( + '--verbose', + 'Emit human-readable HTTP retry / backoff / polling-mode transitions to stderr. Less noisy than --debug; useful for diagnosing hangs without the full trace.', + ) + .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') + .option( + '--dry-run', + 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', + ) + .option( + '--request-timeout ', + 'Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. ' + + 'Override via TESTSPRITE_REQUEST_TIMEOUT_MS env var (milliseconds). ' + + 'Range: 1–600. Does not affect the --timeout polling ceiling for `test run/wait`.', + ); + +program.addCommand(createAgentCommand({})); +program.addCommand(createAuthCommand()); +program.addCommand(createInitCommand({})); +program.addCommand(createProjectCommand({})); +program.addCommand(createTestCommand()); +program.addCommand(createUsageCommand()); + +// Propagate exitOverride to every subcommand in the tree. +// Commander's addCommand() does NOT inherit exitOverride from the parent, +// so commands built externally (createTestCommand, createProjectCommand, …) +// and attached via addCommand() still have _exitCallback = null and call +// process.exit directly. Recursively set exitOverride so CommanderError +// bubbles up to our catch block for every leaf subcommand. +function applyExitOverrideDeep(cmd: Command): void { + cmd.exitOverride(); + for (const child of cmd.commands) { + applyExitOverrideDeep(child); + } +} +applyExitOverrideDeep(program); + +program.configureOutput({ + outputError(str, write) { + const rephrased = rephraseUnknownOption(str); + write(rephrased !== null ? `${rephrased}\n` : str); + }, +}); + +try { + await program.parseAsync(process.argv); +} catch (err) { + const rawMode = program.opts<{ output?: string }>().output; + const mode = isOutputMode(rawMode) ? rawMode : 'text'; + if (err instanceof ApiError) { + if (mode === 'json') { + const envelope = { + error: { + code: err.code, + message: err.message, + nextAction: err.nextAction, + requestId: err.requestId, + details: err.details, + }, + }; + process.stderr.write(`${JSON.stringify(envelope, null, 2)}\n`); + } else { + process.stderr.write(`Error: ${err.message}\n`); + if (err.nextAction) process.stderr.write(`${err.nextAction}\n`); + if (err.requestId && err.requestId !== 'local') + process.stderr.write(`requestId: ${err.requestId}\n`); + // C1: surface requiredScopes / grantedScopes on AUTH_FORBIDDEN + if (err.code === 'AUTH_FORBIDDEN') { + const required = err.getDetail('requiredScopes'); + const granted = err.getDetail('grantedScopes'); + if (Array.isArray(required) && required.length > 0) { + process.stderr.write(` required: ${(required as string[]).join(', ')}\n`); + } + if (Array.isArray(granted)) { + process.stderr.write(` granted: ${(granted as string[]).join(', ')}\n`); + } + } + } + process.exit(err.exitCode); + } + const output = new Output(mode); + if (err instanceof RequestTimeoutError) { + // Structured rendering for per-request timeouts: JSON mode emits a + // machine-readable envelope; text mode emits the message with a hint. + if (mode === 'json') { + const envelope = { + error: { + code: 'REQUEST_TIMEOUT', + message: err.message, + nextAction: + 'Increase --request-timeout or set TESTSPRITE_REQUEST_TIMEOUT_MS. ' + + 'Check that the backend is reachable and not overloaded.', + requestId: err.requestId, + details: { timeoutMs: err.timeoutMs }, + }, + }; + process.stderr.write(`${JSON.stringify(envelope, null, 2)}\n`); + } else { + process.stderr.write(`Error: ${err.message}\n`); + } + process.exit(err.exitCode); + } + if (err instanceof CommanderError) { + // Commander already wrote the error message (via configureOutput) or the + // help/version text to stdout. Map exit codes per the CLI taxonomy: + // help / version → 0 (user asked for it — no error) + // parse errors → 5 (VALIDATION_ERROR family: missing arg, invalid + // option, unknown command, etc.) + // + // Two distinct help codes exist in Commander 12: + // 'commander.helpDisplayed' — thrown by `-h/--help` flag handler + // 'commander.help' — thrown by the built-in `help [command]` + // subcommand (used by `test help`, `project + // help`, `help test`, etc.) + // Both are user-initiated "show me help" requests and must exit 0 per the + // AWS-CLI convention. Failing to map 'commander.help' caused these paths + // to fall through to the generic `process.exit(5)` branch (dogfood P1-4). + if ( + err.code === 'commander.helpDisplayed' || + err.code === 'commander.help' || + err.code === 'commander.version' + ) { + process.exit(0); + } + process.exit(5); + } + if (err instanceof CLIError) { + output.error(err.message); + process.exit(err.exitCode); + } + output.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +} diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts new file mode 100644 index 0000000..2a2262c --- /dev/null +++ b/src/lib/agent-targets.test.ts @@ -0,0 +1,382 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { + MANAGED_SECTION_BEGIN, + MANAGED_SECTION_END, + SKILL_DESCRIPTION, + SKILL_NAME, + TARGETS, + loadCodexSkillBody, + loadSkillBody, + renderForTarget, +} from './agent-targets.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Parse the `description:` value out of a YAML frontmatter block. + * The frontmatter is delimited by `---` lines at the top of the file. + * The description value is a single line (no folded/literal block scalars). + */ +function parseFrontmatterDescription(content: string): string | undefined { + const lines = content.split('\n'); + let inFrontmatter = false; + for (const line of lines) { + if (line.trim() === '---') { + if (!inFrontmatter) { + inFrontmatter = true; + continue; + } else { + // End of frontmatter + break; + } + } + if (inFrontmatter && line.startsWith('description: ')) { + return line.slice('description: '.length); + } + } + return undefined; +} + +// Load skill-template.md from repo root (vitest cwd = repo root). +const templateRaw = readFileSync('docs/cli-v1-agent-install/skill-template.md', 'utf8'); +const templateDescription = parseFrontmatterDescription(templateRaw); + +// Stub body for unit tests that don't need the real file, so tests are fast +// and deterministic regardless of asset path resolution. +const STUB_BODY = `# TestSprite Verification Loop + +The verification-loop autopilot + +testsprite test run --wait --target-url --timeout 600 + +testsprite test artifact get --out ./out/ +`; + +// --------------------------------------------------------------------------- +// TARGETS shape +// --------------------------------------------------------------------------- + +describe('TARGETS', () => { + it('has all five required keys', () => { + const keys = Object.keys(TARGETS).sort(); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor']); + }); + + it('claude is GA', () => { + expect(TARGETS.claude.status).toBe('ga'); + }); + + it('cursor, cline, antigravity, and codex are experimental', () => { + expect(TARGETS.cursor.status).toBe('experimental'); + expect(TARGETS.cline.status).toBe('experimental'); + expect(TARGETS.antigravity.status).toBe('experimental'); + expect(TARGETS.codex.status).toBe('experimental'); + }); + + it('each target has a non-empty POSIX path', () => { + for (const [, spec] of Object.entries(TARGETS)) { + expect(spec.path.length).toBeGreaterThan(0); + expect(spec.path).not.toContain('\\'); + } + }); + + it('own-file targets have mode own-file', () => { + expect(TARGETS.claude.mode).toBe('own-file'); + expect(TARGETS.antigravity.mode).toBe('own-file'); + expect(TARGETS.cursor.mode).toBe('own-file'); + expect(TARGETS.cline.mode).toBe('own-file'); + }); + + it('codex target has mode managed-section', () => { + expect(TARGETS.codex.mode).toBe('managed-section'); + }); + + it('codex target path is AGENTS.md', () => { + expect(TARGETS.codex.path).toBe('AGENTS.md'); + }); +}); + +// --------------------------------------------------------------------------- +// SKILL_DESCRIPTION +// --------------------------------------------------------------------------- + +describe('SKILL_DESCRIPTION', () => { + it('is ≤ 1536 characters (claude description cap)', () => { + expect(SKILL_DESCRIPTION.length).toBeLessThanOrEqual(1536); + }); + + it('is byte-identical to skill-template.md frontmatter description', () => { + expect(templateDescription).toBeDefined(); + expect(SKILL_DESCRIPTION).toBe(templateDescription); + }); + + it('begins with TestSprite', () => { + expect(SKILL_DESCRIPTION.startsWith('TestSprite')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// SKILL_NAME +// --------------------------------------------------------------------------- + +describe('SKILL_NAME', () => { + it('is "testsprite-verify"', () => { + expect(SKILL_NAME).toBe('testsprite-verify'); + }); +}); + +// --------------------------------------------------------------------------- +// loadSkillBody +// --------------------------------------------------------------------------- + +describe('loadSkillBody', () => { + it('returns a non-empty string when using the injectable read stub', () => { + const body = loadSkillBody(() => STUB_BODY); + expect(body).toBe(STUB_BODY); + }); + + it('real loadSkillBody() reads the actual asset and starts with the TestSprite Verification Loop H1', () => { + // This exercises the real URL resolution path — proves the asset is reachable. + const body = loadSkillBody(); + expect(body.length).toBeGreaterThan(0); + expect(body.trimStart().startsWith('# TestSprite Verification Loop')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// renderForTarget — frontmatter shape per target +// --------------------------------------------------------------------------- + +describe('renderForTarget("claude")', () => { + const result = renderForTarget('claude', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.claude/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('content ends with a trailing newline', () => { + expect(result.content.endsWith('\n')).toBe(true); + }); +}); + +describe('renderForTarget("antigravity")', () => { + const result = renderForTarget('antigravity', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.agents/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); +}); + +describe('renderForTarget("claude") vs renderForTarget("antigravity")', () => { + it('produce the same frontmatter lines (name + description)', () => { + const claude = renderForTarget('claude', STUB_BODY); + const antigravity = renderForTarget('antigravity', STUB_BODY); + + // Extract the frontmatter block from each + const extractFrontmatter = (content: string): string => { + const match = /^---\n([\s\S]*?)\n---/.exec(content); + return match?.[1] ?? ''; + }; + + expect(extractFrontmatter(claude.content)).toBe(extractFrontmatter(antigravity.content)); + }); + + it('differ only in their landing path', () => { + const claude = renderForTarget('claude', STUB_BODY); + const antigravity = renderForTarget('antigravity', STUB_BODY); + + expect(claude.path).not.toBe(antigravity.path); + // Body content should be identical + expect(claude.content).toBe(antigravity.content); + }); +}); + +describe('renderForTarget("cursor")', () => { + const result = renderForTarget('cursor', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.cursor/rules/testsprite-verify.mdc'); + }); + + it('frontmatter has alwaysApply: false', () => { + expect(result.content).toContain('alwaysApply: false'); + }); + + it('frontmatter has description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('frontmatter does NOT have a globs: line', () => { + // Extract frontmatter block + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('globs:'); + }); + + it('frontmatter does NOT have a name: line', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); + }); +}); + +describe('renderForTarget("cline")', () => { + const result = renderForTarget('cline', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.clinerules/testsprite-verify.md'); + }); + + it('has no --- frontmatter fence', () => { + expect(result.content).not.toContain('---'); + }); + + it('starts with the TestSprite Verification Loop H1 (the body H1)', () => { + expect(result.content.trimStart().startsWith('# TestSprite Verification Loop')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Content integrity — load-bearing command strings must survive any body trim +// --------------------------------------------------------------------------- + +describe('content integrity — own-file targets', () => { + const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity'> = [ + 'claude', + 'cursor', + 'cline', + 'antigravity', + ]; + + // Use the real body for these checks, since we're guarding against trimming. + for (const target of ownFileTargets) { + describe(`target: ${target}`, () => { + const result = renderForTarget(target); + + it('contains the TestSprite Verification Loop H1', () => { + // The skill body opens with the renamed H1. + expect(result.content).toContain('# TestSprite Verification Loop'); + expect(result.content).toContain('The verification loop that flies'); + }); + + it('contains testsprite test run', () => { + expect(result.content).toContain('testsprite test run'); + }); + + it('contains --wait flag', () => { + expect(result.content).toContain('--wait'); + }); + + it('contains test artifact get', () => { + expect(result.content).toContain('test artifact get'); + }); + }); + } +}); + +// --------------------------------------------------------------------------- +// loadCodexSkillBody +// --------------------------------------------------------------------------- + +describe('loadCodexSkillBody', () => { + it('returns a non-empty string when using the injectable read stub', () => { + const body = loadCodexSkillBody(() => '# stub codex body'); + expect(body).toBe('# stub codex body'); + }); + + it('real loadCodexSkillBody() reads the actual asset and starts with the TestSprite Verification Loop H1', () => { + const body = loadCodexSkillBody(); + expect(body.length).toBeGreaterThan(0); + expect(body.trimStart().startsWith('# TestSprite Verification Loop')).toBe(true); + }); + + it('testsprite-verify.codex.md is ≤ 6144 bytes (6 KiB trim budget)', () => { + const body = loadCodexSkillBody(); + expect(Buffer.byteLength(body, 'utf8')).toBeLessThanOrEqual(6144); + }); +}); + +// --------------------------------------------------------------------------- +// MANAGED_SECTION sentinels +// --------------------------------------------------------------------------- + +describe('MANAGED_SECTION sentinels', () => { + it('BEGIN sentinel is an HTML comment', () => { + expect(MANAGED_SECTION_BEGIN.startsWith('')).toBe(true); + }); + + it('END sentinel is an HTML comment', () => { + expect(MANAGED_SECTION_END.startsWith('')).toBe(true); + }); + + it('sentinels contain testsprite identity marker', () => { + expect(MANAGED_SECTION_BEGIN.toLowerCase()).toContain('testsprite'); + expect(MANAGED_SECTION_END.toLowerCase()).toContain('testsprite'); + }); +}); + +// --------------------------------------------------------------------------- +// content integrity — codex target +// --------------------------------------------------------------------------- + +describe('content integrity — codex target (testsprite-verify.codex.md)', () => { + it('contains testsprite test run (load-bearing command)', () => { + const body = loadCodexSkillBody(); + expect(body).toContain('testsprite test run'); + }); + + it('contains --wait flag (load-bearing command string)', () => { + const body = loadCodexSkillBody(); + expect(body).toContain('--wait'); + }); + + it('contains test artifact get (load-bearing command)', () => { + const body = loadCodexSkillBody(); + expect(body).toContain('test artifact get'); + }); + + it('renderForTarget("codex") path is AGENTS.md', () => { + const STUB_CODEX_BODY = + '# TestSprite Verification Loop\ntestsprite test run\n--wait\ntest artifact get\n'; + const result = renderForTarget('codex', STUB_CODEX_BODY); + expect(result.path).toBe('AGENTS.md'); + }); + + it('renderForTarget("codex") content is the body unwrapped (no frontmatter)', () => { + const STUB_CODEX_BODY = + '# TestSprite Verification Loop\ntestsprite test run\n--wait\ntest artifact get\n'; + const result = renderForTarget('codex', STUB_CODEX_BODY); + // codex wrap is identity — no frontmatter fences + expect(result.content).toBe(STUB_CODEX_BODY); + expect(result.content).not.toContain('---'); + }); + + it('renderForTarget("codex") without body arg uses codex asset (not full skill body)', () => { + // The real codex asset is trimmed (no acronym line). + const result = renderForTarget('codex'); + // Plain Markdown; no frontmatter fences from own-file wraps + expect(result.content).not.toContain('name: testsprite-verify'); + expect(result.content).not.toContain('alwaysApply:'); + }); +}); diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts new file mode 100644 index 0000000..a212b9f --- /dev/null +++ b/src/lib/agent-targets.ts @@ -0,0 +1,129 @@ +import { readFileSync } from 'node:fs'; + +export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex'; + +export interface TargetSpec { + status: 'ga' | 'experimental'; + /** repo-relative landing path, POSIX separators */ + path: string; + /** + * 'own-file': the CLI owns the whole file (existing 4 targets). + * 'managed-section': the CLI writes only a sentinel-delimited section inside + * a potentially user-authored file (codex target, AGENTS.md). + */ + mode: 'own-file' | 'managed-section'; + /** wrap the canonical body in this target's frontmatter/header */ + wrap(body: string): string; +} + +export const SKILL_NAME = 'testsprite-verify'; + +/** + * Mirrors skill-template.md frontmatter `description`. ≤1536 chars (claude cap). + * A unit test asserts byte-identity with the template file. + */ +export const SKILL_DESCRIPTION = + 'TestSprite verification loop — after finishing a feature or fix in a TestSprite-tested repo, use the `testsprite` CLI to run the relevant TestSprite tests against the change and inspect any failure artifacts before reporting the work as done. Use whenever code has changed outside docs/config and is about to be reported complete — by running an existing test that covers the change, or by creating a new TestSprite test (a frontend plan, or a backend Python assertion) and running it to a terminal verdict.'; + +function wrapSkill(body: string): string { + return `---\nname: ${SKILL_NAME}\ndescription: ${SKILL_DESCRIPTION}\n---\n\n${body}\n`; +} + +function wrapMdc(body: string): string { + return `---\ndescription: ${SKILL_DESCRIPTION}\nalwaysApply: false\n---\n\n${body}\n`; +} + +export const TARGETS: Record = { + claude: { + status: 'ga', + path: '.claude/skills/testsprite-verify/SKILL.md', + mode: 'own-file', + wrap: wrapSkill, + }, + antigravity: { + status: 'experimental', + path: '.agents/skills/testsprite-verify/SKILL.md', + mode: 'own-file', + wrap: wrapSkill, + }, + cursor: { + status: 'experimental', + path: '.cursor/rules/testsprite-verify.mdc', + mode: 'own-file', + wrap: wrapMdc, + }, + cline: { + status: 'experimental', + path: '.clinerules/testsprite-verify.md', + mode: 'own-file', + wrap: body => body, + }, + /** + * codex target — managed-section mode. + * + * Codex auto-loads AGENTS.md from the project root (always-on, 32 KiB budget + * for the whole file). Unlike own-file targets, we must NOT clobber a user's + * existing AGENTS.md: we write only a sentinel-delimited section so other + * project instructions coexist. The sentinel pair is the canonical identity + * marker; the content between them is ours to replace. + * + * --force with managed-section: replaces the section unconditionally but + * NEVER destroys content outside the sentinels. No whole-file .bak is written + * for a section-only change — only a whole-file backup makes sense if the + * entire file was ours to own (own-file mode). User content is never at risk. + */ + codex: { + status: 'experimental', + path: 'AGENTS.md', + mode: 'managed-section', + // wrap is a no-op for managed-section — content is authored as plain Markdown + // with no frontmatter (AGENTS.md is plain prose, not a skill schema). + wrap: body => body, + }, +}; + +/** Sentinel pair that bounds our managed section in AGENTS.md. */ +export const MANAGED_SECTION_BEGIN = + ''; +export const MANAGED_SECTION_END = ''; + +type ReadFn = (url: URL) => string; + +const defaultRead: ReadFn = (url: URL) => readFileSync(url, 'utf8'); + +/** + * Load the canonical skill body. `../assets/...` resolves to `src/assets/...` + * under vitest (source) and `dist/assets/...` in the built binary — postbuild + * copies the tree so both paths exist. + * + * Injectable `read` fn keeps unit tests off disk. + */ +export function loadSkillBody(read: ReadFn = defaultRead): string { + return read(new URL('../assets/agent-skill/testsprite-verify.skill.md', import.meta.url)); +} + +/** + * Load the trimmed codex skill body (plain Markdown, no frontmatter). + * Designed for AGENTS.md managed-section injection. + */ +export function loadCodexSkillBody(read: ReadFn = defaultRead): string { + return read(new URL('../assets/agent-skill/testsprite-verify.codex.md', import.meta.url)); +} + +/** + * Convenience for piece-2: returns the exact bytes to write for a target. + * + * For own-file targets, `body` defaults to the full skill body. + * For the codex managed-section target, the trimmed codex body is used instead — + * pass an explicit `body` to override in tests. + */ +export function renderForTarget(t: AgentTarget, body?: string): { path: string; content: string } { + const spec = TARGETS[t]; + const resolvedBody = + body !== undefined + ? body + : spec.mode === 'managed-section' + ? loadCodexSkillBody() + : loadSkillBody(); + return { path: spec.path, content: spec.wrap(resolvedBody) }; +} diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts new file mode 100644 index 0000000..102208c --- /dev/null +++ b/src/lib/bundle.test.ts @@ -0,0 +1,594 @@ +/** + * Bundle writer unit tests. + * + * These exercise the pure helpers (`assertContextIntegrity`, + * `applyFailedOnly`, `pickCodeExtension`, `buildMeta`, + * `stepFilenamePrefix`) without filesystem I/O. The integration of + * `writeBundle` itself is covered in `commands/test.test.ts` (where + * the full http+fetch path is wired against MSW). + */ + +import { describe, expect, it } from 'vitest'; +import { + applyFailedOnly, + assertContextIntegrity, + BUNDLE_SCHEMA_VERSION, + buildMeta, + pickCodeExtension, + resolveBundleDir, + stepFilenamePrefix, + type AssertContextIntegrityOptions, +} from './bundle.js'; +import type { CliFailureContext } from '../commands/test.js'; + +const baseCtx: CliFailureContext = { + snapshotId: 'snap_abc', + testId: 'test_failed', + projectId: 'project_alice', + result: { + testId: 'test_failed', + status: 'failed', + startedAt: null, + finishedAt: '2026-05-05T12:34:58.000Z', + videoUrl: 'https://video.example.com/x.mp4', + failureAnalysisUrl: null, + snapshotId: 'snap_abc', + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + targetUrl: 'https://staging.example.com/checkout', + failedStepIndex: 5, + failureKind: 'assertion', + summary: { passed: 4, failed: 1, skipped: 0 }, + }, + steps: [3, 4, 5, 6, 7].map(i => ({ + testId: 'test_failed', + stepIndex: i, + action: 'click', + description: `step ${i}`, + status: i === 5 ? ('failed' as const) : ('passed' as const), + screenshotUrl: null, + htmlSnapshotUrl: `https://signed.example.com/${String(i).padStart(2, '0')}.html`, + runIdIfAvailable: 'run_abc', + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-05T12:34:56.000Z', + })), + code: { + testId: 'test_failed', + language: 'typescript', + framework: 'playwright', + code: 'inline', + codeVersion: 'v3', + etag: null, + }, + failure: { + rootCauseHypothesis: 'submit disabled', + recommendedFixTarget: { kind: 'unknown', reference: null, rationale: null }, + evidence: [3, 4, 5, 6, 7].map(i => ({ + kind: 'snapshot' as const, + stepIndex: i, + url: `https://signed.example.com/ev/${i}.html`, + summary: `step ${i}`, + })), + }, +}; + +describe('assertContextIntegrity', () => { + it('passes for a well-formed bundle', () => { + expect(() => assertContextIntegrity(baseCtx, 'req_x')).not.toThrow(); + }); + + it('throws on bundle.snapshotId !== result.snapshotId (the §3 invariant)', () => { + const forged: CliFailureContext = { + ...baseCtx, + snapshotId: 'snap_OUTER', + }; + // Backward-compat: details.reason is still the machine-matchable sentinel. + // New: details carries expectedSnapshotId + actualSnapshotId; message names them. + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('snap_OUTER'), + details: expect.objectContaining({ + reason: 'snapshot_id_mismatch', + expectedSnapshotId: 'snap_OUTER', + actualSnapshotId: baseCtx.result.snapshotId, + }), + }), + ); + }); + + it('throws when steps disagree on runIdIfAvailable', () => { + const [first, second] = baseCtx.steps as [ + (typeof baseCtx.steps)[number], + (typeof baseCtx.steps)[number], + ...Array<(typeof baseCtx.steps)[number]>, + ]; + const forged: CliFailureContext = { + ...baseCtx, + steps: [ + { ...first, runIdIfAvailable: 'run_a' }, + { ...second, runIdIfAvailable: 'run_b' }, + ], + }; + // Backward-compat: details.reason is still the machine-matchable sentinel. + // New: details carries observedRunIds listing all mixed runIds; message names them. + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('run_a'), + details: expect.objectContaining({ + reason: 'run_id_mismatch', + observedRunIds: expect.arrayContaining(['run_a', 'run_b']), + }), + }), + ); + }); + + it('surfaces both run IDs in message and details (dogfood L152)', () => { + // This is the exact L152 scenario: backend returns a bundle where steps + // carry a different runId than what the result row records. The error + // must name BOTH ids so the operator (and support) can diagnose whether + // the mismatch is data corruption or a comparison bug. + const [first, second] = baseCtx.steps as [ + (typeof baseCtx.steps)[number], + (typeof baseCtx.steps)[number], + ...Array<(typeof baseCtx.steps)[number]>, + ]; + const forged: CliFailureContext = { + ...baseCtx, + steps: [ + { ...first, runIdIfAvailable: 'run_xyz' }, + { ...second, runIdIfAvailable: 'run_abc' }, + ], + }; + let thrown: Error | undefined; + try { + assertContextIntegrity(forged, 'req_x'); + } catch (e) { + thrown = e as Error; + } + expect(thrown).toBeDefined(); + expect(thrown).toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ + reason: 'run_id_mismatch', + observedRunIds: expect.arrayContaining(['run_xyz', 'run_abc']), + }), + }); + // The human-readable message must name at least one of the offending ids. + expect(thrown!.message).toMatch(/run_xyz|run_abc/); + }); + + it('throws when evidence is non-empty but does not include the failed step (§6.2)', () => { + const forged: CliFailureContext = { + ...baseCtx, + failure: { + ...baseCtx.failure, + // Evidence covers neighbors only; failed step (5) absent. + evidence: baseCtx.failure.evidence.filter(e => e.stepIndex !== 5), + }, + }; + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ reason: 'evidence_missing_failed_step' }), + }), + ); + }); + + // Per codex round-1 P2: every embedded testId must equal ctx.testId. + // The §6.X failure-context wire shape duplicates `testId` in + // `result`, `code`, and each step so a bundle stitched from rows of + // two different tests is detectable without external state. + it('throws when result.testId !== ctx.testId (codex P2)', () => { + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, testId: 'test_OTHER' }, + }; + // Backward-compat: reason sentinel preserved. + // New: details carries expectedTestId + actualTestId; message names them. + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('test_OTHER'), + details: expect.objectContaining({ + reason: 'test_id_mismatch', + expectedTestId: baseCtx.testId, + actualTestId: 'test_OTHER', + }), + }), + ); + }); + + it('throws when code.testId !== ctx.testId (codex P2)', () => { + const forged: CliFailureContext = { + ...baseCtx, + code: { ...baseCtx.code, testId: 'test_OTHER' }, + }; + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('test_OTHER'), + details: expect.objectContaining({ + reason: 'test_id_mismatch', + expectedTestId: baseCtx.testId, + actualTestId: 'test_OTHER', + }), + }), + ); + }); + + it('throws when any step.testId !== ctx.testId (codex P2)', () => { + const [first, ...rest] = baseCtx.steps as [ + (typeof baseCtx.steps)[number], + ...Array<(typeof baseCtx.steps)[number]>, + ]; + const forged: CliFailureContext = { + ...baseCtx, + steps: [{ ...first, testId: 'test_OTHER' }, ...rest], + }; + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('test_OTHER'), + details: expect.objectContaining({ + reason: 'test_id_mismatch', + expectedTestId: baseCtx.testId, + actualTestId: 'test_OTHER', + }), + }), + ); + }); + + it('throws when result.codeVersion and code.codeVersion are both set but disagree', () => { + // §6.7: code is "version pinned to result.codeVersion." A bundle + // where the two versions disagree is exactly the drift the failure + // bundle exists to prevent — fix the wrong code, ship the wrong + // suggestion. Treat as corrupt. + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, codeVersion: 'v3' }, + code: { ...baseCtx.code, codeVersion: 'v4' }, + }; + // Backward-compat: reason sentinel preserved. + // New: details carries expectedCodeVersion + actualCodeVersion; message names them. + expect(() => assertContextIntegrity(forged, 'req_x')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('v4'), + details: expect.objectContaining({ + reason: 'code_version_mismatch', + expectedCodeVersion: 'v3', + actualCodeVersion: 'v4', + }), + }), + ); + }); + + it('allows codeVersion mismatch when one side is null (M2 transition surface)', () => { + // M2 backend hasn't shipped versioning yet — both result and code + // codeVersion can be null. The integrity check should only fire on + // a disagreement between two non-null values, not on partial + // adoption. + const halfway: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, codeVersion: null }, + code: { ...baseCtx.code, codeVersion: 'v3' }, + }; + expect(() => assertContextIntegrity(halfway, 'req_x')).not.toThrow(); + }); + + it('allows empty evidence (the legal "no diagnosis available" branch)', () => { + const ok: CliFailureContext = { + ...baseCtx, + failure: { ...baseCtx.failure, evidence: [] }, + }; + expect(() => assertContextIntegrity(ok, 'req_x')).not.toThrow(); + }); + + it('allows null failedStepIndex even when evidence is non-empty (assertion-level failure)', () => { + // When the test failed but no per-step row carried status:failed, + // the bundle ships failedStepIndex:null. The §6.2 invariant only + // applies when failedStepIndex is non-null. + const ok: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, failedStepIndex: null }, + }; + expect(() => assertContextIntegrity(ok, 'req_x')).not.toThrow(); + }); + + // M3.3 piece-4 — requireRunId option (opt-in, does not affect M2 callers) + + it('requireRunId: passes when runIdIfAvailable is non-null', () => { + // baseCtx has runIdIfAvailable: 'run_abc' + const opts: AssertContextIntegrityOptions = { requireRunId: true }; + expect(() => assertContextIntegrity(baseCtx, 'req_x', opts)).not.toThrow(); + }); + + it('requireRunId: throws run_id_missing when runIdIfAvailable is null', () => { + const noRunId: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: null }, + }; + const opts: AssertContextIntegrityOptions = { requireRunId: true }; + expect(() => assertContextIntegrity(noRunId, 'req_x', opts)).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ reason: 'run_id_missing' }), + }), + ); + }); + + it('requireRunId: is opt-in — M2 callers without the option pass with null runId', () => { + // This is the M2 non-regression: existing callers pass no opts → no change in behavior. + const m2Ctx: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: null }, + steps: baseCtx.steps.map(s => ({ ...s, runIdIfAvailable: null })), + }; + // Without opts (M2 path) — should not throw + expect(() => assertContextIntegrity(m2Ctx, 'req_x')).not.toThrow(); + // Explicitly opting out — same result + expect(() => assertContextIntegrity(m2Ctx, 'req_x', {})).not.toThrow(); + expect(() => assertContextIntegrity(m2Ctx, 'req_x', { requireRunId: false })).not.toThrow(); + }); + + // Item-10 forged-context tests — run-scoped step assertions + // + // These guard the case where a backend bug (or adversarial response) + // stitches step artifacts from a different run into a result envelope. + // Without the step-level check, `assertContextIntegrity` only detected + // "steps disagree with each other" — it accepted a bundle where all steps + // agreed on `run_other` while result carried `run_abc`. + + it('Item-10: rejects when result.runId is correct but step.runId is a different run', () => { + // Backend stitches step artifacts from "run_other" into a result for "run_abc". + // All steps agree with each other (set-size check passes) but disagree with result. + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc' }, + steps: baseCtx.steps.map(s => ({ ...s, runIdIfAvailable: 'run_other' })), + }; + // Backward-compat: reason sentinel preserved. + // New: details carries expectedRunId + actualRunId; message names them. + expect(() => assertContextIntegrity(forged, 'req_x', { requireRunId: true })).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('run_abc'), + details: expect.objectContaining({ + reason: 'run_id_mismatch', + expectedRunId: 'run_abc', + actualRunId: 'run_other', + }), + }), + ); + }); + + it('Item-10: rejects when a step has runIdIfAvailable === null (run-scoped path)', () => { + // A null step run ID is unacceptable in the run-scoped path: the + // backend must stamp every step with the exact run it belongs to. + // The early "distinct non-null runIds" check tolerates null (a null + // is a legacy un-stamped row, not cross-run stitching — dogfood + // 2026-06-04), so with a single real runId it does NOT fire. The + // run-scoped per-step equality check then catches the null step and + // names the exact offending step. + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc' }, + steps: [ + { ...baseCtx.steps[0]!, runIdIfAvailable: null }, // null step + ...baseCtx.steps.slice(1).map(s => ({ ...s, runIdIfAvailable: 'run_abc' })), + ], + }; + expect(() => assertContextIntegrity(forged, 'req_x', { requireRunId: true })).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ + reason: 'run_id_mismatch', + expectedRunId: 'run_abc', + actualRunId: null, + }), + }), + ); + }); + + it('test-scoped path tolerates a mix of one real runId and nulls (dogfood 2026-06-04)', () => { + // FE Portal step rows accumulate across runs; rows written before + // runId stamping carry `runIdIfAvailable: null`. A test run more than + // once across the M3.1 cutover therefore legitimately has one real + // runId plus several nulls. The test-scoped failure bundle + // (`test failure get `, requireRunId unset) must NOT reject + // it — the null is durable, so the old "re-fetch usually succeeds" + // hint never recovered. Only ≥2 *real* runIds are a true conflict. + const tolerated: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc' }, + steps: [ + { ...baseCtx.steps[0]!, runIdIfAvailable: 'run_abc' }, + { ...baseCtx.steps[1]!, runIdIfAvailable: null }, + ], + }; + expect(() => assertContextIntegrity(tolerated, 'req_x')).not.toThrow(); + }); + + it('Item-10: rejects when ALL steps have a uniform but wrong runId (requireRunId path)', () => { + // When all steps agree on a single runId but that runId differs from result.runIdIfAvailable, + // the "steps disagree among themselves" check passes (set size = 1) but the + // requireRunId per-step check fires, surfacing expectedRunId + actualRunId. + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc' }, + steps: baseCtx.steps.map(s => ({ ...s, runIdIfAvailable: 'run_other' })), + }; + // All steps null-uniform: use a separate ctx where all steps share null + // so the "mixed set" check doesn't fire and only requireRunId fires. + const forgedAllNull: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc' }, + steps: baseCtx.steps.map(s => ({ ...s, runIdIfAvailable: null })), + }; + // New: details carries expectedRunId + actualRunId (null). + expect(() => + assertContextIntegrity(forgedAllNull, 'req_x', { requireRunId: true }), + ).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('run_abc'), + details: expect.objectContaining({ + reason: 'run_id_mismatch', + expectedRunId: 'run_abc', + actualRunId: null, + }), + }), + ); + // When forged (all steps 'run_other'), uses requireRunId path since set size = 1. + expect(() => assertContextIntegrity(forged, 'req_x', { requireRunId: true })).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('run_abc'), + details: expect.objectContaining({ + reason: 'run_id_mismatch', + expectedRunId: 'run_abc', + actualRunId: 'run_other', + }), + }), + ); + }); + + it('Item-10: rejects when step.codeVersion disagrees with result.codeVersion', () => { + // Mixed version bundle — step was captured at v2, result at v3. + const forged: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: 'run_abc', codeVersion: 'v3' }, + steps: [ + { ...baseCtx.steps[0]!, runIdIfAvailable: 'run_abc', codeVersion: 'v2' }, // wrong version + ...baseCtx.steps + .slice(1) + .map(s => ({ ...s, runIdIfAvailable: 'run_abc', codeVersion: 'v3' })), + ], + }; + // Backward-compat: reason sentinel preserved. + // New: details carries expectedCodeVersion + actualCodeVersion; message names them. + expect(() => assertContextIntegrity(forged, 'req_x', { requireRunId: true })).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + message: expect.stringContaining('v3'), + details: expect.objectContaining({ + reason: 'code_version_mismatch', + expectedCodeVersion: 'v3', + actualCodeVersion: 'v2', + }), + }), + ); + }); + + it('Item-10 backward compat: M2 context (steps.runId = null, no requireRunId) still passes', () => { + // M2-shaped context: all runId fields are null. Without requireRunId: true, + // the step-level checks must not fire so M2 callers are unaffected. + const m2Latest: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, runIdIfAvailable: null }, + steps: baseCtx.steps.map(s => ({ ...s, runIdIfAvailable: null })), + }; + // No requireRunId → should not throw + expect(() => assertContextIntegrity(m2Latest, 'req_x')).not.toThrow(); + expect(() => assertContextIntegrity(m2Latest, 'req_x', {})).not.toThrow(); + expect(() => assertContextIntegrity(m2Latest, 'req_x', { requireRunId: false })).not.toThrow(); + }); +}); + +describe('applyFailedOnly', () => { + it('keeps the failed step ± 1 and drops outside-window steps', () => { + const out = applyFailedOnly(baseCtx); + expect(out.steps.map(s => s.stepIndex)).toEqual([4, 5, 6]); + // Evidence is filtered to the same window. + expect(out.failure.evidence.map(e => e.stepIndex)).toEqual([4, 5, 6]); + }); + + it('returns the bundle unchanged when failedStepIndex is null', () => { + const ctx: CliFailureContext = { + ...baseCtx, + result: { ...baseCtx.result, failedStepIndex: null }, + }; + const out = applyFailedOnly(ctx); + expect(out).toBe(ctx); + }); +}); + +describe('pickCodeExtension', () => { + it('language drives the choice', () => { + expect(pickCodeExtension('python', 'pytest')).toBe('py'); + expect(pickCodeExtension('typescript', 'playwright')).toBe('ts'); + expect(pickCodeExtension('javascript', 'playwright')).toBe('js'); + }); + + it('falls back to framework when language is unknown', () => { + expect(pickCodeExtension('opaque', 'pytest')).toBe('py'); + expect(pickCodeExtension('opaque', 'playwright')).toBe('ts'); + }); +}); + +describe('stepFilenamePrefix', () => { + it('zero-pads to 2 digits for index < 100', () => { + expect(stepFilenamePrefix(1)).toBe('01'); + expect(stepFilenamePrefix(5)).toBe('05'); + expect(stepFilenamePrefix(99)).toBe('99'); + }); + + it('widens to 3 digits at index 100 (never truncates)', () => { + // Per §7.2: "Step 100+ widens to three digits". Truncating would + // collide with another step's prefix and corrupt the filename + // map an agent uses to read the bundle. + expect(stepFilenamePrefix(100)).toBe('100'); + expect(stepFilenamePrefix(999)).toBe('999'); + }); +}); + +describe('buildMeta', () => { + it("mirrors the bundle's identity card", () => { + const meta = buildMeta(baseCtx, new Date('2026-05-05T12:35:01.000Z')); + expect(meta.schemaVersion).toBe(BUNDLE_SCHEMA_VERSION); + expect(meta.snapshotId).toBe(baseCtx.snapshotId); + expect(meta.testId).toBe(baseCtx.testId); + expect(meta.projectId).toBe(baseCtx.projectId); + expect(meta.codeVersion).toBe(baseCtx.code.codeVersion); + expect(meta.runIdIfAvailable).toBe(baseCtx.result.runIdIfAvailable); + expect(meta.targetUrl).toBe(baseCtx.result.targetUrl); + expect(meta.failedStepIndex).toBe(baseCtx.result.failedStepIndex); + expect(meta.failureKind).toBe(baseCtx.result.failureKind); + expect(meta.capturedAt).toBe(baseCtx.result.finishedAt); + expect(meta.fetchedAt).toBe('2026-05-05T12:35:01.000Z'); + }); + + it('uses result.codeVersion when code.codeVersion is null', () => { + const ctx: CliFailureContext = { + ...baseCtx, + code: { ...baseCtx.code, codeVersion: null }, + result: { ...baseCtx.result, codeVersion: 'v_from_result' }, + }; + expect(buildMeta(ctx).codeVersion).toBe('v_from_result'); + }); +}); + +describe('resolveBundleDir', () => { + it('rejects an empty path with VALIDATION_ERROR', () => { + expect(() => resolveBundleDir('')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + nextAction: expect.stringContaining('--out'), + }), + ); + }); + + it('resolves a relative path against cwd', () => { + const out = resolveBundleDir('./tmp/x'); + expect(out.endsWith('/tmp/x')).toBe(true); + expect(out.startsWith('/')).toBe(true); + }); + + it('strips a trailing slash', () => { + const out = resolveBundleDir('/tmp/x/'); + expect(out).toBe('/tmp/x'); + }); +}); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts new file mode 100644 index 0000000..d50948c --- /dev/null +++ b/src/lib/bundle.ts @@ -0,0 +1,802 @@ +/** + * §6.7 FailureContext bundle writer. + * + * Given a `FailureContext` JSON envelope (returned by + * `GET /api/cli/v1/tests/{id}/failure`), `writeBundle` dereferences + * every presigned URL into local bytes and writes the §7 disk layout + * under ``. The contract is "agent-safe" — every guarantee in + * the FailureContext spec §3 + §7 is enforced here: + * + * 1. **Atomic** — every artifact in the bundle came from one + * `snapshotId`. We refuse a forged response where + * `bundle.snapshotId !== result.snapshotId` or steps disagree on + * `runIdIfAvailable` (`assertContextIntegrity`). + * 2. **Atomic on disk** — we write to `/.tmp/...` first and + * `rename()` each file into place. `meta.json` is renamed last so + * the presence of `/meta.json` ⇔ "bundle is complete and + * self-consistent". + * 3. **`.partial` on failure** — any download or fs failure leaves + * a `/.partial` marker (with `requestId`, error summary, + * snapshotId) and exits non-zero. Agents check `.partial` before + * consuming. + * 4. **Bounded budget** — the writer streams URL → file (no full- + * buffer), so a 200MB video doesn't sit in V8's heap. + * + * What this module deliberately does NOT do: + * + * - Re-derive `snapshotId`. The CLI trusts whatever the facade + * returned. A snapshotId mismatch is a backend bug and we surface + * it as a typed validation error rather than try to "heal" it. + * - Generate `summary` strings. Those are server-side per §6.1 of + * the spec; the CLI emits whatever the facade produced. + * - Resume after a partial bundle. M2 always re-fetches from scratch + * when the agent re-runs. M3 may add resume. + */ + +import { mkdir, mkdtemp, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'; +import type { Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; +import { createWriteStream } from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import type { CliFailureContext, CliTestStep } from '../commands/test.js'; +import { ApiError, TransportError } from './errors.js'; +import type { FetchImpl } from './http.js'; + +/** Schema version stamped into `meta.json`. Bumps with the contract. */ +export const BUNDLE_SCHEMA_VERSION = 'cli-v1' as const; + +/** Default radius around the failed step when `--failed-only` is set. */ +export const FAILED_ONLY_RADIUS = 1; + +/** + * Sentinel reasons emitted on the `details.reason` field when the + * integrity check throws. Exported so callers can branch on a typed + * literal rather than parsing the message text. + */ +export type BundleIntegrityReason = + | 'snapshot_id_mismatch' + | 'run_id_mismatch' + | 'code_version_mismatch' + | 'evidence_missing_failed_step' + | 'run_id_missing' + | 'test_id_mismatch'; + +/** + * Options for {@link assertContextIntegrity}. + */ +export interface AssertContextIntegrityOptions { + /** + * When `true`, the run-scoped path requires `ctx.meta.runId` (i.e., + * `ctx.result.runIdIfAvailable`) to be present. M2 callers pass + * `{}` (or omit entirely) to preserve the opt-in behavior. + */ + requireRunId?: boolean; +} + +/** + * Identity card written as `/meta.json`. Agents read this first — + * if any other file's `snapshotId` disagrees, the bundle is corrupt + * and they must refuse to act. + * + * `runIdIfAvailable` and `codeVersion` are surfaced from `result` + * (already shared across every step in a valid bundle, so the meta + * carries the bundle-wide value rather than the per-step one). + */ +export interface BundleMeta { + schemaVersion: typeof BUNDLE_SCHEMA_VERSION; + snapshotId: string; + testId: string; + projectId: string; + codeVersion: string | null; + runIdIfAvailable: string | null; + targetUrl: string | null; + failedStepIndex: number | null; + failureKind: string | null; + /** When the backend captured the snapshot. Mirrors `result.finishedAt`. */ + capturedAt: string | null; + /** When the CLI fetched the bundle (this run's clock). */ + fetchedAt: string; +} + +export interface WriteBundleOptions { + /** Absolute or relative target directory. */ + dir: string; + /** Apply the §7.4 `--failed-only` filter (failed step ± 1) before downloading. */ + failedOnly: boolean; + /** Custom fetch impl for tests. Defaults to global `fetch`. */ + fetchImpl?: FetchImpl; + /** Server requestId to embed in `.partial` on failure. */ + requestId?: string; +} + +export interface WriteBundleResult { + meta: BundleMeta; + /** Absolute path of the bundle directory. */ + dir: string; + /** Files actually written, relative to `dir`. */ + files: string[]; +} + +/** + * Compose the FailureContext spec §3 invariants into a single + * trip wire. Throwing here is the agent-safety gate — we trust the + * facade for the data, not for self-consistency. + * + * - Bundle-level `snapshotId` MUST equal `result.snapshotId`. + * - All steps in the bundle MUST share `runIdIfAvailable` (or all + * be `null`). Mixed values means a stitched read leaked through. + * + * The error envelope is `VALIDATION_ERROR` rather than `INTERNAL` + * because the operator-facing remediation is "retry once" (the + * snapshot may genuinely have been mid-mutation; a fresh fetch will + * succeed). `requestId` is the original request id so support can + * trace the bad envelope back. + */ +export function assertContextIntegrity( + ctx: CliFailureContext, + requestId: string, + opts?: AssertContextIntegrityOptions, +): void { + if (opts?.requireRunId && !ctx.result.runIdIfAvailable) { + throw bundleIntegrityError( + 'run_id_missing', + 'meta.runId (result.runIdIfAvailable) is required for run-scoped bundles but is null or missing', + requestId, + ); + } + if (ctx.snapshotId !== ctx.result.snapshotId) { + throw bundleIntegrityError( + 'snapshot_id_mismatch', + `Bundle integrity check failed: expected snapshotId=${ctx.snapshotId} got snapshotId=${ctx.result.snapshotId}`, + requestId, + { expectedSnapshotId: ctx.snapshotId, actualSnapshotId: ctx.result.snapshotId }, + ); + } + // Per codex round-1 P2: every embedded testId must equal ctx.testId. + // §6.X duplicates `testId` in `result`, `code`, and each step so a + // bundle stitched together from rows of two different tests is + // detectable without external state. Without this gate, an agent + // could open `meta.json` for `test_A`, edit the file the bundle + // claims is its code, and have the edit actually target `test_B`'s + // source — exactly the cross-test contamination the failure bundle + // exists to prevent. + if (ctx.result.testId !== ctx.testId) { + throw bundleIntegrityError( + 'test_id_mismatch', + `Bundle integrity check failed: expected testId=${ctx.testId} got testId=${ctx.result.testId} (in result)`, + requestId, + { expectedTestId: ctx.testId, actualTestId: ctx.result.testId }, + ); + } + if (ctx.code.testId !== ctx.testId) { + throw bundleIntegrityError( + 'test_id_mismatch', + `Bundle integrity check failed: expected testId=${ctx.testId} got testId=${ctx.code.testId} (in code)`, + requestId, + { expectedTestId: ctx.testId, actualTestId: ctx.code.testId }, + ); + } + for (const step of ctx.steps) { + if (step.testId !== ctx.testId) { + throw bundleIntegrityError( + 'test_id_mismatch', + `Bundle integrity check failed: expected testId=${ctx.testId} got testId=${step.testId} (in step[${step.stepIndex}])`, + requestId, + { expectedTestId: ctx.testId, actualTestId: step.testId }, + ); + } + } + // §6.7: code is "version pinned to result.codeVersion." If both + // sides are non-null and disagree, the bundle stitched code from + // one version with a result from another — exactly the drift case + // the failure bundle exists to prevent. Both-null is fine (the M2 + // backend hasn't shipped versioning yet); only mismatched pairs + // are corrupt. + if ( + ctx.result.codeVersion !== null && + ctx.code.codeVersion !== null && + ctx.result.codeVersion !== ctx.code.codeVersion + ) { + throw bundleIntegrityError( + 'code_version_mismatch', + `Bundle integrity check failed: expected codeVersion=${ctx.result.codeVersion} got codeVersion=${ctx.code.codeVersion}`, + requestId, + { expectedCodeVersion: ctx.result.codeVersion, actualCodeVersion: ctx.code.codeVersion }, + ); + } + // Only DISTINCT NON-NULL runIds indicate cross-run stitching. A `null` + // `runIdIfAvailable` just means that step row predates runId stamping — + // FE Portal step rows accumulate across runs and older rows were never + // stamped, so a multi-run test legitimately carries a mix of one real + // runId and several nulls. Counting null as a distinct value made + // `test failure get ` permanently fail on any test that ran + // more than once across the M3.1 cutover, and the "re-fetch usually + // succeeds" hint was wrong because the null is durable, not transient + // (dogfood 2026-06-04). Tolerate null; flag only when ≥2 *real* runIds + // are present (genuine cross-run stitching). The run-scoped path below + // (`requireRunId`) keeps the stricter per-step equality check. + const nonNullRunIds = new Set(); + for (const step of ctx.steps) { + if (step.runIdIfAvailable != null) nonNullRunIds.add(step.runIdIfAvailable); + } + if (nonNullRunIds.size > 1) { + const runIdList = [...nonNullRunIds]; + throw bundleIntegrityError( + 'run_id_mismatch', + `Bundle integrity check failed: steps carry mixed runIds [${runIdList.join(', ')}]`, + requestId, + { observedRunIds: runIdList }, + ); + } + + // Run-scoped path: additionally assert that every step's runIdIfAvailable + // matches result.runIdIfAvailable. The check above only catches steps + // that disagree with *each other*; a backend stitching all steps from + // "run_other" into a result for "run_abc" would pass the set-size check + // (all steps agree) but fail here. + // + // Also assert per-step codeVersion matches result.codeVersion when both + // are present — a mixed version bundle could point an agent at the wrong + // fix target. + if (opts?.requireRunId) { + const expectedRunId = ctx.result.runIdIfAvailable!; // guarded by run_id_missing check above + const resultCodeVersion = ctx.result.codeVersion; + for (const step of ctx.steps) { + if (step.runIdIfAvailable !== expectedRunId) { + throw bundleIntegrityError( + 'run_id_mismatch', + `Bundle integrity check failed: expected runId=${expectedRunId} got runId=${String(step.runIdIfAvailable)} (in step[${step.stepIndex}])`, + requestId, + { expectedRunId, actualRunId: step.runIdIfAvailable }, + ); + } + if ( + resultCodeVersion !== null && + step.codeVersion !== null && + step.codeVersion !== resultCodeVersion + ) { + throw bundleIntegrityError( + 'code_version_mismatch', + `Bundle integrity check failed: expected codeVersion=${resultCodeVersion} got codeVersion=${step.codeVersion} (in step[${step.stepIndex}])`, + requestId, + { expectedCodeVersion: resultCodeVersion, actualCodeVersion: step.codeVersion }, + ); + } + } + } + if (ctx.failure.evidence.length > 0 && ctx.result.failedStepIndex !== null) { + const hasFailedStep = ctx.failure.evidence.some( + e => e.stepIndex === ctx.result.failedStepIndex, + ); + if (!hasFailedStep) { + // §6.2: when evidence[] is non-empty, at least one entry must + // attach to the failed step so an agent can route on it. + throw bundleIntegrityError( + 'evidence_missing_failed_step', + `bundle evidence does not include failedStepIndex=${ctx.result.failedStepIndex}`, + requestId, + ); + } + } +} + +/** + * Apply the `--failed-only` filter — drop neighbor steps and their + * evidence, keep the failed step. Idempotent: returns `ctx` unchanged + * when `failedStepIndex` is null (no step-level diagnosis to filter) + * or when the existing window already matches. + */ +export function applyFailedOnly(ctx: CliFailureContext): CliFailureContext { + const failedIdx = ctx.result.failedStepIndex; + if (failedIdx === null) return ctx; + const inWindow = (idx: number) => Math.abs(idx - failedIdx) <= FAILED_ONLY_RADIUS; + const filteredSteps = ctx.steps.filter(s => inWindow(s.stepIndex)); + const filteredEvidence = ctx.failure.evidence.filter(e => inWindow(e.stepIndex)); + return { + ...ctx, + steps: filteredSteps, + failure: { ...ctx.failure, evidence: filteredEvidence }, + }; +} + +/** + * Resolve the user-supplied `--out` path into an absolute directory. + * Empty strings are rejected with `VALIDATION_ERROR` for consistency + * with `test code get --out`. We do NOT pre-create the directory or + * its `.tmp` child — `writeBundle` mkdir's after the integrity check + * passes so a forged response never modifies the operator's filesystem. + */ +export function resolveBundleDir(rawPath: string): string { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Invalid request.', + nextAction: 'Flag `--out` is invalid: must be a non-empty directory path.', + requestId: 'local', + details: { field: 'out', reason: 'must be a non-empty directory path' }, + }, + }); + } + const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + return isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed); +} + +/** + * Pick a code-file extension for `/code.` based on the §6.3 + * `language` + `framework`. The extension matches the test code's + * language, not the user-app code under test. + */ +export function pickCodeExtension(language: string, framework: string): string { + if (language === 'python') return 'py'; + if (language === 'javascript') return 'js'; + if (language === 'typescript') return 'ts'; + // Fallback: framework-keyed default. pytest is python, playwright TS. + if (framework === 'pytest') return 'py'; + return 'ts'; +} + +/** + * Step filename per §7.2 — 1-based index, zero-padded to two digits + * for indices ≤ 99, three digits for ≥ 100. `${stepIndex}-snapshot.html` + * etc. Never truncates: an agent constructing the path with + * `padStart(2, '0')` works for the common case, and step ≥ 100 widens + * naturally without breaking existing fixtures. + */ +export function stepFilenamePrefix(stepIndex: number): string { + return stepIndex >= 100 ? String(stepIndex).padStart(3, '0') : String(stepIndex).padStart(2, '0'); +} + +/** + * Build the §7.1 meta.json given a context. Pure function, no I/O — + * lets specs assert the meta shape without touching disk. + */ +export function buildMeta(ctx: CliFailureContext, fetchedAt: Date = new Date()): BundleMeta { + return { + schemaVersion: BUNDLE_SCHEMA_VERSION, + snapshotId: ctx.snapshotId, + testId: ctx.testId, + projectId: ctx.projectId, + codeVersion: ctx.code.codeVersion ?? ctx.result.codeVersion ?? null, + runIdIfAvailable: ctx.result.runIdIfAvailable, + targetUrl: ctx.result.targetUrl, + failedStepIndex: ctx.result.failedStepIndex, + failureKind: ctx.result.failureKind, + capturedAt: ctx.result.finishedAt, + fetchedAt: fetchedAt.toISOString(), + }; +} + +/** + * Top-level bundle writer. The order matters: + * + * 1. `assertContextIntegrity` — fail closed on a forged envelope. + * 2. `applyFailedOnly` — narrow before download (saves bytes). + * 3. `mkdir /.tmp/` — fresh; clean any stale temp. + * 4. `writeFile result.json / failure.json / code.` — local data. + * 5. `fetch + stream` for `video.mp4` (when set) and per-step + * snapshot/screenshot/evidence-json files. + * 6. `writeFile meta.json` LAST — its presence means "bundle complete". + * 7. `rename .tmp/ -> ` for every file. Last rename + * makes meta.json visible. + * + * On any failure between (3) and (6), write a `/.partial` marker + * and re-throw so `index.ts` produces a typed exit code (UNAVAILABLE + * → 10 by default; the original ApiError is preserved when applicable). + */ +export async function writeBundle( + ctx: CliFailureContext, + options: WriteBundleOptions, +): Promise { + const requestId = options.requestId ?? 'local'; + assertContextIntegrity(ctx, requestId); + + const filtered = options.failedOnly ? applyFailedOnly(ctx) : ctx; + // Re-check after filtering — in theory a buggy filter could leave a + // mismatched evidence list. Cheap defense-in-depth. + assertContextIntegrity(filtered, requestId); + + const dir = resolveBundleDir(options.dir); + const fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis); + const meta = buildMeta(filtered); + const codeExt = pickCodeExtension(filtered.code.language, filtered.code.framework); + + await mkdir(dir, { recursive: true }); + // Fresh `.tmp/` each run. A previous SIGKILL'd run can leave bytes + // behind; we don't want to mix them with the new fetch. + const tmpDir = await freshTmpDir(dir); + const stepsTmpDir = join(tmpDir, 'steps'); + await mkdir(stepsTmpDir, { recursive: true }); + + const filesWritten: string[] = []; + + try { + // result.json + failure.json + code. are local writes; no + // network, no streaming. + await writeFile( + join(tmpDir, 'result.json'), + JSON.stringify(filtered.result, null, 2) + '\n', + 'utf8', + ); + filesWritten.push('result.json'); + + await writeFile( + join(tmpDir, 'failure.json'), + JSON.stringify(filtered.failure, null, 2) + '\n', + 'utf8', + ); + filesWritten.push('failure.json'); + + const codeFile = `code.${codeExt}`; + if (isPresignedUrl(filtered.code.code)) { + // §6.3 alt: when the `code` field is a presigned URL (>= 100 KB + // bodies), stream the URL into the file rather than embedding + // the URL in code.. M2's backend hasn't shipped the + // >=100KB branch yet, but the contract supports it. + await streamUrlToFile(filtered.code.code, join(tmpDir, codeFile), fetchImpl); + } else { + await writeFile(join(tmpDir, codeFile), filtered.code.code, 'utf8'); + } + filesWritten.push(codeFile); + + // Optional video. Wire field is `result.videoUrl` (not in `failure`), + // and the CLI surfaces it as a top-level on-disk artifact for agent + // ergonomics — the agent doesn't have to know which sub-object held + // it on the wire. + if (filtered.result.videoUrl) { + await streamUrlToFile(filtered.result.videoUrl, join(tmpDir, 'video.mp4'), fetchImpl); + filesWritten.push('video.mp4'); + } + + for (const step of filtered.steps) { + await writeStepArtifacts( + step, + filtered.failure.evidence, + stepsTmpDir, + fetchImpl, + filesWritten, + ); + } + + // meta.json LAST. Its presence is the sentinel that the bundle + // is complete. An agent that opens and finds meta.json can + // safely consume. + await writeFile(join(tmpDir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n', 'utf8'); + filesWritten.push('meta.json'); + + // Atomic rename: move every file from /.tmp/* into /*. + // Steps subdir gets renamed wholesale; meta.json renames last so + // its visibility implies "bundle is complete and atomic on disk." + await commitBundle(tmpDir, dir, filesWritten); + + return { meta, dir, files: filesWritten }; + } catch (err) { + await writePartialMarker(dir, err, requestId, ctx.snapshotId).catch(() => undefined); + throw err; + } +} + +/** + * Make a fresh `/.tmp/` directory, removing any pre-existing + * orphaned tmp from a prior aborted run. Using `mkdtemp` would dodge + * the rm step but would make the rename target unpredictable; this + * way the temp path is stable (`/.tmp`) so a SIGKILL between two + * runs doesn't leave a tree of `.tmp.` directories. + */ +async function freshTmpDir(dir: string): Promise { + const tmpDir = join(dir, '.tmp'); + await rm(tmpDir, { recursive: true, force: true }); + await mkdir(tmpDir, { recursive: true }); + return tmpDir; +} + +/** + * Rename `/` → `/` for every file in `files`. + * + * Critical ordering for atomicity (the §3 "agent-safe" contract): + * + * 1. **Remove the OLD `meta.json` first.** The bundle's completion + * signal is `meta.json`'s presence; an agent reading `` while + * we're mutating it must see "no meta → bundle absent or + * mid-write" rather than "meta points at a snapshot that's already + * been partially overwritten." Removing the old meta is what + * makes the rest of the swap safe to do in place. + * 2. Wipe stale top-level files (e.g. an old `video.mp4` when the new + * bundle has no video). Without this, a fresh bundle could ship + * with a stale video lingering at the top level. + * 3. Replace `/steps/` wholesale. + * 4. Rename top-level files into place. + * 5. **Rename `meta.json` LAST.** Its visible presence is the atomic + * completion signal; until step 5 lands, agents see "incomplete." + * + * The window between (1) and (5) is bounded by a handful of `rename` + * syscalls — small enough that a SIGKILL there is rare, and any agent + * caught reading the dir during it sees no meta and refuses to consume + * (per §7.3). That's what we want. + */ +async function commitBundle( + tmpDir: string, + dir: string, + files: ReadonlyArray, +): Promise { + // (1) Remove the prior bundle's completion signal FIRST. + await unlink(join(dir, 'meta.json')).catch(() => undefined); + + // (2) Sweep stale top-level files that the new bundle won't write. + // If the prior run wrote `video.mp4` and the new run has no video, + // an in-place rename leaves the old video lingering. Enumerate + // current top-level entries and remove anything that isn't being + // freshly renamed in. + const topLevel = files.filter(f => !f.startsWith('steps/')); + const newTopLevelSet = new Set(topLevel); + newTopLevelSet.add('meta.json'); // about to land last, do not delete + const existing = await readdir(dir).catch(() => [] as string[]); + for (const entry of existing) { + // Preserve the writer's own scratch dir + the .partial marker + // (we'll re-evaluate .partial at the end of commit). Anything else + // not-listed in the new bundle is stale. + if (entry === '.tmp' || entry === '.partial') continue; + if (newTopLevelSet.has(entry)) continue; + if (entry === 'steps') continue; // handled below + await rm(join(dir, entry), { recursive: true, force: true }); + } + + // (3) Replace `/steps/` with `/steps/`. + const stepsTmp = join(tmpDir, 'steps'); + const stepsDir = join(dir, 'steps'); + await rm(stepsDir, { recursive: true, force: true }); + if (await dirExists(stepsTmp)) { + await rename(stepsTmp, stepsDir); + } + + // (4) Top-level files (result/failure/code/video). meta.json renames + // LAST; track it separately. + const metaIdx = topLevel.indexOf('meta.json'); + const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; + for (const file of beforeMeta) { + await rename(join(tmpDir, file), join(dir, file)); + } + + // (5) meta.json LAST → atomic completion signal. + if (metaIdx >= 0) { + await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); + } + + // .partial from a prior aborted run is now stale. Remove it so an + // agent inspecting the dir sees only the fresh bundle. + await unlink(join(dir, '.partial')).catch(() => undefined); + + // Clean up the now-empty tmp dir. + await rm(tmpDir, { recursive: true, force: true }); +} + +async function dirExists(path: string): Promise { + try { + const s = await stat(path); + return s.isDirectory(); + } catch { + return false; + } +} + +/** + * Write per-step artifacts under `/steps/`. Each step gets up to + * three files: `-screenshot.png` (when a screenshot URL exists — + * M2 backend never sets one yet), `-snapshot.html` (when + * `htmlSnapshotUrl` exists), and `-evidence.json` (a small JSON + * file containing the non-snapshot evidence summaries — log/network/ + * console — for that step). The evidence-json file is omitted when + * there's nothing to put in it; it's NOT a placeholder. + */ +async function writeStepArtifacts( + step: CliTestStep, + allEvidence: ReadonlyArray, + stepsTmpDir: string, + fetchImpl: FetchImpl, + filesWritten: string[], +): Promise { + const prefix = stepFilenamePrefix(step.stepIndex); + + if (step.screenshotUrl) { + const file = `${prefix}-screenshot.png`; + await streamUrlToFile(step.screenshotUrl, join(stepsTmpDir, file), fetchImpl); + filesWritten.push(`steps/${file}`); + } + + if (step.htmlSnapshotUrl) { + const file = `${prefix}-snapshot.html`; + await streamUrlToFile(step.htmlSnapshotUrl, join(stepsTmpDir, file), fetchImpl); + filesWritten.push(`steps/${file}`); + } + + // Evidence sidecar artifacts. Per codex round-1 P2: the bundle must + // be self-contained — every `evidence[].url` in the failure response + // resolves to a local file by the time `meta.json` is written. The + // earlier filter dropped screenshot/snapshot kinds on the assumption + // they were always duplicates of `step.screenshotUrl` / + // `step.htmlSnapshotUrl`, but if the evidence carries a different URL + // (different snapshot variant, a per-evidence sidecar shot, etc.) the + // bundle silently lost it and `failure.json` still referenced an + // expiring presigned link. Rule now: include every evidence entry; + // remap to the existing step file when URLs match, download a fresh + // sidecar otherwise. + // + // Critical: leaving any URL inside `-evidence.json` means the + // bundle claims completeness but actually points at presigned URLs + // that expire after 15 min — an agent opening the bundle one hour + // later would see metadata referencing dead links. Streaming the + // bytes here makes the bundle self-contained. + const sidecar = allEvidence.filter(e => e.stepIndex === step.stepIndex); + if (sidecar.length > 0) { + const dereferenced = await Promise.all( + sidecar.map(async (entry, i) => { + // Reuse the already-downloaded step file when the evidence URL + // matches the step's primary screenshot/snapshot URL. Cheap + // dedupe — no extra HTTP round-trip, no duplicate bytes on disk. + if (entry.kind === 'screenshot' && step.screenshotUrl && step.screenshotUrl === entry.url) { + return { + kind: entry.kind, + stepIndex: entry.stepIndex, + summary: entry.summary, + path: `steps/${prefix}-screenshot.png`, + }; + } + if ( + entry.kind === 'snapshot' && + step.htmlSnapshotUrl && + step.htmlSnapshotUrl === entry.url + ) { + return { + kind: entry.kind, + stepIndex: entry.stepIndex, + summary: entry.summary, + path: `steps/${prefix}-snapshot.html`, + }; + } + const ext = sidecarExtension(entry.kind); + const filename = `${prefix}-${entry.kind}-${i}.${ext}`; + await streamUrlToFile(entry.url, join(stepsTmpDir, filename), fetchImpl); + filesWritten.push(`steps/${filename}`); + return { + kind: entry.kind, + stepIndex: entry.stepIndex, + summary: entry.summary, + // Replace the (soon-expired) URL with a path relative to the + // bundle root so an agent can resolve it without ambient + // knowledge of the bundle's working directory. + path: `steps/${filename}`, + }; + }), + ); + const file = `${prefix}-evidence.json`; + await writeFile(join(stepsTmpDir, file), JSON.stringify(dereferenced, null, 2) + '\n', 'utf8'); + filesWritten.push(`steps/${file}`); + } +} + +function sidecarExtension(kind: 'log' | 'network' | 'console' | 'screenshot' | 'snapshot'): string { + if (kind === 'network' || kind === 'console') return 'json'; + if (kind === 'screenshot') return 'png'; + if (kind === 'snapshot') return 'html'; + // log evidence is text-shaped — `.txt` is the safest default. The + // actual content-type the URL serves may be log-specific (e.g. + // .ndjson for stream-of-events logs); we don't try to guess. + return 'txt'; +} + +/** + * Stream a presigned URL into a file with bounded retry on transport + * failures. 4xx (presigned URL expired or unauthorized) is NOT + * retried — the URL won't recover on its own; an upstream re-fetch + * is the only fix. Maps to `UNAVAILABLE` so `index.ts` exits 10. + * + * Streaming uses `pipeline` to honor backpressure: a slow disk pauses + * the upstream reader rather than buffering chunks in V8 heap. This + * matters for video files (multi-MB) and for very large HTML + * snapshots. + */ +export async function streamUrlToFile( + url: string, + filePath: string, + fetchImpl: FetchImpl, +): Promise { + let response: Response; + try { + response = await fetchImpl(url); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + } + if (!response.ok) { + throw ApiError.fromEnvelope({ + error: { + code: 'UNAVAILABLE', + message: `Failed to download presigned URL (HTTP ${response.status}).`, + nextAction: + 'Re-run `testsprite test failure get`. Presigned URLs in the bundle expire after 15 minutes.', + requestId: 'local', + details: { status: response.status, url }, + }, + }); + } + if (!response.body) { + // Some test runtimes / fetch polyfills don't expose `body` as a + // ReadableStream. Fall back to a buffered write — same correctness, + // just no streaming benefit. The bundle is bounded by the + // backend's 15-min TTL, so even a multi-MB video buffers fully in + // a tolerable amount of memory. + const buffer = Buffer.from(await response.arrayBuffer()); + await writeFile(filePath, buffer); + return; + } + await mkdir(dirname(filePath), { recursive: true }); + // `response.body` is a Web ReadableStream. Node's `pipeline` accepts + // it via `Readable.fromWeb` (Node ≥ 18). Wrap in a try so any error + // from the stream propagates as a TransportError, preserving the + // exit-code contract. + const fileSink = createWriteStream(filePath); + try { + const webBody = response.body as unknown as NodeReadableStream; + const { Readable } = await import('node:stream'); + const nodeStream = Readable.fromWeb(webBody); + await pipeline(nodeStream, fileSink as unknown as Writable); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new TransportError(`Failed mid-download of ${url}: ${message}`); + } +} + +function isPresignedUrl(value: string): boolean { + return value.startsWith('https://'); +} + +/** + * Write `/.partial` so an agent inspecting the dir can detect + * an aborted bundle. The marker is small JSON — the contract is "if + * `.partial` exists, do not consume `` until the CLI is re-run + * and `.partial` disappears." + * + * We DO NOT clean the `.tmp/` contents on partial — leaving them on + * disk is intentional for forensics. The next successful `writeBundle` + * call clears them. + */ +async function writePartialMarker( + dir: string, + err: unknown, + requestId: string, + snapshotId: string, +): Promise { + await mkdir(dir, { recursive: true }); + const body = { + schemaVersion: BUNDLE_SCHEMA_VERSION, + snapshotId, + requestId, + error: err instanceof Error ? err.message : String(err), + createdAt: new Date().toISOString(), + }; + await writeFile(join(dir, '.partial'), JSON.stringify(body, null, 2) + '\n', 'utf8'); +} + +function bundleIntegrityError( + reason: BundleIntegrityReason, + message: string, + requestId: string, + ids?: Record, +): ApiError { + return ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message, + nextAction: + 'Re-run `testsprite test failure get`. The backend may have been mid-snapshot — a fresh fetch usually succeeds. Report the requestId to support if it persists.', + requestId, + details: { reason, ...(ids ?? {}) }, + }, + }); +} + +// Avoid unused-import lint warning for the `mkdtemp` and `readdir` +// helpers reserved for forensics + future resume work. +void mkdtemp; +void readdir; diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts new file mode 100644 index 0000000..f18568e --- /dev/null +++ b/src/lib/client-factory.test.ts @@ -0,0 +1,286 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { + DRY_RUN_API_KEY, + DRY_RUN_BANNER, + emitDryRunBanner, + makeHttpClient, + resetDryRunBannerForTesting, + resolveRequestTimeoutMs, +} from './client-factory.js'; +import { + REQUEST_TIMEOUT_DEFAULT_MS, + REQUEST_TIMEOUT_MAX_MS, + REQUEST_TIMEOUT_MIN_MS, +} from './http.js'; + +const NO_CREDS_PATH = '/tmp/testsprite-cli-test-no-such-file-1234.ini'; + +describe('makeHttpClient — dry-run path', () => { + afterEach(() => { + resetDryRunBannerForTesting(); + }); + + it('does not require a credentials file or env var', async () => { + const stderrLines: string[] = []; + // Real path would throw AUTH_REQUIRED here; dry-run should succeed. + const client = makeHttpClient( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: line => stderrLines.push(line), + }, + ); + const me = await client.get<{ userId: string }>('/me'); + expect(me.userId).toBeTruthy(); + expect(stderrLines).toContain(DRY_RUN_BANNER); + }); + + it('emits the banner once per process even across multiple clients', () => { + const lines: string[] = []; + const stderr = (line: string) => lines.push(line); + emitDryRunBanner(stderr); + emitDryRunBanner(stderr); + emitDryRunBanner(stderr); + expect(lines.filter(l => l === DRY_RUN_BANNER)).toHaveLength(1); + }); + + it('debug events include mode: "dry-run"', async () => { + const stderrLines: string[] = []; + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: true, dryRun: true }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: line => stderrLines.push(line), + }, + ); + await client.get('/projects'); + // Format is now: [debug ] {...} + const debug = stderrLines.filter(l => l.startsWith('[debug ')); + expect(debug.length).toBeGreaterThan(0); + for (const line of debug) { + expect(line).toContain('"mode":"dry-run"'); + } + }); + + it('debug lines carry an ISO 8601 timestamp (dogfood item 2)', async () => { + const stderrLines: string[] = []; + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: true, dryRun: true }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: line => stderrLines.push(line), + }, + ); + await client.get('/projects'); + const debug = stderrLines.filter(l => l.startsWith('[debug ')); + expect(debug.length).toBeGreaterThan(0); + // Each line should match: [debug 2026-05-21T12:34:56.789Z] {...} + for (const line of debug) { + expect(line).toMatch(/^\[debug \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\]/); + } + }); + + it('--verbose wires onTransition to stderr', async () => { + const stderrLines: string[] = []; + // The dry-run fetch doesn't produce retries, but we verify the option is + // accepted and no error is thrown. + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, verbose: true, dryRun: true }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: line => stderrLines.push(line), + }, + ); + await client.get('/projects'); + // No transitions fire on a clean request; just confirm no throw. + expect(stderrLines).toContain(DRY_RUN_BANNER); + }); + + it('uses the fake API key constant (never real)', () => { + expect(DRY_RUN_API_KEY).toBe('sk-user-DRY-RUN'); + }); +}); + +// M3.3 piece-4 — getRunFailure dry-run path +describe('makeHttpClient — getRunFailure dry-run path (M3.3 piece-4)', () => { + afterEach(() => { + resetDryRunBannerForTesting(); + }); + + it('GET /runs/{runId}/failure returns a FailureContext-shaped body in dry-run', async () => { + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: true }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: () => {}, + }, + ); + const result = await client.get<{ + snapshotId: string; + testId: string; + result: { snapshotId: string; runIdIfAvailable: string }; + }>('/runs/run_abc/failure'); + expect(result.snapshotId).toBeTruthy(); + expect(result.testId).toBeTruthy(); + expect(result.result.snapshotId).toBe(result.snapshotId); + // Run-scoped bundle must have runIdIfAvailable set + expect(result.result.runIdIfAvailable).toBe('run_abc'); + }); + + it('GET /runs/{runId} returns a run-status-shaped body in dry-run', async () => { + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: true }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: () => {}, + }, + ); + const result = await client.get<{ + runId: string; + testId: string; + status: string; + }>('/runs/run_abc'); + expect(result.runId).toBeTruthy(); + expect(result.testId).toBeTruthy(); + expect(result.status).toBeTruthy(); + }); +}); + +// --------------------------------------------------------------------------- +// resolveRequestTimeoutMs — flag / env / default precedence +// --------------------------------------------------------------------------- + +describe('resolveRequestTimeoutMs', () => { + it('returns the default when no flag or env var is set', () => { + expect(resolveRequestTimeoutMs({}, {})).toBe(REQUEST_TIMEOUT_DEFAULT_MS); + }); + + it('flag (requestTimeoutMs) takes precedence over env var', () => { + expect( + resolveRequestTimeoutMs( + { requestTimeoutMs: 30_000 }, // 30s + { TESTSPRITE_REQUEST_TIMEOUT_MS: '60000' }, // 60s + ), + ).toBe(30_000); + }); + + it('env var is used when flag is absent', () => { + expect(resolveRequestTimeoutMs({}, { TESTSPRITE_REQUEST_TIMEOUT_MS: '45000' })).toBe(45_000); + }); + + it('clamps values below the minimum to REQUEST_TIMEOUT_MIN_MS', () => { + expect( + resolveRequestTimeoutMs({ requestTimeoutMs: 100 }, {}), // 100ms < 1s min + ).toBe(REQUEST_TIMEOUT_MIN_MS); + }); + + it('clamps values above the maximum to REQUEST_TIMEOUT_MAX_MS', () => { + expect(resolveRequestTimeoutMs({ requestTimeoutMs: 999_999_999 }, {})).toBe( + REQUEST_TIMEOUT_MAX_MS, + ); + }); + + it('ignores a non-numeric TESTSPRITE_REQUEST_TIMEOUT_MS env var', () => { + expect(resolveRequestTimeoutMs({}, { TESTSPRITE_REQUEST_TIMEOUT_MS: 'not-a-number' })).toBe( + REQUEST_TIMEOUT_DEFAULT_MS, + ); + }); + + it('ignores a zero or negative TESTSPRITE_REQUEST_TIMEOUT_MS env var', () => { + expect(resolveRequestTimeoutMs({}, { TESTSPRITE_REQUEST_TIMEOUT_MS: '0' })).toBe( + REQUEST_TIMEOUT_DEFAULT_MS, + ); + expect(resolveRequestTimeoutMs({}, { TESTSPRITE_REQUEST_TIMEOUT_MS: '-100' })).toBe( + REQUEST_TIMEOUT_DEFAULT_MS, + ); + }); + + it('accepts a valid env var within range', () => { + expect(resolveRequestTimeoutMs({}, { TESTSPRITE_REQUEST_TIMEOUT_MS: '5000' })).toBe(5_000); + }); +}); + +// --------------------------------------------------------------------------- +// makeHttpClient — requestTimeoutMs propagation +// --------------------------------------------------------------------------- + +describe('makeHttpClient — requestTimeoutMs propagation', () => { + afterEach(() => { + resetDryRunBannerForTesting(); + }); + + it('passes requestTimeoutMs from flag to the HttpClient (dry-run path)', () => { + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: true, requestTimeoutMs: 30_000 }, + { env: {} as NodeJS.ProcessEnv, credentialsPath: '/tmp/no-such-file.ini', stderr: () => {} }, + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((client as any).requestTimeoutMs).toBe(30_000); + }); + + it('resolves requestTimeoutMs from env var when flag is not set (dry-run path)', () => { + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: true }, + { + env: { TESTSPRITE_REQUEST_TIMEOUT_MS: '10000' } as NodeJS.ProcessEnv, + credentialsPath: '/tmp/no-such-file.ini', + stderr: () => {}, + }, + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((client as any).requestTimeoutMs).toBe(10_000); + }); + + it('falls back to REQUEST_TIMEOUT_DEFAULT_MS when neither flag nor env is set', () => { + const client = makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: true }, + { env: {} as NodeJS.ProcessEnv, credentialsPath: '/tmp/no-such-file.ini', stderr: () => {} }, + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((client as any).requestTimeoutMs).toBe(REQUEST_TIMEOUT_DEFAULT_MS); + }); +}); + +describe('makeHttpClient — real path (regression)', () => { + afterEach(() => { + resetDryRunBannerForTesting(); + }); + + it('throws AUTH_REQUIRED when no credentials are configured and not dry-run', () => { + expect(() => + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + }, + ), + ).toThrow(/authentication is required/i); + }); + + it('does not emit the dry-run banner when dryRun is false', () => { + const stderrLines: string[] = []; + expect(() => + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + stderr: line => stderrLines.push(line), + }, + ), + ).toThrow(); + expect(stderrLines).not.toContain(DRY_RUN_BANNER); + }); +}); diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts new file mode 100644 index 0000000..fad2fab --- /dev/null +++ b/src/lib/client-factory.ts @@ -0,0 +1,183 @@ +/** + * Shared `HttpClient` factory used by every M2 command. Two reasons for + * the extraction: + * + * 1. Three command files (`auth`, `project`, `test`) had byte-equivalent + * `makeClient` helpers — drift between them would silently change + * behavior depending on which command you called. + * 2. P6's `--dry-run` branch needs to bypass `loadConfig` (so it works + * with no `~/.testsprite/credentials`) and substitute a canned-fetch + * impl. Doing that once here is safer than three near-copies. + * + * The factory is intentionally narrow: it only takes the HTTP-relevant + * deps. Per-command deps (auth's `prompt`, test's `rawStdout`) stay + * with the command. + */ +import { loadConfig } from './config.js'; +import { defaultCredentialsPath } from './credentials.js'; +import { ApiError } from './errors.js'; +import { facadeBaseUrl } from './facade.js'; +import type { DebugEvent, FetchImpl } from './http.js'; +import { + HttpClient, + REQUEST_TIMEOUT_DEFAULT_MS, + REQUEST_TIMEOUT_MAX_MS, + REQUEST_TIMEOUT_MIN_MS, +} from './http.js'; +import type { OutputMode } from './output.js'; +import { createDryRunFetch } from './dry-run/fetch.js'; + +export interface CommonOptions { + profile: string; + output: OutputMode; + endpointUrl?: string; + debug: boolean; + /** + * When true: emit human-readable transition messages (HTTP retry, rate-limit, + * polling-mode switch) to stderr without the full debug JSON firehose. + * Sits between the default (silent retries) and `--debug` (full trace). + */ + verbose?: boolean; + /** + * When true: skip credential read, skip the network, return canned + * samples per `src/lib/dry-run/samples.ts`. The CLI binary still + * runs all argument validation, output formatting, and exit-code + * mapping, so dry-run output is byte-identical to a real success + * response (modulo the data values being canned). + * + * Optional so legacy call sites (P1–P5 tests) don't need to pass + * `false` everywhere — undefined is treated as `false`. + */ + dryRun?: boolean; + /** + * Per-request wall-clock timeout in milliseconds. Applied to every + * outgoing fetch call. Defaults to {@link REQUEST_TIMEOUT_DEFAULT_MS} + * (120 000 ms). Set via `--request-timeout ` flag (seconds) or + * `TESTSPRITE_REQUEST_TIMEOUT_MS` env var (milliseconds). + * + * Precedence: `--request-timeout` flag > `TESTSPRITE_REQUEST_TIMEOUT_MS` + * env var > default (120s). + * + * Clamped to [{@link REQUEST_TIMEOUT_MIN_MS}, {@link REQUEST_TIMEOUT_MAX_MS}]. + */ + requestTimeoutMs?: number; +} + +export interface ClientFactoryDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stderr?: (line: string) => void; +} + +/** + * The fake API key used in dry-run. Never sent — the dry-run fetch + * impl ignores headers and returns a canned sample. Documented in the + * runbook so secret-scanners don't flag it as a leak. + */ +export const DRY_RUN_API_KEY = 'sk-user-DRY-RUN'; + +const DRY_RUN_DEFAULT_ENDPOINT = 'https://api.testsprite.com'; + +/** Stable banner text. Snapshot tests assert on this string. */ +export const DRY_RUN_BANNER = '[dry-run] sample response — not from the server'; + +/** + * Emit the dry-run banner to stderr. Idempotent per process — the + * banner prints only on the first call so scripts that run several + * commands sequentially under one shell don't drown in repeats. The + * gate is reset by {@link resetDryRunBannerForTesting} for unit tests. + */ +let bannerEmitted = false; + +export function emitDryRunBanner(stderr: (line: string) => void): void { + if (bannerEmitted) return; + bannerEmitted = true; + stderr(DRY_RUN_BANNER); +} + +export function resetDryRunBannerForTesting(): void { + bannerEmitted = false; +} + +/** + * Resolve per-request timeout in milliseconds. + * + * Precedence: `--request-timeout` flag (already converted to ms by the caller) + * > `TESTSPRITE_REQUEST_TIMEOUT_MS` env var > default (120s). + * + * The flag is supplied in seconds (consistent with `--timeout`); the caller + * multiplies by 1000 before storing in `opts.requestTimeoutMs`. The env var + * is in milliseconds to give scripts sub-second granularity and match the + * typical convention for `*_MS` env vars. + * + * Values are clamped to [REQUEST_TIMEOUT_MIN_MS, REQUEST_TIMEOUT_MAX_MS]; + * values below the minimum are raised to 1s, values above the maximum are + * lowered to 10m — both silently (no exit 5), since this is a tuning knob, + * not a safety-critical gate. + */ +export function resolveRequestTimeoutMs( + opts: Pick, + env: NodeJS.ProcessEnv, +): number { + // Flag (already in ms) takes precedence. + if (opts.requestTimeoutMs !== undefined) { + return clampRequestTimeout(opts.requestTimeoutMs); + } + // Env var (milliseconds). + const envRaw = env.TESTSPRITE_REQUEST_TIMEOUT_MS; + if (envRaw !== undefined) { + const parsed = Number(envRaw); + if (Number.isFinite(parsed) && parsed > 0) { + return clampRequestTimeout(parsed); + } + } + return REQUEST_TIMEOUT_DEFAULT_MS; +} + +function clampRequestTimeout(ms: number): number { + return Math.min(Math.max(Math.round(ms), REQUEST_TIMEOUT_MIN_MS), REQUEST_TIMEOUT_MAX_MS); +} + +export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {}): HttpClient { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const env = deps.env ?? process.env; + const requestTimeoutMs = resolveRequestTimeoutMs(opts, env); + + if (opts.dryRun) { + emitDryRunBanner(stderr); + return new HttpClient({ + baseUrl: facadeBaseUrl(opts.endpointUrl ?? DRY_RUN_DEFAULT_ENDPOINT), + apiKey: DRY_RUN_API_KEY, + fetchImpl: deps.fetchImpl ?? createDryRunFetch(), + onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDryRunDebug(event)) : undefined, + onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, + requestTimeoutMs, + }); + } + + const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath, + }); + if (!config.apiKey) throw ApiError.authRequired(); + return new HttpClient({ + baseUrl: facadeBaseUrl(config.apiUrl), + apiKey: config.apiKey, + fetchImpl: deps.fetchImpl, + onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDebug(event)) : undefined, + onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, + requestTimeoutMs, + }); +} + +function formatDebug(event: DebugEvent): string { + return `[debug ${new Date().toISOString()}] ${JSON.stringify(event)}`; +} + +function formatDryRunDebug(event: DebugEvent): string { + return `[debug ${new Date().toISOString()}] ${JSON.stringify({ ...event, mode: 'dry-run' })}`; +} diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts new file mode 100644 index 0000000..463ddcc --- /dev/null +++ b/src/lib/config.test.ts @@ -0,0 +1,90 @@ +import { mkdtempSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { defaultConfigPath, loadConfig } from './config.js'; +import { writeProfile } from './credentials.js'; + +let tmpRoot: string; +let credentialsPath: string; + +beforeEach(() => { + tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-config-')); + credentialsPath = join(tmpRoot, 'credentials'); +}); + +describe('loadConfig', () => { + it('returns built-in defaults when nothing is provided', () => { + const config = loadConfig({ env: {}, credentialsPath }); + expect(config.apiUrl).toBe('https://api.testsprite.com'); + expect(config.apiKey).toBeUndefined(); + expect(config.profile).toBe('default'); + }); + + it('honors TESTSPRITE_API_URL over the file', () => { + writeProfile('default', { apiUrl: 'https://from-file.example.com' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_URL: 'https://from-env.example.com' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://from-env.example.com'); + }); + + it('honors TESTSPRITE_API_KEY over the file', () => { + writeProfile('default', { apiKey: 'sk-from-file' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_KEY: 'sk-from-env' }, + credentialsPath, + }); + expect(config.apiKey).toBe('sk-from-env'); + }); + + it('honors TESTSPRITE_PROFILE', () => { + expect(loadConfig({ env: { TESTSPRITE_PROFILE: 'staging' }, credentialsPath }).profile).toBe( + 'staging', + ); + }); + + it('option.profile overrides env var', () => { + expect( + loadConfig({ + profile: 'option-profile', + env: { TESTSPRITE_PROFILE: 'env-profile' }, + credentialsPath, + }).profile, + ).toBe('option-profile'); + }); + + it('option.endpointUrl overrides everything', () => { + writeProfile('default', { apiUrl: 'https://file' }, { path: credentialsPath }); + const config = loadConfig({ + endpointUrl: 'https://flag', + env: { TESTSPRITE_API_URL: 'https://env' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://flag'); + }); + + it('falls back to credentials file when env is unset', () => { + writeProfile( + 'default', + { apiKey: 'sk-file', apiUrl: 'https://from-file.example.com' }, + { path: credentialsPath }, + ); + const config = loadConfig({ env: {}, credentialsPath }); + expect(config.apiKey).toBe('sk-file'); + expect(config.apiUrl).toBe('https://from-file.example.com'); + }); + + it('reads the requested profile, not just default', () => { + writeProfile('dev', { apiKey: 'sk-dev' }, { path: credentialsPath }); + const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); + expect(config.apiKey).toBe('sk-dev'); + }); +}); + +describe('defaultConfigPath', () => { + it('points to ~/.testsprite/config', () => { + expect(defaultConfigPath()).toBe(join(homedir(), '.testsprite', 'config')); + }); +}); diff --git a/src/lib/config.ts b/src/lib/config.ts new file mode 100644 index 0000000..363c394 --- /dev/null +++ b/src/lib/config.ts @@ -0,0 +1,46 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { DEFAULT_PROFILE, defaultCredentialsPath, readProfile } from './credentials.js'; + +export interface Config { + apiUrl: string; + apiKey?: string; + profile: string; +} + +export interface LoadConfigOptions { + profile?: string; + endpointUrl?: string; + env?: NodeJS.ProcessEnv; + credentialsPath?: string; +} + +const DEFAULT_API_URL = 'https://api.testsprite.com'; + +export function defaultConfigPath(): string { + return join(homedir(), '.testsprite', 'config'); +} + +/** + * Resolves the active profile name and its (apiUrl, apiKey) pair. + * + * Resolution order, highest precedence first: + * profile name: options.profile > env.TESTSPRITE_PROFILE > "default" + * apiKey: env.TESTSPRITE_API_KEY > credentials file profile entry + * apiUrl: options.endpointUrl > env.TESTSPRITE_API_URL > credentials file > built-in default + * + * Env wins over the credentials file so CI / scripted callers can run without touching + * the user's ~/.testsprite/credentials. + */ +export function loadConfig(options: LoadConfigOptions = {}): Config { + const env = options.env ?? process.env; + const profile = options.profile ?? env.TESTSPRITE_PROFILE ?? DEFAULT_PROFILE; + const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); + const fileEntry = readProfile(profile, { path: credentialsPath }); + + return { + apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, + profile, + }; +} diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts new file mode 100644 index 0000000..8be03af --- /dev/null +++ b/src/lib/credentials.test.ts @@ -0,0 +1,170 @@ +import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + DEFAULT_PROFILE, + defaultCredentialsPath, + deleteProfile, + ensureRestrictiveMode, + parseCredentials, + readCredentialsFile, + readProfile, + serializeCredentials, + writeProfile, +} from './credentials.js'; + +let tmpRoot: string; +let credentialsPath: string; + +beforeEach(() => { + tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-creds-')); + credentialsPath = join(tmpRoot, 'credentials'); +}); + +afterEach(() => { + // mkdtempSync directory is small and short-lived; OS cleans it up. + // Tests intentionally do not rely on cleanup ordering. +}); + +describe('parseCredentials', () => { + it('returns empty for empty input', () => { + expect(parseCredentials('')).toEqual({}); + }); + + it('parses a single profile', () => { + const file = parseCredentials(`[default]\napi_key = sk-test\napi_url = https://example.com\n`); + expect(file).toEqual({ + default: { apiKey: 'sk-test', apiUrl: 'https://example.com' }, + }); + }); + + it('parses multiple profiles and ignores comments and blanks', () => { + const content = ` +# top-level comment +[default] +api_key = sk-default + +; semicolon comment +[dev] +api_key = sk-dev +api_url = https://api.example.com:8443 +`; + expect(parseCredentials(content)).toEqual({ + default: { apiKey: 'sk-default' }, + dev: { apiKey: 'sk-dev', apiUrl: 'https://api.example.com:8443' }, + }); + }); + + it('ignores unknown keys and key-value lines outside any section', () => { + const content = ` +api_key = orphan +[default] +api_key = sk-real +unknown_key = ignored +`; + expect(parseCredentials(content)).toEqual({ default: { apiKey: 'sk-real' } }); + }); +}); + +describe('serializeCredentials', () => { + it('round-trips with default profile first', () => { + const file = { + zeta: { apiKey: 'sk-z' }, + default: { apiKey: 'sk-d', apiUrl: 'https://example.com' }, + alpha: { apiKey: 'sk-a' }, + }; + const text = serializeCredentials(file); + expect(text.startsWith('[default]')).toBe(true); + expect(text).toContain('[alpha]'); + expect(text).toContain('[zeta]'); + expect(parseCredentials(text)).toEqual(file); + }); + + it('omits empty fields', () => { + const text = serializeCredentials({ default: { apiKey: 'sk', apiUrl: '' } }); + expect(text).toContain('api_key = sk'); + expect(text).not.toContain('api_url'); + }); +}); + +describe('readCredentialsFile / readProfile', () => { + it('returns empty when file is missing', () => { + expect(readCredentialsFile({ path: credentialsPath })).toEqual({}); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + + it('reads an existing file', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, '[default]\napi_key = sk-x\n'); + expect(readProfile('default', { path: credentialsPath })).toEqual({ apiKey: 'sk-x' }); + }); +}); + +describe('writeProfile', () => { + it('creates the file with mode 0600 and writes the profile', () => { + writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); + expect(existsSync(credentialsPath)).toBe(true); + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); + }); + + it('preserves other profiles when updating one', () => { + writeProfile('default', { apiKey: 'sk-d' }, { path: credentialsPath }); + writeProfile('dev', { apiKey: 'sk-dev', apiUrl: 'https://dev' }, { path: credentialsPath }); + writeProfile('default', { apiUrl: 'https://prod' }, { path: credentialsPath }); + + const file = readCredentialsFile({ path: credentialsPath }); + expect(file.default).toEqual({ apiKey: 'sk-d', apiUrl: 'https://prod' }); + expect(file.dev).toEqual({ apiKey: 'sk-dev', apiUrl: 'https://dev' }); + }); + + it('does not leak the api key into the on-disk file format aside from the value itself', () => { + writeProfile('default', { apiKey: 'sk-secret-12345' }, { path: credentialsPath }); + const onDisk = readFileSync(credentialsPath, 'utf-8'); + expect(onDisk).toContain('api_key = sk-secret-12345'); + expect(onDisk.split('\n').filter(line => line.includes('sk-secret-12345'))).toHaveLength(1); + }); +}); + +describe('deleteProfile', () => { + it('returns false when the profile is missing', () => { + expect(deleteProfile('nope', { path: credentialsPath })).toBe(false); + }); + + it('removes the named profile and leaves others intact', () => { + writeProfile('default', { apiKey: 'sk-d' }, { path: credentialsPath }); + writeProfile('dev', { apiKey: 'sk-dev' }, { path: credentialsPath }); + expect(deleteProfile('dev', { path: credentialsPath })).toBe(true); + expect(readProfile('dev', { path: credentialsPath })).toBeUndefined(); + expect(readProfile('default', { path: credentialsPath })).toEqual({ apiKey: 'sk-d' }); + }); + + it('leaves an empty file when the last profile is removed', () => { + writeProfile('default', { apiKey: 'sk-d' }, { path: credentialsPath }); + expect(deleteProfile('default', { path: credentialsPath })).toBe(true); + expect(readCredentialsFile({ path: credentialsPath })).toEqual({}); + expect(existsSync(credentialsPath)).toBe(true); + }); +}); + +describe('ensureRestrictiveMode', () => { + it('is a no-op when the file is missing', () => { + expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); + }); + + it('downgrades over-permissive modes', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data', { mode: 0o644 }); + ensureRestrictiveMode(credentialsPath); + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + }); +}); + +describe('defaultCredentialsPath', () => { + it('points at ~/.testsprite/credentials', () => { + expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); + }); +}); diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts new file mode 100644 index 0000000..c59de98 --- /dev/null +++ b/src/lib/credentials.ts @@ -0,0 +1,146 @@ +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + renameSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; + +export const DEFAULT_PROFILE = 'default'; + +export function defaultCredentialsPath(): string { + return join(homedir(), '.testsprite', 'credentials'); +} + +export interface ProfileEntry { + apiKey?: string; + apiUrl?: string; +} + +export type CredentialsFile = Record; + +export interface CredentialsOptions { + path?: string; +} + +const FILE_KEY_TO_FIELD: Record = { + api_key: 'apiKey', + api_url: 'apiUrl', +}; + +const FIELD_TO_FILE_KEY: Record = { + apiKey: 'api_key', + apiUrl: 'api_url', +}; + +export function parseCredentials(content: string): CredentialsFile { + const result: CredentialsFile = {}; + let currentEntry: ProfileEntry | null = null; + for (const rawLine of content.split('\n')) { + const line = rawLine.trim(); + if (line === '' || line.startsWith('#') || line.startsWith(';')) continue; + const sectionMatch = /^\[([^\]]+)\]$/.exec(line); + if (sectionMatch) { + const sectionName = sectionMatch[1]!.trim(); + const existing = result[sectionName]; + if (existing) { + currentEntry = existing; + } else { + const newEntry: ProfileEntry = {}; + result[sectionName] = newEntry; + currentEntry = newEntry; + } + continue; + } + if (currentEntry === null) continue; + const eqIndex = line.indexOf('='); + if (eqIndex < 0) continue; + const rawKey = line.slice(0, eqIndex).trim(); + const rawValue = line.slice(eqIndex + 1).trim(); + const field = FILE_KEY_TO_FIELD[rawKey]; + if (field) currentEntry[field] = rawValue; + } + return result; +} + +export function serializeCredentials(file: CredentialsFile): string { + const orderedSections = Object.keys(file).sort((a, b) => { + if (a === DEFAULT_PROFILE) return -1; + if (b === DEFAULT_PROFILE) return 1; + return a.localeCompare(b); + }); + const lines: string[] = []; + for (const section of orderedSections) { + const entry = file[section]; + if (!entry) continue; + lines.push(`[${section}]`); + const fields = Object.keys(entry).sort() as Array; + for (const field of fields) { + const value = entry[field]; + if (value === undefined || value === '') continue; + lines.push(`${FIELD_TO_FILE_KEY[field]} = ${value}`); + } + lines.push(''); + } + return lines.join('\n').trimEnd() + '\n'; +} + +export function readCredentialsFile(options: CredentialsOptions = {}): CredentialsFile { + const path = resolvePath(options); + if (!existsSync(path)) return {}; + return parseCredentials(readFileSync(path, 'utf-8')); +} + +export function readProfile( + profile: string, + options: CredentialsOptions = {}, +): ProfileEntry | undefined { + const file = readCredentialsFile(options); + return file[profile]; +} + +export function writeProfile( + profile: string, + entry: ProfileEntry, + options: CredentialsOptions = {}, +): void { + const path = resolvePath(options); + const file = readCredentialsFile(options); + file[profile] = { ...file[profile], ...entry }; + writeCredentialsAtomic(path, file); +} + +export function deleteProfile(profile: string, options: CredentialsOptions = {}): boolean { + const path = resolvePath(options); + const file = readCredentialsFile(options); + if (!(profile in file)) return false; + delete file[profile]; + if (Object.keys(file).length === 0) { + writeCredentialsAtomic(path, {}); + } else { + writeCredentialsAtomic(path, file); + } + return true; +} + +export function ensureRestrictiveMode(path: string): void { + if (!existsSync(path)) return; + const overpermissive = (statSync(path).mode & 0o077) !== 0; + if (overpermissive) chmodSync(path, 0o600); +} + +function resolvePath(options: CredentialsOptions): string { + return options.path ?? defaultCredentialsPath(); +} + +function writeCredentialsAtomic(path: string, file: CredentialsFile): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync(tmp, serializeCredentials(file), { mode: 0o600, encoding: 'utf8' }); + renameSync(tmp, path); + ensureRestrictiveMode(path); +} diff --git a/src/lib/dry-run/fetch.test.ts b/src/lib/dry-run/fetch.test.ts new file mode 100644 index 0000000..c61496f --- /dev/null +++ b/src/lib/dry-run/fetch.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { createDryRunFetch } from './fetch.js'; + +describe('createDryRunFetch', () => { + const fetch = createDryRunFetch(); + + it('returns 200 + JSON for a known endpoint', async () => { + const res = await fetch('https://api.testsprite.com/api/cli/v1/me'); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/json'); + expect(res.headers.get('x-request-id')).toBe('req_dry-run'); + const body = (await res.json()) as { userId: string }; + expect(body.userId).toBeTruthy(); + }); + + it('returns 500 INTERNAL envelope for an unknown endpoint', async () => { + const res = await fetch('https://api.testsprite.com/api/cli/v1/nope'); + expect(res.status).toBe(500); + const body = (await res.json()) as { error: { code: string; nextAction: string } }; + expect(body.error.code).toBe('INTERNAL'); + expect(body.error.nextAction).toContain('samples.ts'); + }); + + it('accepts a Request object as input', async () => { + const req = new Request('https://api.testsprite.com/api/cli/v1/projects'); + const res = await fetch(req); + expect(res.status).toBe(200); + }); + + it('accepts a URL object as input', async () => { + const url = new URL('https://api.testsprite.com/api/cli/v1/projects'); + const res = await fetch(url); + expect(res.status).toBe(200); + }); + + it('routes by method (POST /projects returns 200 because createProject sample exists)', async () => { + const res = await fetch('https://api.testsprite.com/api/cli/v1/projects', { method: 'POST' }); + expect(res.status).toBe(200); + }); + + it('routes by method (DELETE /projects returns 500 because no sample exists)', async () => { + const res = await fetch('https://api.testsprite.com/api/cli/v1/projects', { + method: 'DELETE', + }); + expect(res.status).toBe(500); + }); + + it('ignores query strings when matching', async () => { + const res = await fetch('https://api.testsprite.com/api/cli/v1/projects?pageSize=2'); + expect(res.status).toBe(200); + }); +}); diff --git a/src/lib/dry-run/fetch.ts b/src/lib/dry-run/fetch.ts new file mode 100644 index 0000000..c90b0f4 --- /dev/null +++ b/src/lib/dry-run/fetch.ts @@ -0,0 +1,70 @@ +/** + * Dry-run fetch implementation. Drop-in `FetchImpl` that the + * `client-factory` swaps in when `--dry-run` is set; the existing + * `HttpClient` keeps its single retry / debug / error path and never + * has to know dry-run exists. + * + * Behavior: + * - Looks up a canned sample by (method, path). Path lookup ignores + * the host and query string (auto-pagination's `cursor` query is a + * no-op in M2 dry-run; see {@link findSample}). + * - Returns 200 with the sample body and a stable `x-request-id` so + * the caller's debug output is deterministic. + * - Missing sample → 500 INTERNAL envelope. Loud-and-broken beats a + * silent dry-run that returns empty data; the test suite asserts + * every M2 endpoint resolves. + */ +import type { FetchImpl } from '../http.js'; +import { findSample, SAMPLE_DRY_RUN_REQUEST_ID } from './samples.js'; + +const JSON_HEADERS = { + 'content-type': 'application/json', + 'x-request-id': SAMPLE_DRY_RUN_REQUEST_ID, +}; + +export function createDryRunFetch(): FetchImpl { + return async (input, init) => { + const url = resolveUrl(input); + const method = (init?.method ?? 'GET').toUpperCase(); + + // Parse the request body so input-derived samples (updateTest, + // putPlanSteps, createTestBatch) can echo the user's actual fields. + let requestBody: unknown; + if (init?.body != null) { + try { + // Treat the body as a string if possible; fall back to String() + // for any other serialisable type (e.g. Uint8Array in Node 22 + // whose toString() gives a comma-joined byte list, but in + // practice fetch bodies are always JSON strings here). + const raw = typeof init.body === 'string' ? init.body : String(init.body); + requestBody = JSON.parse(raw); + } catch { + // Unparseable body — leave requestBody undefined; samples fall + // back to their no-input defaults. + } + } + + const sample = findSample(method, url, requestBody); + if (sample === undefined) { + const body = JSON.stringify({ + error: { + code: 'INTERNAL', + message: `No dry-run sample registered for ${method} ${url}`, + nextAction: 'Add a sample to `src/lib/dry-run/samples.ts` and re-run.', + requestId: SAMPLE_DRY_RUN_REQUEST_ID, + details: { method, url }, + }, + }); + return new Response(body, { status: 500, headers: JSON_HEADERS }); + } + return new Response(JSON.stringify(sample.body()), { status: 200, headers: JSON_HEADERS }); + }; +} + +function resolveUrl(input: Parameters[0]): string { + if (typeof input === 'string') return input; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const inp = input as any; + if (inp?.href !== undefined) return inp.href; // URL instance + return inp.url; // Request instance +} diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts new file mode 100644 index 0000000..aa2d135 --- /dev/null +++ b/src/lib/dry-run/samples.test.ts @@ -0,0 +1,630 @@ +import { describe, expect, it } from 'vitest'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; + +describe('findSample', () => { + it('resolves /me', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/me'); + expect(e?.operationId).toBe('whoami'); + expect((e?.body() as { userId: string }).userId).toBeTruthy(); + }); + + it('resolves /projects (list)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/projects'); + expect(e?.operationId).toBe('listProjects'); + const body = e?.body() as { items: unknown[]; nextToken: null }; + expect(Array.isArray(body.items)).toBe(true); + expect(body.nextToken).toBeNull(); + }); + + it('resolves /projects/{id}', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/projects/proj_anything'); + expect(e?.operationId).toBe('getProject'); + }); + + it('resolves /tests (list) — must not collide with /tests/{id}', () => { + const list = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests'); + expect(list?.operationId).toBe('listTests'); + const detail = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_anything'); + expect(detail?.operationId).toBe('getTest'); + }); + + it('resolves nested /tests/{id}/{code,steps,result,failure}', () => { + const code = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/t_x/code'); + expect(code?.operationId).toBe('getTestCode'); + const steps = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/t_x/steps'); + expect(steps?.operationId).toBe('listTestSteps'); + const result = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/t_x/result'); + expect(result?.operationId).toBe('getTestResult'); + const failure = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/t_x/failure'); + expect(failure?.operationId).toBe('getTestFailure'); + }); + + it('resolves /tests/{id}/failure/summary distinct from /failure (M2.1 piece 3)', () => { + // The summary endpoint must match BEFORE the bundle endpoint so a + // dry-run user practising `failure summary` doesn't get the bundle + // shape back. Pattern order is significant inside the entry list. + const summary = findSample( + 'GET', + 'https://api.testsprite.com/api/cli/v1/tests/t_x/failure/summary', + ); + expect(summary?.operationId).toBe('getTestFailureSummary'); + const bundle = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/t_x/failure'); + expect(bundle?.operationId).toBe('getTestFailure'); + }); + + it('ignores query string when matching', () => { + const a = findSample('GET', 'https://api.testsprite.com/api/cli/v1/projects?pageSize=2'); + const b = findSample('GET', 'https://api.testsprite.com/api/cli/v1/projects'); + expect(a?.operationId).toBe(b?.operationId); + }); + + it('returns undefined for unknown paths', () => { + expect(findSample('GET', 'https://api.testsprite.com/api/cli/v1/nope')).toBeUndefined(); + }); + + it('returns undefined for unsupported methods (no matching sample)', () => { + // DELETE /projects is not registered — should be undefined. + const del = findSample('DELETE', 'https://api.testsprite.com/api/cli/v1/projects'); + expect(del).toBeUndefined(); + }); + + it('handles paths without the facade prefix (defensive)', () => { + // Some test setups may strip the facade prefix before calling fetch. + const e = findSample('GET', '/projects'); + expect(e?.operationId).toBe('listProjects'); + }); + + it('every entry has a body matching the §6.x non-null required fields', () => { + for (const e of DRY_RUN_SAMPLE_ENTRIES) { + const body = e.body() as Record; + switch (e.operationId) { + case 'whoami': + expect(body).toMatchObject({ + userId: expect.any(String), + keyId: expect.any(String), + scopes: expect.any(Array), + env: expect.any(String), + }); + break; + case 'listProjects': + case 'listTests': + case 'listTestSteps': + expect(body).toMatchObject({ items: expect.any(Array), nextToken: null }); + break; + case 'getProject': + expect(body).toMatchObject({ id: expect.any(String), name: expect.any(String) }); + break; + case 'getTest': + // G1a — priority must be present (truthy string or null). + expect(body).toMatchObject({ id: expect.any(String), name: expect.any(String) }); + expect('priority' in body).toBe(true); + break; + case 'getTestCode': + expect(body).toMatchObject({ + testId: expect.any(String), + language: expect.any(String), + code: expect.any(String), + }); + break; + case 'getTestResult': + expect(body).toMatchObject({ + testId: expect.any(String), + status: expect.any(String), + snapshotId: expect.any(String), + summary: expect.any(Object), + targetUrlSource: 'run', + }); + break; + case 'getTestFailure': + expect(body).toMatchObject({ + snapshotId: expect.any(String), + testId: expect.any(String), + projectId: expect.any(String), + result: expect.any(Object), + steps: expect.any(Array), + code: expect.any(Object), + failure: expect.any(Object), + }); + break; + case 'getTestFailureSummary': + // §5.2 / M2.1 piece 3 — flat shape, no bundle metadata. + expect(body).toMatchObject({ + testId: expect.any(String), + status: expect.any(String), + snapshotId: expect.any(String), + }); + break; + case 'createTestFromCode': + // M3.2 piece-2 — `CreateTestResponse` echoed from a POST. + // Piece-5 reuses the same sample for `--plan-from`; the + // sampler doesn't body-inspect, so the same body covers both. + expect(body).toMatchObject({ + testId: expect.any(String), + type: expect.any(String), + codeVersion: expect.any(String), + createdAt: expect.any(String), + }); + break; + case 'putPlanSteps': + // M3.2 piece-6 — `PutPlanStepsResponse` echoed from a PUT. + // FE-only; BE tests get a 400 from the server. + expect(body).toMatchObject({ + testId: expect.any(String), + planStepsHash: expect.any(String), + stepCount: expect.any(Number), + updatedAt: expect.any(String), + }); + break; + case 'putTestCode': + // M3.2 piece-4 — `PutTestCodeResponse` echoed from a PUT. + // The bumped codeVersion drives the next call's If-Match. + expect(body).toMatchObject({ + testId: expect.any(String), + codeVersion: expect.any(String), + updatedAt: expect.any(String), + }); + break; + case 'updateTest': + // M3.2 piece-3 — `UpdateTestResponse` echoed from a PUT. + expect(body).toMatchObject({ + testId: expect.any(String), + updatedFields: expect.any(Array), + updatedAt: expect.any(String), + }); + break; + case 'deleteTest': + // M3.2 piece-3 — `DeleteTestResponse` echoed from a DELETE + // (permanent hard-delete; no restore window). + expect(body).toMatchObject({ + testId: expect.any(String), + deletedAt: expect.any(String), + }); + break; + case 'deleteBatch': + // delete-batch dispatches DELETE /tests/{testId} per test (no batch + // endpoint). The canned sample shows CliBulkDeleteSummary shape with + // a mix of deleted + skipped results. + expect(body).toMatchObject({ + results: expect.any(Array), + summary: expect.objectContaining({ + total: expect.any(Number), + deleted: expect.any(Number), + skipped: expect.any(Number), + failed: expect.any(Number), + }), + }); + break; + case 'createTestBatch': + // M3.2 piece-5 — batch create response (FE-only). Per-spec + // results preserve input order; mixed-status sample shows + // 2 created + 1 validation_error. + expect(body).toMatchObject({ + results: expect.any(Array), + summary: expect.objectContaining({ + total: expect.any(Number), + created: expect.any(Number), + failed: expect.any(Number), + }), + }); + break; + case 'triggerRun': + // M3.3 piece-3 — POST /tests/{testId}/runs → TriggerRunResponse. + expect(body).toMatchObject({ + runId: expect.any(String), + status: 'queued', + enqueuedAt: expect.any(String), + codeVersion: expect.any(String), + targetUrl: expect.any(String), + }); + break; + case 'getRun': + // M3.3 piece-3 — GET /runs/{runId} → RunResponse. + expect(body).toMatchObject({ + runId: expect.any(String), + testId: expect.any(String), + projectId: expect.any(String), + userId: expect.any(String), + status: expect.any(String), + stepSummary: expect.any(Object), + }); + break; + case 'getRunFailure': + // M3.3 piece-4 — run-scoped failure bundle (same FailureContext + // shape as `getTestFailure` but addressed by runId). + expect(body).toMatchObject({ + snapshotId: expect.any(String), + testId: expect.any(String), + result: expect.objectContaining({ snapshotId: expect.any(String) }), + }); + break; + case 'triggerRerun': + // M3.4 piece-3 — POST /tests/{testId}/runs/rerun → RerunResponse. + // G1c — closure is ALWAYS present (null for FE, object for BE). + expect(body).toMatchObject({ + runId: expect.any(String), + status: 'queued', + enqueuedAt: expect.any(String), + codeVersion: expect.any(String), + autoHeal: expect.any(Boolean), + }); + // closure must be explicitly present (not undefined); may be object or null. + expect('closure' in body).toBe(true); + break; + case 'triggerBatchRerun': + // M3.4 piece-3 — POST /tests/batch/rerun → BatchRerunResponse. + expect(body).toMatchObject({ + accepted: expect.any(Array), + deferred: expect.any(Array), + conflicts: expect.any(Array), + closure: expect.any(Object), + }); + break; + case 'triggerBatchRunFresh': + // M4 piece-2 — POST /tests/batch/run → BatchRunFreshResponse. + expect(body).toMatchObject({ + accepted: expect.any(Array), + skippedFrontend: expect.any(Array), + }); + // Each accepted entry must carry testId + enqueuedAt. + // runId may be undefined for draft tests but the dry-run sample provides it. + expect( + (body as { accepted: Array> }).accepted.length, + ).toBeGreaterThan(0); + break; + case 'listTestRuns': { + // M3.4 piece-5 — GET /tests/{testId}/runs → ListRunsResponse. + // G1b — run rows include targetUrl + targetUrlSource fields. + expect(body).toMatchObject({ + runs: expect.any(Array), + nextCursor: null, + meta: expect.any(Object), + }); + // At least one run row present for illustrative purposes. + const runs = (body as { runs: Array> }).runs; + expect(runs.length).toBeGreaterThan(0); + // All rows must carry the G1b fields (present, even if null). + for (const row of runs) { + expect('targetUrl' in row).toBe(true); + expect('targetUrlSource' in row).toBe(true); + } + // First row: a real URL with source='run'. + expect(runs[0]).toMatchObject({ targetUrl: expect.any(String), targetUrlSource: 'run' }); + // Second row: unresolved shape (null URL, source='unresolved'). + expect(runs[1]).toMatchObject({ targetUrl: null, targetUrlSource: 'unresolved' }); + break; + } + case 'createProject': + // P6 — POST /projects → CliProject shape. + expect(body).toMatchObject({ + id: expect.any(String), + type: expect.any(String), + name: expect.any(String), + createdFrom: expect.any(String), + createdAt: expect.any(String), + }); + break; + case 'updateProject': + // P7 — PATCH /projects/{id} → CliUpdateProjectResponse shape. + expect(body).toMatchObject({ + id: expect.any(String), + updatedFields: expect.any(Array), + updatedAt: expect.any(String), + }); + break; + default: + throw new Error(`Unexpected operationId in samples: ${e.operationId}`); + } + } + }); + + it('GET /tests/{testId}/runs resolves listTestRuns (not getTest)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc/runs'); + expect(e?.operationId).toBe('listTestRuns'); + const body = e?.body() as { + runs: Array>; + nextCursor: null; + meta: unknown; + }; + expect(Array.isArray(body.runs)).toBe(true); + expect(body.runs.length).toBeGreaterThan(0); + expect(body.nextCursor).toBeNull(); + // G1b — every row must carry targetUrl + targetUrlSource. + for (const row of body.runs) { + expect('targetUrl' in row).toBe(true); + expect('targetUrlSource' in row).toBe(true); + } + }); + + it('GET /tests/{testId}/runs: first row has isRerun:false, second has isRerun:true', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc/runs'); + const body = e?.body() as { + runs: Array<{ + runId: string; + isRerun: boolean; + failureKind: string | null; + targetUrl: string | null; + targetUrlSource: string | null; + }>; + }; + expect(body.runs[0]?.isRerun).toBe(false); + expect(body.runs[0]?.failureKind).toBeNull(); + // G1b — first row: real URL with source='run'. + expect(body.runs[0]?.targetUrlSource).toBe('run'); + expect(typeof body.runs[0]?.targetUrl).toBe('string'); + expect(body.runs[1]?.isRerun).toBe(true); + expect(body.runs[1]?.failureKind).toBe('assertion'); + // G1b — second row: unresolved shape. + expect(body.runs[1]?.targetUrl).toBeNull(); + expect(body.runs[1]?.targetUrlSource).toBe('unresolved'); + }); + + it('failure context maintains §6.7 atomicity invariant (snapshotId === result.snapshotId)', () => { + const failure = findSample('GET', '/tests/t_x/failure'); + const body = failure?.body() as { snapshotId: string; result: { snapshotId: string } }; + expect(body.result.snapshotId).toBe(body.snapshotId); + }); + + // updateTest method regression guard. + // Backend route is `@Put('/:testId')` in cli-tests.controller.ts:577; + // the CLI `runUpdate` calls `client.put()`; the dry-run sample is also + // registered as PUT. Prior audit incorrectly claimed PATCH was the wire + // verb based on CLAUDE.md's aspirational table — actual wire reality is + // PUT. This test pins the verb so a future "fix" doesn't silently break + // `test update` against the deployed backend. + it('PUT /tests/{id} resolves updateTest', () => { + const e = findSample('PUT', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); + expect(e?.operationId).toBe('updateTest'); + const body = e?.body() as { testId: string; updatedFields: string[]; updatedAt: string }; + expect(body.testId).toBeTruthy(); + expect(Array.isArray(body.updatedFields)).toBe(true); + expect(body.updatedAt).toBeTruthy(); + }); + + it('PATCH /tests/{id} has no sample (backend route is PUT)', () => { + const e = findSample('PATCH', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); + expect(e).toBeUndefined(); + }); + + // defect-2 fix: getRun sample must return the passed shape (first-match-wins + // in findSample). Prior to fix, a duplicate failed-shape entry appeared + // before the passed-shape entry; `test wait --dry-run` always resolved to + // status: "failed", giving agents the wrong happy-path canned response. + it('GET /runs/{runId} resolves to the passed-shape getRun (not the failed shape)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz'); + expect(e?.operationId).toBe('getRun'); + const body = e?.body() as { + status: string; + runId: string; + stepSummary: { failedCount: number }; + }; + expect(body.status).toBe('passed'); + expect(body.stepSummary.failedCount).toBe(0); + }); + + it('getTest sample carries priority field (G1a)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); + expect(e?.operationId).toBe('getTest'); + const body = e?.body() as Record; + expect('priority' in body).toBe(true); + // First test in the sample has priority 'p1'. + expect(body.priority).toBe('p1'); + }); + + it('listTests sample items carry priority field (G1a)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests'); + expect(e?.operationId).toBe('listTests'); + const body = e?.body() as { items: Array> }; + expect(body.items.length).toBeGreaterThan(0); + // Every item in the list must carry the priority field. + for (const item of body.items) { + expect('priority' in item).toBe(true); + } + // At least one item has a truthy priority to exercise the render branch. + const hasTruthy = body.items.some(item => item.priority != null); + expect(hasTruthy).toBe(true); + }); + + it('triggerRerun sample closure is explicitly present (G1c — always-present nullable)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/be_test/runs/rerun'); + const body = e?.body() as Record; + // closure must be present (not undefined); BE sample carries the object. + expect('closure' in body).toBe(true); + expect(body.closure).not.toBeUndefined(); + }); + + it('POST /tests/{testId}/runs/rerun resolves triggerRerun (not triggerRun)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/test_abc/runs/rerun'); + expect(e?.operationId).toBe('triggerRerun'); + const body = e?.body() as { runId: string; status: string; autoHeal: boolean }; + expect(body.runId).toBeTruthy(); + expect(body.status).toBe('queued'); + expect(typeof body.autoHeal).toBe('boolean'); + }); + + it('POST /tests/batch/rerun resolves triggerBatchRerun (not createTestBatch)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch/rerun'); + expect(e?.operationId).toBe('triggerBatchRerun'); + const body = e?.body() as { accepted: unknown[]; deferred: unknown[] }; + expect(Array.isArray(body.accepted)).toBe(true); + expect(Array.isArray(body.deferred)).toBe(true); + }); + + it('POST /tests/batch/run resolves triggerBatchRunFresh (not rerun or create)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch/run'); + expect(e?.operationId).toBe('triggerBatchRunFresh'); + const body = e?.body() as { accepted: unknown[]; skippedFrontend: unknown[] }; + expect(Array.isArray(body.accepted)).toBe(true); + expect(Array.isArray(body.skippedFrontend)).toBe(true); + expect(body.accepted.length).toBeGreaterThan(0); + }); + + it('triggerBatchRunFresh sample accepted entries carry testId, runId, enqueuedAt', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch/run'); + const body = e?.body() as { + accepted: Array<{ testId: string; runId: string; enqueuedAt: string }>; + }; + expect(body.accepted[0]).toMatchObject({ + testId: expect.any(String), + runId: expect.any(String), + enqueuedAt: expect.any(String), + }); + }); + + it('triggerRerun sample carries closure.members with per-member runIds (C2)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/be_test/runs/rerun'); + const body = e?.body() as { + closure?: { members: Array<{ testId: string; runId: string; role: string }> }; + }; + expect(body.closure?.members).toEqual( + expect.arrayContaining([ + expect.objectContaining({ runId: expect.any(String), role: expect.any(String) }), + ]), + ); + }); + + it('triggerBatchRerun sample carries deferred[] example (C1)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch/rerun'); + const body = e?.body() as { deferred: Array<{ testId: string; reason: string }> }; + expect(body.deferred.length).toBeGreaterThan(0); + expect(body.deferred[0]).toMatchObject({ + testId: expect.any(String), + reason: expect.any(String), + }); + }); + + it('DELETE /tests/batch resolves deleteBatch documentation sample', () => { + // delete-batch is inline-dry-run (no network call) but the sample is + // registered for shape documentation and build-guard purposes. + const e = findSample('DELETE', 'https://api.testsprite.com/api/cli/v1/tests/batch'); + expect(e?.operationId).toBe('deleteBatch'); + const body = e?.body() as { results: unknown[]; summary: Record }; + expect(Array.isArray(body.results)).toBe(true); + expect(body.summary.total).toBeGreaterThanOrEqual(1); + }); + + it('only one getRun entry exists in the registry (no duplicate)', () => { + // Guards against re-introducing the duplicate by ensuring exactly one + // sample is registered for GET /runs/{runId}. + const matches = DRY_RUN_SAMPLE_ENTRIES.filter( + e => e.method === 'GET' && e.operationId === 'getRun', + ); + expect(matches).toHaveLength(1); + }); + + // Input-derived sample tests (Fix #1 — dogfood 2026-05-15) + describe('input-derived samples echo user flags', () => { + it('updateTest: updatedFields reflects the actual request body keys', () => { + const e = findSample('PUT', 'https://api.testsprite.com/api/cli/v1/tests/test_abc', { + name: 'my-new-name', + }); + const body = e?.body() as { updatedFields: string[] }; + expect(body.updatedFields).toEqual(['name']); + }); + + it('updateTest: updatedFields reflects multiple fields', () => { + const e = findSample('PUT', 'https://api.testsprite.com/api/cli/v1/tests/test_abc', { + name: 'n', + description: 'd', + priority: 'high', + }); + const body = e?.body() as { updatedFields: string[] }; + expect(body.updatedFields).toEqual(['name', 'description', 'priority']); + }); + + it('updateTest: falls back to default ["name","description"] when no request body', () => { + const e = findSample('PUT', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); + const body = e?.body() as { updatedFields: string[] }; + expect(body.updatedFields).toEqual(['name', 'description']); + }); + + it('putPlanSteps: stepCount reflects actual planSteps array length', () => { + const planSteps = [ + { type: 'action', description: 'step 1' }, + { type: 'assertion', description: 'step 2' }, + ]; + const e = findSample( + 'PUT', + 'https://api.testsprite.com/api/cli/v1/tests/test_abc/plan-steps', + { planSteps }, + ); + const body = e?.body() as { stepCount: number }; + expect(body.stepCount).toBe(2); + }); + + it('putPlanSteps: falls back to stepCount 3 when no planSteps in body', () => { + const e = findSample( + 'PUT', + 'https://api.testsprite.com/api/cli/v1/tests/test_abc/plan-steps', + ); + const body = e?.body() as { stepCount: number }; + expect(body.stepCount).toBe(3); + }); + + it('createTestBatch: one result per supplied spec', () => { + const tests = [ + { type: 'frontend', name: 'A' }, + { type: 'frontend', name: 'B' }, + ]; + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch', { tests }); + const body = e?.body() as { + results: Array<{ specIndex: number; status: string }>; + summary: { total: number; created: number }; + }; + expect(body.results).toHaveLength(2); + expect(body.results[0]?.specIndex).toBe(0); + expect(body.results[1]?.specIndex).toBe(1); + expect(body.summary.total).toBe(2); + expect(body.summary.created).toBe(2); + }); + + it('createTestBatch: falls back to 3-entry sample when no tests in body', () => { + // Preserves the educational mixed-status sample for learners who run + // dry-run without a body (e.g. direct fetch calls in tests). + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch'); + const body = e?.body() as { summary: { total: number; created: number; failed: number } }; + expect(body.summary.total).toBe(3); + expect(body.summary.created).toBe(2); + expect(body.summary.failed).toBe(1); + }); + + it('createTestBatch: backend specs are echoed as validation_error (mirroring runCreateBatch)', () => { + // codex-review round-2 P2 (2026-05-28): the round-1 fix added a + // type === 'backend' branch but had no regression guard. This test + // covers the rejection path so a future refactor can't silently + // start marking BE specs as `created` again. + const tests = [ + { type: 'frontend', name: 'fe-spec' }, + { type: 'backend', name: 'be-spec' }, + { type: 'frontend', name: 'another-fe' }, + ]; + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/tests/batch', { tests }); + const body = e?.body() as { + results: Array<{ + specIndex: number; + status: string; + testId: string | null; + error?: { code: string; message: string; field: string }; + }>; + summary: { total: number; created: number; failed: number }; + }; + expect(body.results).toHaveLength(3); + // Position 0 (FE) → created + expect(body.results[0]?.status).toBe('created'); + expect(body.results[0]?.testId).toBe('test_dryrun_batch_0'); + // Position 1 (BE) → validation_error, testId null, error wired with field=type + expect(body.results[1]?.status).toBe('validation_error'); + expect(body.results[1]?.testId).toBeNull(); + expect(body.results[1]?.error?.code).toBe('VALIDATION_ERROR'); + expect(body.results[1]?.error?.field).toBe('type'); + // The error message should reference the BE workaround so the user + // sees the production guidance even in dry-run. + expect(body.results[1]?.error?.message).toMatch(/test create --type backend --code-file/); + // Position 2 (FE) → created (BE rejection does not abort siblings) + expect(body.results[2]?.status).toBe('created'); + // Summary counts the FE specs as created (2) and the BE spec as failed (1). + expect(body.summary.total).toBe(3); + expect(body.summary.created).toBe(2); + expect(body.summary.failed).toBe(1); + }); + }); +}); diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts new file mode 100644 index 0000000..3602844 --- /dev/null +++ b/src/lib/dry-run/samples.ts @@ -0,0 +1,777 @@ +/** + * Canned sample responses for `--dry-run`. One entry per M2 endpoint; + * shapes match the backend facade contract and the MSW happy-path fixtures + * in `test/mock-backend/fixtures.ts` so an agent that learned the surface + * via dry-run sees the same wire shape it would in a real call. + * + * Why duplicate the test fixtures here instead of importing them: + * - Test files (`test/**`) are excluded from the published artifact via + * `package.json#files` (`["dist"]`). Anything under `src/lib/**` ships; + * anything under `test/**` does not. Dry-run must work for installed + * users, so its data has to live in `src/`. + * - `samples.test.ts` cross-checks this file against + * `test/mock-backend/fixtures.ts` shape-for-shape so drift between the + * two surfaces fails the build. + * + * If the CLI OpenAPI spec changes, both this file AND + * `test/mock-backend/fixtures.ts` must be updated in the same PR. + */ +import type { CliProject, CliUpdateProjectResponse } from '../../commands/project.js'; +import type { + CliBulkDeleteSummary, + CliFailureContext, + CliFailureSummary, + CliLatestResult, + CliTest, + CliTestCode, + CliTestStep, +} from '../../commands/test.js'; +import type { MeResponse } from '../../commands/auth.js'; +import type { Page } from '../pagination.js'; +import type { + TriggerRunResponse, + RunResponse, + RerunResponse, + BatchRerunResponse, + BatchRunFreshResponse, + ListRunsResponse, +} from '../runs.types.js'; + +const SAMPLE_USER_ID = '11111111-1111-4111-8111-111111111111'; +const SAMPLE_KEY_ID = 'key_dryrun_2026'; + +const SAMPLE_PROJECT_ID = 'project_b3c91efa'; +const SAMPLE_PROJECT_ID_BACKEND = 'project_a47b2c11'; +const SAMPLE_TEST_ID_FAILED = 'test_8f2a4d10'; +const SAMPLE_TEST_ID_PASSED = 'test_3a91bb02'; +const SAMPLE_TEST_ID_BLOCKED = 'test_blocked_4f7a'; +export const SAMPLE_RUN_ID = 'run_abc'; +// M3.4 rerun dry-run sample IDs +const SAMPLE_RERUN_ID_BE_NAMED = 'run_rerun_be_named'; +const SAMPLE_RERUN_ID_BE_PRODUCER = 'run_rerun_be_producer'; +const SAMPLE_TEST_ID_BE_CONSUMER = 'test_be_consumer_01'; +const SAMPLE_TEST_ID_BE_PRODUCER = 'test_be_producer_01'; +const SAMPLE_RERUN_BATCH_ID_1 = 'run_batch_rerun_001'; +const SAMPLE_RERUN_BATCH_ID_2 = 'run_batch_rerun_002'; +const SAMPLE_TEST_ID_BATCH_1 = 'test_batch_01'; +const SAMPLE_TEST_ID_BATCH_2 = 'test_batch_02'; +const SAMPLE_TEST_ID_DEFERRED = 'test_deferred_01'; +// M4 piece-2 — batch fresh run sample IDs +const SAMPLE_BATCH_FRESH_RUN_ID_1 = 'run_fresh_batch_001'; +const SAMPLE_BATCH_FRESH_RUN_ID_2 = 'run_fresh_batch_002'; +const SAMPLE_TEST_ID_FRESH_1 = 'test_fresh_wave_01'; +const SAMPLE_TEST_ID_FRESH_2 = 'test_fresh_wave_02'; +const SAMPLE_SNAPSHOT_ID = 'snap_2026_05_05_b2f9a1c8'; +const SAMPLE_TARGET_URL = 'https://staging.example.com/checkout'; +const SAMPLE_REQUEST_ID = 'req_dry-run'; + +export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; + +const me: MeResponse = { + userId: SAMPLE_USER_ID, + keyId: SAMPLE_KEY_ID, + scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], + env: 'development', +}; + +const projects: CliProject[] = [ + { + id: SAMPLE_PROJECT_ID, + name: 'Checkout', + type: 'frontend', + createdFrom: 'portal', + createdAt: '2026-04-15T10:23:00.000Z', + updatedAt: '2026-05-05T08:12:00.000Z', + }, + { + id: SAMPLE_PROJECT_ID_BACKEND, + name: 'Internal API', + type: 'backend', + createdFrom: 'mcp', + createdAt: '2026-03-01T14:00:00.000Z', + updatedAt: '2026-05-04T19:30:00.000Z', + }, +]; + +const tests: CliTest[] = [ + { + id: SAMPLE_TEST_ID_FAILED, + projectId: SAMPLE_PROJECT_ID, + name: 'Checkout happy path', + type: 'frontend', + createdFrom: 'portal', + status: 'failed', + // G1a — priority label shown on one row so dry-run learners see the field. + priority: 'p1', + createdAt: '2026-04-20T11:00:00.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', + details: { + processingStatus: 'Idle', + testStatus: 'Failed', + rawStatus: 'ps=Idle; ts=Failed', + }, + }, + { + id: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + name: 'Checkout — declined card', + type: 'frontend', + createdFrom: 'portal', + status: 'passed', + priority: null, + createdAt: '2026-04-20T11:00:00.000Z', + updatedAt: '2026-05-05T08:00:00.000Z', + details: { + processingStatus: 'Idle', + testStatus: 'Passed', + rawStatus: 'ps=Idle; ts=Passed', + }, + }, + { + // M2.1 piece 1: distinct `blocked` status surfaces here so a + // dry-run learner sees how blocked rows render. Pre-M2.1 this + // would have shown up as `failed` and would have been + // indistinguishable from the row above. + id: SAMPLE_TEST_ID_BLOCKED, + projectId: SAMPLE_PROJECT_ID, + name: 'Checkout — coupon redemption', + type: 'frontend', + createdFrom: 'portal', + status: 'blocked', + priority: null, + createdAt: '2026-04-21T09:00:00.000Z', + updatedAt: '2026-05-05T13:01:12.000Z', + details: { + processingStatus: 'Idle', + testStatus: 'Blocked', + rawStatus: 'ps=Idle; ts=Blocked', + }, + }, +]; + +const testCode: CliTestCode = { + testId: SAMPLE_TEST_ID_FAILED, + language: 'typescript', + framework: 'playwright', + code: [ + "import { test, expect } from '@playwright/test';", + "test('checkout happy path', async ({ page }) => {", + ' await page.goto(process.env.TARGET_URL!);', + ' await page.click(\'[data-testid="cart"]\');', + ' await page.click(\'[data-testid="submit"]\');', + " await expect(page.getByRole('heading', { name: 'Order placed' })).toBeVisible();", + '});', + '', + ].join('\n'), + codeVersion: 'v3', + etag: 'sha256:c7c4a4f6c1b8c2e5', +}; + +// Sample step list. Includes both `outcomeContributesToFailure` per-step +// flags (M2.1 piece 4) and the synthetic terminal "assertion" row +// (piece-4 follow-up — also surfaced on `/steps`, not just `/failure`). +// Step 5 is a real per-step failure → flag `true` only on that row; +// step 7 is the synthesizer's fallback for assertion-level failures +// where no per-step row caught the verdict, so dry-run learners see +// both shapes in one fixture rather than only the per-step case. +const testSteps: CliTestStep[] = [ + { + testId: SAMPLE_TEST_ID_FAILED, + stepIndex: 4, + action: 'click', + description: 'Click the cart icon', + status: 'passed', + screenshotUrl: 'https://s3-presigned.example.com/snap/04.png?X-Amz-dryrun', + htmlSnapshotUrl: 'https://s3-presigned.example.com/snap/04.html?X-Amz-dryrun', + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: '2026-05-05T12:34:55.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', + outcomeContributesToFailure: false, + }, + { + testId: SAMPLE_TEST_ID_FAILED, + stepIndex: 5, + action: 'click', + description: 'Click the submit button', + status: 'failed', + screenshotUrl: 'https://s3-presigned.example.com/snap/05.png?X-Amz-dryrun', + htmlSnapshotUrl: 'https://s3-presigned.example.com/snap/05.html?X-Amz-dryrun', + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: '2026-05-05T12:34:56.000Z', + updatedAt: '2026-05-05T12:34:56.000Z', + outcomeContributesToFailure: true, + }, + { + testId: SAMPLE_TEST_ID_FAILED, + stepIndex: 6, + action: 'expect', + description: 'Expect order confirmation heading', + status: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + capturedAt: null, + updatedAt: '2026-05-05T12:34:58.000Z', + outcomeContributesToFailure: false, + }, +]; + +const latestResult: CliLatestResult = { + testId: SAMPLE_TEST_ID_FAILED, + status: 'failed', + startedAt: '2026-05-05T12:34:00.000Z', + finishedAt: '2026-05-05T12:34:58.000Z', + videoUrl: 'https://s3-presigned.example.com/video/run_abc.mp4?X-Amz-dryrun', + failureAnalysisUrl: 'https://s3-presigned.example.com/analysis/run_abc.json?X-Amz-dryrun', + snapshotId: SAMPLE_SNAPSHOT_ID, + runIdIfAvailable: SAMPLE_RUN_ID, + codeVersion: 'v3', + targetUrl: SAMPLE_TARGET_URL, + targetUrlSource: 'run', + failedStepIndex: 5, + failureKind: 'assertion', + summary: { passed: 4, failed: 1, skipped: 0 }, +}; + +const failureContext: CliFailureContext = { + snapshotId: SAMPLE_SNAPSHOT_ID, + testId: SAMPLE_TEST_ID_FAILED, + projectId: SAMPLE_PROJECT_ID, + result: latestResult, + steps: testSteps, + code: testCode, + failure: { + rootCauseHypothesis: + 'Submit button has `opacity: 0.4` and is rendered as disabled. The checkout ' + + "form's `isFormValid` predicate evaluates to false because the credit-card " + + 'field is empty.', + recommendedFixTarget: { + kind: 'code', + reference: 'src/components/CheckoutForm.tsx:412', + rationale: + 'The disabled state originates from `isFormValid()` in CheckoutForm.tsx; the ' + + 'submit button only enables when `card.number.length === 16`. The test fixture ' + + 'leaves the card field empty.', + }, + evidence: [ + { + kind: 'screenshot', + stepIndex: 5, + url: 'https://s3-presigned.example.com/evidence/run_abc/05.png?X-Amz-dryrun', + summary: + 'Submit button has `opacity: 0.4` and is rendered as disabled. Credit-card ' + + 'field is empty. No visible error message.', + }, + { + kind: 'snapshot', + stepIndex: 5, + url: 'https://s3-presigned.example.com/evidence/run_abc/05.html?X-Amz-dryrun', + summary: + '`