diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 890f9c7..edde076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,32 +48,39 @@ jobs: - run: npm run typecheck test: - name: Unit Tests + name: Unit Tests (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci + - run: npm run build - run: npm test env: CI: true build: - name: Build + name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4eb15a9..ebb3331 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -73,7 +73,7 @@ jobs: fi - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' with: recreate: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e0d49..4f1a0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +### Added + +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. +- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. + ## [0.2.0] - 2026-06-29 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b995b9..2110f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ the thread or in Discord is welcome. ## Prerequisites -- Node 20 or newer (development happens on Node 22). +- Node 20.19+, Node 22.13+, or Node 24+ (development happens on Node 22). ## Build from source diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7e50c36..6b798ef 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -35,7 +35,7 @@ Or run it without installing: npx @testsprite/testsprite-cli --version ``` -Requires **Node.js ≥ 20**. +Requires **Node.js 20.19+**, **22.13+**, or **24+**. Confirm the binary works **without** configuring an API key: @@ -112,15 +112,18 @@ testsprite agent install claude # install the skill for Claude Code testsprite agent install codex # install into AGENTS.md for Codex (managed-section) testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md +testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 5 targets with status + mode + path +testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md +testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md +testsprite agent list # list all 8 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. ## Command reference @@ -319,10 +322,10 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--target-url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. ```bash -testsprite project create --name "Checkout" --target-url https://staging.example.com +testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` @@ -344,9 +347,19 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json + +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json ``` -`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr at `--verbose`); pass `--idempotency-key ` to control it explicitly. +Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. + +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -365,10 +378,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json # Rerun every test in a project (batch) testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json +# Batch rerun with JUnit XML for CI +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + # Several specific tests testsprite test rerun test_aaaa test_bbbb --wait --output json ``` +Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project ` explicitly (required under `--dry-run`). + Flags: - `--all` — rerun every test in the resolved project; requires `--project `. @@ -376,10 +399,34 @@ Flags: - `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. -- `--idempotency-key ` — auto-minted when omitted. +- `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). +- `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. +#### `testsprite test flaky ` + +Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Frontend replays are free verbatim script replays; backend tests re-run their dependency closure and may cost credits (a one-line stderr advisory is printed before the run). + +```bash +# Replay 10 times and print a stability score +testsprite test flaky test_xxxxxxxx --runs 10 + +# Fast "is it flaky at all?" — stop at the first non-passing attempt +testsprite test flaky test_xxxxxxxx --runs 10 --until-fail + +# Machine-readable stability report for CI +testsprite test flaky test_xxxxxxxx --runs 10 --output json +``` + +Flags: + +- `--runs ` — number of replays (1–10, default 5). +- `--until-fail` — stop at the first attempt that does not pass. +- `--timeout ` — per-attempt polling deadline (same semantics as `test wait`). + +`--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. + #### `testsprite test wait ` Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. @@ -426,12 +473,25 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | + +### Update notice + +Interactive runs print a one-line "new version available" notice on stderr when +a newer release exists. To learn this, the CLI contacts the public npm registry +(`registry.npmjs.org`) at most once per 24 hours; the request carries the +package name only — never your API key, project data, or command line. The +check is skipped in CI, when stderr is not a TTY, under `--output json` / +`--dry-run`, and entirely when `TESTSPRITE_NO_UPDATE_NOTIFIER` is set. Any +failure is silent: the notice can never break or delay a command. This is the +only outbound call the CLI makes besides your configured API endpoint. ### Scopes diff --git a/README.md b/README.md index 2d90012..404f237 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ AI ships code in minutes — verifying it hasn't. `testsprite` opens your live a

npm version npm downloads - Node >= 20 + Node 20.19+, 22.13+, or 24+ License Apache 2.0 CI

@@ -54,14 +54,14 @@ If you find `testsprite` useful, a GitHub Star ⭐️ would be greatly appreciat ## Quickstart -Requires **Node.js ≥ 20**. (No global install? `npx @testsprite/testsprite-cli` works too.) +Requires **Node.js 20.19+**, **22.13+**, or **24+**. (No global install? `npx @testsprite/testsprite-cli` works too.) ```bash npm install -g @testsprite/testsprite-cli testsprite setup ``` -`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): +`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `windsurf`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): ```bash TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude @@ -89,28 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). @@ -173,7 +173,7 @@ That's the point of all of this: you no longer need the biggest, most expensive ## Contributing -Contributions are welcome — the CLI is plain TypeScript/Node (≥ 20), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: +Contributions are welcome — the CLI is plain TypeScript/Node (20.19+, 22.13+, or 24+), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: ```bash git clone https://github.com/TestSprite/testsprite-cli.git diff --git a/package-lock.json b/package-lock.json index 4ea700c..b254e0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "bin": { @@ -29,7 +30,7 @@ "vitest": "^2.1.4" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@ampproject/remapping": { @@ -1024,9 +1025,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1041,9 +1039,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1058,9 +1053,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1075,9 +1067,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1092,9 +1081,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1109,9 +1095,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1126,9 +1109,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1143,9 +1123,6 @@ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1160,9 +1137,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1177,9 +1151,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1194,9 +1165,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1211,9 +1179,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1228,9 +1193,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3855,6 +3817,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 2037d26..341e713 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:e2e": "npm run build && vitest run -c vitest.e2e.config.ts" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "publishConfig": { "access": "public" @@ -50,6 +50,7 @@ "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "devDependencies": { diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 523c5fc..4f5fdd3 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -9,13 +9,21 @@ import { MANAGED_SECTION_END, ONBOARD_CODEX_LINE, SKILLS, + buildSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, TARGETS, type AgentTarget, } from '../lib/agent-targets.js'; -import type { AgentDeps, AgentFs, InstallResult, ListResult } from './agent.js'; -import { AGENTS_MD_CODEX_BUDGET_BYTES, createAgentCommand, runInstall, runList } from './agent.js'; +import type { AgentDeps, AgentFs, InstallResult, ListResult, StatusResult } from './agent.js'; +import { + AGENTS_MD_CODEX_BUDGET_BYTES, + createAgentCommand, + runInstall, + runList, + runStatus, +} from './agent.js'; // --------------------------------------------------------------------------- // In-memory AgentFs backed by a Map @@ -741,6 +749,7 @@ describe('runList', () => { expect(out).toContain('cursor'); expect(out).toContain('cline'); expect(out).toContain('antigravity'); + expect(out).toContain('kiro'); expect(out).toContain('codex'); expect(out).toContain('ga'); expect(out).toContain('experimental'); @@ -749,6 +758,7 @@ describe('runList', () => { expect(out).toContain(TARGETS.cursor.path); expect(out).toContain(TARGETS.cline.path); expect(out).toContain(TARGETS.antigravity.path); + expect(out).toContain(TARGETS.kiro.path); expect(out).toContain(TARGETS.codex.path); }); @@ -759,13 +769,16 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 5 targets × 2 default skills = 10 rows - expect(json).toHaveLength(10); + // 8 targets × 2 default skills = 16 rows + expect(json).toHaveLength(16); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); + expect(targets).toContain('windsurf'); + expect(targets).toContain('copilot'); expect(targets).toContain('antigravity'); + expect(targets).toContain('kiro'); expect(targets).toContain('codex'); // skill field present on each row const skills = json.map(r => r.skill); @@ -905,11 +918,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All four own-file targets installed at once +// All own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all four own-file targets', () => { - it('installs all four own-file targets in one invocation', async () => { +describe('runInstall — all own-file targets', () => { + it('installs every own-file target in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -919,7 +932,7 @@ describe('runInstall — all four own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: [...OWN_FILE_TARGETS], skills: ['testsprite-verify'], force: false, }, @@ -937,11 +950,11 @@ describe('runInstall — all four own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all four own-file targets +// Dry-run for all seven own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the four own-file targets (default 2 skills = 8 would-write lines)', async () => { + it('writes nothing for any of the seven own-file targets (default 2 skills = 14 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -951,7 +964,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf', 'copilot'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -961,9 +974,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 4 targets × 2 default skills = 8 would-write lines + // 7 targets × 2 default skills = 14 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(8); + expect(wouldWriteLines.length).toBe(14); }); }); @@ -1311,6 +1324,50 @@ describe('runInstall — symlink safety', () => { expect(writeCalls.length).toBe(0); // never wrote a .bak nor through the link }); + it('dry-run: refuses (exit 5) when the target file is a symlink (parity with real install)', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // Same planted SKILL.md symlink as the real-install case above: dry-run + // must report the same refusal the real install would, not a success. + seedSymlink(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + + it('dry-run: refuses (exit 5) when a parent path component is a symlink', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + seedSymlink(path.resolve(CWD, '.claude')); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + it('does not write through a symlinked .bak slot — backs up to a numbered slot', async () => { const { store, fs: agentFs, seedFile, seedSymlink } = makeMemFs(); const abs = path.resolve(CWD, TARGETS.claude.path); @@ -2383,3 +2440,120 @@ describe('runInstall — SKILLS registry / DEFAULT_SKILLS contract', () => { expect(ONBOARD_CODEX_LINE).toContain('**First-time setup:**'); }); }); + +// --------------------------------------------------------------------------- +// runStatus — `agent status` (issue #123) +// --------------------------------------------------------------------------- + +describe('runStatus — agent status (issue #123)', () => { + const statusOpts = { + profile: 'default' as const, + output: 'json' as const, + debug: false, + dryRun: false, + }; + + /** Run status against the given fs and return the printed rows. */ + async function statusRows(agentFs: AgentFs): Promise<{ rows: StatusResult[]; thrown: unknown }> { + const { capture, deps } = makeCapture(); + let thrown: unknown; + try { + await runStatus(statusOpts, { cwd: CWD, fs: agentFs, ...deps }); + } catch (err) { + thrown = err; + } + return { rows: JSON.parse(capture.stdout.join('')) as StatusResult[], thrown }; + } + + it('nothing installed: every row is absent and the command exits 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + expect(rows).toHaveLength(Object.keys(TARGETS).length * DEFAULT_SKILLS.length); + expect(rows.every(row => row.state === 'absent')).toBe(true); + }); + + it('fresh installs read ok (own-file and codex managed section), exit 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'codex'], + skills: [...DEFAULT_SKILLS], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + for (const skill of DEFAULT_SKILLS) { + expect(rows.find(r => r.target === 'claude' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'codex' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'cursor' && r.skill === skill)?.state).toBe('absent'); + } + }); + + it('stale: a marker whose hash matches an OLDER body reads stale and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const oldBody = '# TestSprite Verification Loop\n\nold body from a previous CLI release\n'; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + buildSkillMarker('testsprite-verify', oldBody), + oldBody, + ), + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'stale', + ); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(1); + expect((thrown as CLIError).message).toContain('need attention'); + }); + + it('modified: current hash but edited bytes reads modified and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const canonical = renderForTarget('claude', 'testsprite-verify').content; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + `${canonical}\n\n`, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'modified', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('unmarked: an artifact without a marker line reads unmarked and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + '# hand-rolled skill file with no marker\n', + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'unmarked', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('rejects an explicit empty --dir (exit 5), matching the resolve-to-cwd hazard', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await expect( + runStatus({ ...statusOpts, dir: ' ' }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + }); +}); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 3347453..0a40c4b 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -4,17 +4,23 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { CLIError, localValidationError } from '../lib/errors.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptText } from '../lib/prompt.js'; import { type AgentTarget, TARGETS, SKILLS, DEFAULT_SKILLS, + MARKER_SKILL_SEPARATOR, pathFor, loadSkillBodyFor, + bodyHash12, + compactBodyFor, buildCodexAggregate, + buildSkillMarker, + parseSkillMarker, renderForTarget, + renderOwnFileWithMarker, MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, } from '../lib/agent-targets.js'; @@ -150,11 +156,15 @@ async function writeBackup(agentFs: AgentFs, abs: string, existing: string): Pro // --------------------------------------------------------------------------- /** - * Build the section block to inject (sentinels + body + trailing newline). + * Build the section block to inject (sentinels + marker + body + trailing + * newline). The provenance marker line sits just inside the BEGIN sentinel so + * `agent status` can fingerprint the section. The same skill set + CLI version + * + body always produce byte-identical output, so the classifySection + * 'unchanged' fast-path keeps working across re-installs. * Uses \n throughout; the caller handles CRLF normalisation. */ -function buildSection(body: string): string { - return `${MANAGED_SECTION_BEGIN}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; +function buildSection(body: string, markerLine: string): string { + return `${MANAGED_SECTION_BEGIN}\n${markerLine}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; } /** @@ -434,10 +444,32 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr } return b; }; + // Budget-capped own-file targets (e.g. windsurf) render the compact per-skill + // body so the rule file isn't truncated by the agent. Cached separately; must + // match renderForTarget's default selection so written bytes equal the asserted + // render. + const compactBodyCache = new Map(); + const compactBodyForSkill = (skill: string): string => { + let b = compactBodyCache.get(skill); + if (b === undefined) { + b = compactBodyFor(skill); + compactBodyCache.set(skill, b); + } + return b; + }; + const ownFileBodyFor = (t: AgentTarget, skill: string): string => + TARGETS[t].compactBody ? compactBodyForSkill(skill) : bodyForSkill(skill); let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { - codexSectionCache = buildSection(buildCodexAggregate(skills)); + const aggregate = buildCodexAggregate(skills); + // ONE marker for the whole managed section: it names every aggregated + // skill ('+'-joined) and hashes the canonical aggregate body, so + // `agent status` can attribute and fingerprint the section per skill. + codexSectionCache = buildSection( + aggregate, + buildSkillMarker(skills.join(MARKER_SKILL_SEPARATOR), aggregate), + ); } return codexSectionCache; }; @@ -635,9 +667,20 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (abs !== root && !abs.startsWith(root + path.sep)) { throw new CLIError(`refusing to write outside --dir: ${relPath}`, 5); } - const content = renderForTarget(t, skill, bodyForSkill(skill)).content; + const content = renderForTarget(t, skill, ownFileBodyFor(t, skill)).content; if (opts.dryRun) { + // Apply the SAME symlink fail-close guard as the real install path + // below (the codex managed-section branch already does this). Without + // it, dry-run reports success for a planted symlink that the real + // install would refuse with exit 5. + const dryRunSt = await inspectTargetPath(agentFs, root, relPath); + if (dryRunSt !== null && !dryRunSt.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } const bytes = Buffer.byteLength(content, 'utf8'); dryRunLines.push({ abs, bytes, note: '' }); results.push({ target: t, path: relPath, action: 'dry-run', skills: [skill] }); @@ -778,6 +821,236 @@ export async function runList(opts: CommonOptions, deps: AgentDeps = {}): Promis }); } +// --------------------------------------------------------------------------- +// runStatus (issue #123: detect silently stale installed skill files) +// --------------------------------------------------------------------------- + +/** + * Health of one installed skill artifact, as reported by `agent status`. + * + * Decision order (first match wins): + * - 'absent' : nothing at the landing path (codex: no managed section, + * including an AGENTS.md that exists without our sentinels). + * - 'corrupt' : codex only. Dangling or duplicated sentinels, the same + * classification `agent install` refuses on; status REPORTS it + * instead of refusing. + * - 'unmarked' : artifact present but carries no testsprite-skill marker + * (installed before markers existed), or the landing path is + * occupied by a non-regular file (never followed). + * - 'stale' : marker present, but its hash differs from the current + * canonical body: a re-install would change the content. Edits + * on top of an OLD install also read stale (older renders + * cannot be reproduced); the remedy is the same re-install. + * - 'modified' : marker hash matches the current body, but the artifact bytes + * differ from the canonical render carrying that same marker + * line: the user edited the artifact after install. + * - 'ok' : marker hash matches and the bytes equal the canonical render + * with the file's own marker line (a version-string-only lag + * with an unchanged body still reads ok). + * + * For the codex managed section, ONE marker names every aggregated skill + * ('+'-joined); skills not named by the marker report 'absent'. + */ +export type SkillArtifactState = 'ok' | 'stale' | 'modified' | 'unmarked' | 'absent' | 'corrupt'; + +export interface StatusResult { + target: AgentTarget; + skill: string; + path: string; + state: SkillArtifactState; +} + +interface StatusOptions extends CommonOptions { + dir?: string; +} + +/** + * Classify one own-file artifact per the {@link SkillArtifactState} contract. + * Comparisons are byte-exact, matching the installer's own skipped/blocked + * comparison for own-file targets. + */ +async function classifyOwnFileState( + agentFs: AgentFs, + abs: string, + target: AgentTarget, + skill: string, + bodyForSkill: (skill: string) => string, +): Promise { + const stat = await agentFs.lstat(abs); + if (stat === null) return 'absent'; + // Occupied by a directory or symlink: not something our installer wrote, and + // never followed (mirrors the installer's fail-closed stance on symlinks). + if (!stat.isFile) return 'unmarked'; + + const existing = await agentFs.readFile(abs); + const marker = parseSkillMarker(existing); + if (marker === null) return 'unmarked'; + + const canonicalBody = bodyForSkill(skill); + if (marker.hash12 !== bodyHash12(canonicalBody)) return 'stale'; + + // Hash matches the current body: pristine iff the file equals the canonical + // render carrying its own marker line, so a marker whose version string lags + // behind an unchanged body still reads ok. + const reRender = renderOwnFileWithMarker(target, skill, marker.line, canonicalBody); + return existing === reRender ? 'ok' : 'modified'; +} + +/** + * Classify the codex managed section per skill. The section is ONE artifact + * carrying ONE marker that names every aggregated skill, so a single + * inspection answers all skill rows; the returned function maps a skill name + * to its state. Comparisons are CRLF-insensitive on the section bytes. + */ +async function classifyManagedSectionStates( + agentFs: AgentFs, + abs: string, +): Promise<(skill: string) => SkillArtifactState> { + const constantState = + (state: SkillArtifactState): ((skill: string) => SkillArtifactState) => + () => + state; + + const stat = await agentFs.lstat(abs); + if (stat === null) return constantState('absent'); + // Occupied by a directory or symlink: never followed (fail-closed). + if (!stat.isFile) return constantState('unmarked'); + + const existing = await agentFs.readFile(abs); + + // Current canonical section for the default skill set. classifySection's + // 'unchanged' answers the common all-defaults-fresh case; its + // corrupt/append classification is reused verbatim for status verdicts. + const defaultAggregate = buildCodexAggregate(DEFAULT_SKILLS); + const defaultSection = buildSection( + defaultAggregate, + buildSkillMarker(DEFAULT_SKILLS.join(MARKER_SKILL_SEPARATOR), defaultAggregate), + ); + const sectionState = classifySection(existing, defaultSection); + + if (sectionState.kind === 'corrupt') return constantState('corrupt'); + // No standalone sentinels anywhere: the managed section is not installed. + if (sectionState.kind === 'append') return constantState('absent'); + if (sectionState.kind === 'unchanged') { + // Byte-identical to today's default install. + return skill => ((DEFAULT_SKILLS as readonly string[]).includes(skill) ? 'ok' : 'absent'); + } + if (sectionState.kind !== 'replace') { + // 'create' is unreachable when the file exists; treat defensively as absent. + return constantState('absent'); + } + + // Sentinels are present but the section differs from today's default + // canonical: slice the live section bytes out of the file and inspect its + // own marker (before/after are exact byte prefix/suffix around the section). + const sectionContent = existing.slice( + sectionState.before.length, + existing.length - sectionState.after.length, + ); + const marker = parseSkillMarker(sectionContent); + if (marker === null) return constantState('unmarked'); + + const installedSkills = marker.skill.split(MARKER_SKILL_SEPARATOR); + const coversSkill = (skill: string): boolean => installedSkills.includes(skill); + + // A marker naming a skill this CLI does not ship cannot be re-rendered; + // report the named skills stale (a re-install refreshes the section). + if (installedSkills.some(name => SKILLS[name] === undefined)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + const canonicalAggregate = buildCodexAggregate(installedSkills); + if (marker.hash12 !== bodyHash12(canonicalAggregate)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + // Hash matches the current aggregate: the section is pristine iff its bytes + // equal a re-render carrying its own marker line (version-string-only lag + // with an unchanged body still reads ok). + const pristine = + sectionContent.replace(/\r\n/g, '\n') === buildSection(canonicalAggregate, marker.line); + return skill => (coversSkill(skill) ? (pristine ? 'ok' : 'modified') : 'absent'); +} + +/** + * `agent status`: one row per (target × default skill), each classified per + * the {@link SkillArtifactState} contract. Exit contract: returns normally + * (exit 0) when every row is 'ok' or 'absent'; throws CLIError exit 1 when any + * row is stale/modified/unmarked/corrupt, so the command can gate CI. + */ +export async function runStatus(opts: StatusOptions, deps: AgentDeps = {}): Promise { + const agentFs = deps.fs ?? defaultAgentFs; + const out = makeOutput(opts.output, deps); + + // An explicit but empty --dir must not silently resolve to cwd + // (path.resolve('') === cwd). + if (opts.dir !== undefined && opts.dir.trim() === '') { + throw localValidationError('dir', 'must not be empty'); + } + const dir = opts.dir !== undefined ? opts.dir.trim() : (deps.cwd ?? process.cwd()); + const root = path.resolve(dir); + + // Canonical own-file bodies, read once per skill (same lazy caching pattern + // as runInstall's bodyForSkill). + const skillBodyCache = new Map(); + const bodyForSkill = (skill: string): string => { + let cachedBody = skillBodyCache.get(skill); + if (cachedBody === undefined) { + cachedBody = loadSkillBodyFor(skill); + skillBodyCache.set(skill, cachedBody); + } + return cachedBody; + }; + + const results: StatusResult[] = []; + for (const [target, spec] of Object.entries(TARGETS) as [ + AgentTarget, + { mode: string; path: string }, + ][]) { + if (spec.mode === 'managed-section') { + const stateFor = await classifyManagedSectionStates(agentFs, path.resolve(root, spec.path)); + for (const skill of DEFAULT_SKILLS) { + results.push({ target, skill, path: spec.path, state: stateFor(skill) }); + } + continue; + } + for (const skill of DEFAULT_SKILLS) { + const relPath = pathFor(target, skill); + results.push({ + target, + skill, + path: relPath, + state: await classifyOwnFileState( + agentFs, + path.resolve(root, relPath), + target, + skill, + bodyForSkill, + ), + }); + } + } + + out.print(results, data => { + const items = data as StatusResult[]; + const header = `${'TARGET'.padEnd(14)} ${'SKILL'.padEnd(20)} ${'STATE'.padEnd(10)} PATH`; + const rows = items.map( + row => `${row.target.padEnd(14)} ${row.skill.padEnd(20)} ${row.state.padEnd(10)} ${row.path}`, + ); + return [header, ...rows].join('\n'); + }); + + const needingAttention = results.filter( + result => result.state !== 'ok' && result.state !== 'absent', + ); + if (needingAttention.length > 0) { + throw new CLIError( + `${needingAttention.length} skill artifact(s) need attention (stale/modified/unmarked/corrupt); re-run \`testsprite agent install\` (add --force for own-file targets) to refresh them.`, + 1, + ); + } +} + // --------------------------------------------------------------------------- // Command factory // --------------------------------------------------------------------------- @@ -788,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex)', ); agent @@ -798,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', collect, [], ) @@ -841,6 +1114,17 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { await runList(resolveCommonOptions(command), deps); }); + agent + .command('status') + .description( + 'Check installed TestSprite skill files against this CLI version: ok, stale, modified, unmarked, absent, or corrupt (exits 1 when anything needs attention, so it can gate CI)', + ) + .option('--dir ', 'Project root to inspect (default: cwd)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: { dir?: string }, command: Command) => { + await runStatus({ ...resolveCommonOptions(command), dir: cmdOpts.dir }, deps); + }); + return agent; } @@ -852,7 +1136,7 @@ function resolveCommonOptions(command: Command): CommonOptions { const globals = command.optsWithGlobals() as Partial; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index d65e6cc..c0b4e00 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -110,6 +110,50 @@ describe('runConfigure', () => { ); }); + it('uses requestTimeoutMs for the pre-write key validation ping', async () => { + const { deps } = makeCapture(); + let sawAbort = false; + const fetchImpl = vi.fn( + async (_input: string | URL | Request, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + const timeout = setTimeout(() => { + reject(new Error('requestTimeoutMs was not applied to the validation ping')); + }, 50); + signal?.addEventListener( + 'abort', + () => { + sawAbort = true; + clearTimeout(timeout); + reject(new DOMException('The operation timed out.', 'TimeoutError')); + }, + { once: true }, + ); + }), + ) as unknown as typeof fetch; + + await expect( + runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + requestTimeoutMs: 1, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + expect(sawAbort).toBe(true); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + it('throws VALIDATION_ERROR when --from-env is set but key is missing', async () => { const { deps } = makeCapture(); await expect( @@ -120,6 +164,100 @@ describe('runConfigure', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('rejects a malformed endpoint before key validation fetch', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('API key rejected'); + }); + + it('rejects a non-http endpoint before key validation fetch', async () => { + const { deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'ftp://example.com', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + + it('rejects a malformed dry-run endpoint before emitting dry-run output', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: false, + dryRun: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('[dry-run]'); + expect(capture.stdout).toEqual([]); + }); + it('prompts only for the API key (never the endpoint) and defaults to prod', async () => { const { capture, deps } = makeCapture(); // Prompt object exposes ONLY `secret`. If runConfigure tried to prompt for @@ -140,6 +278,35 @@ describe('runConfigure', () => { expect(capture.prelude.join('')).toContain('Configuring profile "default"'); }); + it('routes the interactive prelude to stderr by default, keeping stdout for the result', async () => { + // Regression: the prelude used to default to process.stdout, polluting the + // result stream (and the JSON document under --output json). With no + // injected preludeWrite/stderr, the default must land on stderr, not stdout. + const stdout: string[] = []; + const errChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + try { + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { + stdout: line => stdout.push(line), + prompt: { secret: vi.fn(async () => 'sk-typed') }, + fetchImpl: meOkFetch, + credentialsPath, + env: {}, + }, + ); + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + } + expect(errChunks.join('')).toContain('Configuring profile "default"'); + expect(stdout.join('\n')).not.toContain('Configuring profile'); + }); + it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { const { capture, deps } = makeCapture(); const prompt = { secret: vi.fn(async () => 'sk-typed') }; @@ -268,6 +435,46 @@ describe('runConfigure', () => { expect(capture.stderr.join('\n')).toContain('profile NOT updated'); }); + it('key-rejected error preserves the typed ApiError envelope (JSON contract)', async () => { + const { deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: { reason: 'malformed' }, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + // The thrown error must be an ApiError (with code, nextAction, requestId) + // — not a CLIError wrapper that drops those fields. Under --output json, + // index.ts renders ApiError as the full typed envelope; CLIError would + // render only {"error":"...string..."}, violating the JSON contract. + await expect( + runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toMatchObject({ + code: 'AUTH_INVALID', + exitCode: 3, + nextAction: 'Rotate your key.', + requestId: 'req_reject', + }); + }); + // The old "run `testsprite agent install`" self-bootstrap tip was removed with // the setup consolidation — runConfigure now runs ONLY as part of `setup`, // which installs the skill itself. These guard that the tip stays gone. @@ -643,6 +850,20 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + ...deps, + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('endpoint: https://api.testsprite.com'); + }); + it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', diff --git a/src/commands/auth.ts b/src/commands/auth.ts index de1eda4..59cd43f 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -1,7 +1,9 @@ import { Command } from 'commander'; import { + assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import type { ErrorCode } from '../lib/errors.js'; @@ -15,10 +17,10 @@ import { readProfile, writeProfile, } from '../lib/credentials.js'; -import { loadConfig } from '../lib/config.js'; +import { loadConfig, normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; export interface MeResponse { @@ -73,21 +75,25 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): const env = deps.env ?? process.env; const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const out = makeOutput(opts.output, deps); - const prelude = deps.preludeWrite ?? ((chunk: string) => process.stdout.write(chunk)); + // The "Configuring profile …" prelude is informational, not result data, so + // it defaults to stderr — stdout stays a pure result stream (the configured + // JSON/text), which matters under `--output json` (§8.1 stdout purity). + const prelude = deps.preludeWrite ?? ((chunk: string) => process.stderr.write(chunk)); const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Normalize the env endpoint: an empty / whitespace-only TESTSPRITE_API_URL is // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a // shell profile) is non-nullish and would short-circuit the `??` chains below to // an empty endpoint instead of falling through to the profile / prod default. - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); // Dry-run: do not prompt, do not read env, do not write credentials. // Print the canned success shape so an agent sees exactly the JSON it // would get on a real configure (modulo the endpoint string). if (opts.dryRun) { - emitDryRunBanner(stderr); const apiUrl = opts.endpointUrl ?? envApiUrl ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); + emitDryRunBanner(stderr); stderr(`[dry-run] would write credentials for profile="${opts.profile}" to ${credentialsPath}`); out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { const d = data as { profile: string; apiUrl: string }; @@ -113,6 +119,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // api_url doesn't silently validate a new key against the default endpoint. const resolvedFromProfile = existingProfile?.apiUrl; const apiUrl = opts.endpointUrl ?? envApiUrl ?? resolvedFromProfile ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); if (opts.fromEnv) { apiKey = env.TESTSPRITE_API_KEY?.trim(); @@ -145,6 +152,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): baseUrl: facadeBaseUrl(apiUrl), apiKey, fetchImpl: deps.fetchImpl, + requestTimeoutMs: opts.requestTimeoutMs, }); try { // Tag the validation call with the originating command (when provided) so @@ -158,13 +166,22 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): } catch (err) { const message = err instanceof Error ? err.message : String(err); stderr(`API key rejected by ${apiUrl}: ${message} — profile NOT updated`); - const exitCode = err instanceof ApiError ? err.exitCode : 3; - // Include the resolved endpoint in the thrown message so the user knows - // which host rejected the key. This prevents the "invalid or revoked" - // message from being ambiguous when the key is valid for a different env. + // When the verification call returned a typed API error (AUTH_INVALID, + // AUTH_FORBIDDEN, etc.), re-throw it directly so `index.ts` renders the + // full typed envelope under `--output json` (code, nextAction, requestId, + // details). Previously wrapping it in CLIError discarded those fields and + // emitted a bare `{"error":"...string..."}` — violating the JSON contract. + // Augment the message with the endpoint context so text-mode users still + // see which host rejected the key. + if (err instanceof ApiError) { + err.message = `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`; + throw err; + } + // Non-ApiError (truly unexpected throws like a TypeError from a + // misconfigured fetchImpl). Exit 3 (auth family). throw new CLIError( `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`, - exitCode, + 3, ); } @@ -191,7 +208,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi // displayed URL always matches where requests actually go (dogfood L1788). let resolvedEndpoint: string; if (opts.dryRun) { - resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; + resolvedEndpoint = + opts.endpointUrl ?? normalizeEnvVar(env.TESTSPRITE_API_URL) ?? 'https://api.testsprite.com'; } else { const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const config = loadConfig({ @@ -318,7 +336,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -327,19 +345,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values — the - * factory applies the same clamp so there is no double-clamp risk. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeOutput(mode: OutputMode, deps: AuthDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts new file mode 100644 index 0000000..230398a --- /dev/null +++ b/src/commands/doctor.test.ts @@ -0,0 +1,229 @@ +/** + * Unit tests for `testsprite doctor`. + * + * The command reuses the real resolution helpers (loadConfig, makeHttpClient, + * isVerifySkillInstalled), so these tests inject env/credentials/fetch/fs and + * assert on the rendered report + the exit-on-failure contract. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CLIError } from '../lib/errors.js'; +import { writeProfile } from '../lib/credentials.js'; +import type { DoctorDeps, DoctorReport } from './doctor.js'; +import { createDoctorCommand, runDoctor } from './doctor.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { capture: CapturedOutput; deps: Pick } { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +function makeFetch(body: unknown, status = 200): DoctorDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as DoctorDeps['fetchImpl']; +} + +const OK_ME = { userId: 'u-doc', keyId: 'k-doc' }; + +/** Base deps shared by the healthy-path tests: node OK, skill installed, empty env. */ +function healthyDeps(credentialsPath: string, extra: Partial = {}): DoctorDeps { + return { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, // skill landing file present + fetchImpl: makeFetch(OK_ME), + ...extra, + }; +} + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-doctor-')), 'credentials'); +}); + +describe('runDoctor — healthy environment', () => { + it('returns an all-passing report and does not throw', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBe(0); + const out = capture.stdout.join('\n'); + expect(out).toContain('[OK]'); + expect(out).toContain('All checks passed.'); + expect(out).toContain('reached GET /me'); + }); + + it('never prints the API key anywhere in the report', async () => { + writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const all = capture.stdout.join('\n') + capture.stderr.join('\n'); + expect(all).not.toContain('sk-super-secret-value'); + }); + + it('emits a machine-readable report under --output json without leaking the API key', async () => { + writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'json', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const raw = capture.stdout.join(''); + // Security: the JSON serialization path is distinct from the text renderer, + // so assert the key never leaks here either. + expect(raw).not.toContain('sk-json-secret-value'); + const parsed = JSON.parse(raw) as DoctorReport; + expect(parsed.failures).toBe(0); + expect(Array.isArray(parsed.checks)).toBe(true); + expect( + parsed.checks.some(check => check.name === 'Connectivity' && check.status === 'ok'), + ).toBe(true); + }); +}); + +describe('runDoctor — failing checks exit non-zero', () => { + it('missing API key fails Credentials and throws CLIError (exit 1)', async () => { + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // no profile written => no key + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(rejection).toMatchObject({ exitCode: 1 }); + const out = capture.stdout.join('\n'); + expect(out).toContain('[FAIL]'); + expect(out).toContain('Credentials'); + }); + + it('invalid endpoint URL fails the API endpoint check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, + { ...healthyDeps(credentialsPath), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('API endpoint'); + expect(out).toContain('not a valid'); + }); + + it('rejected API key surfaces as a Connectivity failure', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const authError = { + error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(authError, 401) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Connectivity'); + expect(out).toContain('API key rejected (AUTH_INVALID)'); + }); + + it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const notFound = { + error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(notFound, 404) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(capture.stdout.join('\n')).toContain('GET /me failed (NOT_FOUND)'); + }); + + it('an outdated Node runtime fails the Node.js check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { nodeVersion: '18.0.0' }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Node.js'); + expect(out).toContain('below the required Node 20'); + }); +}); + +describe('runDoctor — warnings do not fail', () => { + it('missing verify skill is a warning, not a failure', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { existsSync: () => false }), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBeGreaterThanOrEqual(1); + const out = capture.stdout.join('\n'); + expect(out).toContain('[WARN]'); + expect(out).toContain('Verify skill'); + }); + + it('--dry-run skips connectivity and never calls fetch, missing key is a warning', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('fetch must not be called under --dry-run'); + }) as unknown as DoctorDeps['fetchImpl']; + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, + fetchImpl, + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(capture.stdout.join('\n')).toContain('skipped under --dry-run'); + }); +}); + +describe('createDoctorCommand wiring', () => { + it('exposes the doctor command name', () => { + expect(createDoctorCommand().name()).toBe('doctor'); + }); + + it('--help describes the diagnostic', () => { + expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); + }); +}); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts new file mode 100644 index 0000000..2fa0c57 --- /dev/null +++ b/src/commands/doctor.ts @@ -0,0 +1,285 @@ +/** + * `testsprite doctor` — one-shot environment diagnostic. + * + * Runs a fixed checklist (CLI version, Node.js runtime, active profile, API + * endpoint, credentials, live connectivity + key validity, and whether the + * verify skill is installed in the current project) and prints an OK/WARN/FAIL + * report. Exits non-zero when any check FAILS so it can gate a CI step or an + * agent preflight (`testsprite doctor && testsprite test run ...`). Warnings + * (e.g. skill not installed) do not fail the process. + * + * Every check is reused from the same helpers the real commands use, so the + * report reflects exactly what a subsequent command would resolve: `loadConfig` + * for profile/endpoint/key, `assertValidEndpointUrl` for the endpoint gate, + * `makeHttpClient` + `GET /me` for connectivity, and `isVerifySkillInstalled` + * for the skill check. + */ + +import { Command } from 'commander'; +import { + assertValidEndpointUrl, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { loadConfig } from '../lib/config.js'; +import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { VERSION } from '../version.js'; +import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; + +export type DoctorStatus = 'ok' | 'warn' | 'fail'; + +export interface DoctorCheck { + /** Short, stable label (also the JSON key-ish name). */ + name: string; + status: DoctorStatus; + /** Human-readable one-line result. Never contains the API key. */ + detail: string; +} + +export interface DoctorReport { + checks: DoctorCheck[]; + failures: number; + warnings: number; +} + +/** Minimal projection of `GET /me` we read for the connectivity detail. */ +interface MeIdentity { + userId?: string; + keyId?: string; +} + +export interface DoctorDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + /** Project dir for the skill check. Defaults to `process.cwd()`. */ + cwd?: string; + /** Runtime version string (e.g. "22.9.0"). Defaults to `process.versions.node`. */ + nodeVersion?: string; + existsSync?: (p: string) => boolean; + readFileSync?: (p: string) => string; +} + +type CommonOptions = FactoryCommonOptions; + +export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const env = deps.env ?? process.env; + const cwd = deps.cwd ?? process.cwd(); + const nodeVersion = deps.nodeVersion ?? process.versions.node; + + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath: deps.credentialsPath, + }); + const endpointCheck = checkEndpoint(config.apiUrl); + const hasKey = Boolean(config.apiKey); + + const checks: DoctorCheck[] = [ + { name: 'CLI version', status: 'ok', detail: VERSION }, + checkNodeVersion(nodeVersion), + { name: 'Profile', status: 'ok', detail: config.profile }, + endpointCheck, + checkCredentials(hasKey, config.profile, opts.dryRun ?? false), + await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }), + checkSkill(cwd, deps), + ]; + + const failures = checks.filter(check => check.status === 'fail').length; + const warnings = checks.filter(check => check.status === 'warn').length; + const report: DoctorReport = { checks, failures, warnings }; + + out.print(report, () => renderDoctor(report)); + + if (failures > 0) { + // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent + // preflight. The full report already printed above; this line is the stderr + // summary index.ts renders before exiting 1. + throw new CLIError(`doctor: ${failures} check(s) failed, ${warnings} warning(s)`, 1); + } + return report; +} + +function checkNodeVersion(nodeVersion: string): DoctorCheck { + // Reuse the CLI's own runtime guard so the verdict matches exactly what the + // entrypoint enforces at startup, rather than a divergent hardcoded check. + // The precise engines floor (20.19+/22.13+/24+) is enforced by npm at install + // time via .npmrc engine-strict. sourceRef: src/version-guard.ts. + const rejected = shouldRejectNodeVersion(nodeVersion); + return { + name: 'Node.js', + status: rejected ? 'fail' : 'ok', + detail: rejected + ? `v${nodeVersion} is below the required Node ${MIN_SUPPORTED_NODE_MAJOR}; upgrade Node.js` + : `v${nodeVersion} (>=${MIN_SUPPORTED_NODE_MAJOR} required)`, + }; +} + +function checkEndpoint(apiUrl: string): DoctorCheck { + try { + assertValidEndpointUrl(apiUrl); + return { name: 'API endpoint', status: 'ok', detail: apiUrl }; + } catch { + return { + name: 'API endpoint', + status: 'fail', + detail: `"${apiUrl}" is not a valid http(s) URL`, + }; + } +} + +function checkCredentials(hasKey: boolean, profile: string, dryRun: boolean): DoctorCheck { + if (hasKey) { + // Never print any part of the key (security). Confirm presence only. + return { + name: 'Credentials', + status: 'ok', + detail: `API key configured (profile "${profile}")`, + }; + } + // Under --dry-run no key is expected, so a missing key is not a failure. + return { + name: 'Credentials', + status: dryRun ? 'warn' : 'fail', + detail: dryRun + ? 'no API key (not needed under --dry-run)' + : 'no API key found; run `testsprite setup` (or set TESTSPRITE_API_KEY)', + }; +} + +function checkSkill(cwd: string, deps: DoctorDeps): DoctorCheck { + const installed = isVerifySkillInstalled(cwd, { + existsSync: deps.existsSync, + readFileSync: deps.readFileSync, + }); + return { + name: 'Verify skill', + status: installed ? 'ok' : 'warn', + detail: installed + ? 'installed in this project' + : 'not installed here; run `testsprite setup` so your agent verifies its changes', + }; +} + +async function checkConnectivity( + opts: CommonOptions, + deps: DoctorDeps, + ctx: { hasKey: boolean; endpointOk: boolean }, +): Promise { + const name = 'Connectivity'; + if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; + if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; + if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; + + try { + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + const me = await client.get('/me'); + const who = me.userId ? ` (userId ${me.userId})` : ''; + return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; + } catch (error) { + if (error instanceof ApiError) { + if ( + error.code === 'AUTH_REQUIRED' || + error.code === 'AUTH_INVALID' || + error.code === 'AUTH_FORBIDDEN' + ) { + return { name, status: 'fail', detail: `API key rejected (${error.code})` }; + } + return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; + } + return { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }; + } +} + +const STATUS_LABEL: Record = { + ok: '[OK] ', + warn: '[WARN]', + fail: '[FAIL]', +}; + +function renderDoctor(report: DoctorReport): string { + const nameWidth = Math.max(...report.checks.map(check => check.name.length)); + const lines: string[] = ['TestSprite doctor', '']; + for (const check of report.checks) { + lines.push(` ${STATUS_LABEL[check.status]} ${check.name.padEnd(nameWidth)} ${check.detail}`); + } + lines.push(''); + lines.push( + report.failures === 0 && report.warnings === 0 + ? 'All checks passed.' + : `${report.failures} failure(s), ${report.warnings} warning(s).`, + ); + return lines.join('\n'); +} + +export function createDoctorCommand(deps: DoctorDeps = {}): Command { + const cmd = new Command('doctor') + .description( + 'Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite doctor # run all checks (exit 1 if any fails)\n' + + ' testsprite doctor --output json # machine-readable report\n' + + ' testsprite doctor && testsprite test run # gate a command on a healthy setup', + ) + .action(async (_cmdOpts, command: Command) => { + await runDoctor(resolveCommonOptions(command), deps); + }); + + return cmd; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) { + // Match the other commands: a malformed --request-timeout is a validation + // error, not a silently-ignored default. + throw localValidationError( + 'request-timeout', + `must be a positive number of seconds (got "${raw}")`, + ); + } + return Math.round(seconds * 1000); +} + +function makeOutput(mode: OutputMode, deps: DoctorDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index be19095..75fdff0 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -542,6 +542,36 @@ describe('runInit — codex-review hardening', () => { expect(fetchImpl).toHaveBeenCalled(); }); + it('rejects malformed --endpoint-url before setup key verification', async () => { + const { captured, deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + + await expect( + runInit( + makeBaseOpts({ + fromEnv: true, + endpointUrl: 'not-a-url', + noAgent: true, + output: 'json', + }), + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + fetchImpl, + credentialsPath, + isTTY: false, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(captured.stderr.join('\n')).not.toContain('API key rejected'); + }); + it('whoami banner uses --api-key, not a stale TESTSPRITE_API_KEY in env (E2E 2026-06-09)', async () => { const { captured, deps } = makeCapture(); // Key-aware fetch: only the real key gets a 200 + identity; the stale env key 401s. diff --git a/src/commands/init.ts b/src/commands/init.ts index 6a432c2..aff0644 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -11,10 +11,14 @@ */ import { Command } from 'commander'; -import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { + parseRequestTimeoutFlag, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import type { AuthDeps, MeResponse } from './auth.js'; import { runConfigure, runWhoami } from './auth.js'; import type { AgentDeps, AgentFs, InstallResult } from './agent.js'; @@ -36,7 +40,7 @@ const DEFAULT_API_URL = 'https://api.testsprite.com'; */ function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { const env = deps.env ?? process.env; - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); let existing: string | undefined; try { existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; @@ -424,7 +428,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -433,13 +437,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - const SETUP_DESCRIPTION = 'Set up TestSprite: configure your API key and install the TestSprite agent skills for your coding agent'; diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 057296a..f63928c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -199,6 +199,47 @@ describe('runList', () => { }); }); + it('rejects invalid pagination before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-p2-no-creds-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 1.5 }, + { credentialsPath, fetchImpl: fetchImpl as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects invalid dry-run pagination before emitting the dry-run banner', async () => { + const stderr: string[] = []; + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, dryRun: true, pageSize: 1.5 }, + { + credentialsPath: join(mkdtempSync(join(tmpdir(), 'cli-p2-dryrun-')), 'credentials'), + fetchImpl: fetchImpl as unknown as typeof globalThis.fetch, + stderr: line => stderr.push(line), + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(stderr.join('\n')).not.toContain(DRY_RUN_BANNER); + }); + it('rejects pageSize=101 with VALIDATION_ERROR exit 5 (Fix 7 — upper-bound enforced client-side)', async () => { // Previously silently clamped to 100; now rejected so callers get fast feedback. const { credentialsPath } = makeCreds(); @@ -569,6 +610,60 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: ' ', + targetUrl: 'https://example.com', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network - validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Password Guard Project', + targetUrl: 'https://example.com', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -646,6 +741,55 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + name: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); @@ -677,6 +821,33 @@ describe('runUpdate', () => { expect(err).toContain(DRY_RUN_BANNER); }); + it('P7 — dry-run with --password-file does not read the filesystem', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + passwordFile: '/tmp/definitely-not-here-testsprite', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.id).toBe('proj_dry'); + expect(result.updatedFields).toContain('password'); + }); + it('P7 — renders text mode with updatedFields and updatedAt', async () => { const { credentialsPath } = makeCreds(); const updateResponse: CliUpdateProjectResponse = { diff --git a/src/commands/project.ts b/src/commands/project.ts index 41abab1..ef4d5e8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -4,12 +4,13 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { ApiError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { @@ -50,13 +51,13 @@ export async function runList( deps: ProjectDeps = {}, ): Promise> { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const paginationFlags: PaginationFlags = validatePaginationFlags({ pageSize: opts.pageSize, startingToken: opts.startingToken, maxItems: opts.maxItems, }); + const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask // for --max-items, treat that as a "give me one page and the cursor" @@ -141,6 +142,18 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); + // Reject empty / whitespace-only names so a junk record never reaches the + // backend — matches the `requireString` whitespace guard `test create` uses + // (dogfood P1 fix #1). Without this, `--name " "` passes the action + // handler's `if (!name)` check (a non-empty string is truthy) and is sent + // verbatim, creating a blank-named project. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } + // P1-3: client-side length checks matching server limits. if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); @@ -251,6 +264,12 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } @@ -258,27 +277,24 @@ export async function runUpdate( throw localValidationError('--description must be at most 2000 characters'); } - // Resolve password - let password = opts.password; - if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); } - const mutableFields: Record = { - name: opts.name, - targetUrl: opts.targetUrl, - username: opts.username, - password, - description: opts.description, - instruction: opts.instruction, + const passwordSupplied = opts.password !== undefined || opts.passwordFile !== undefined; + const mutableFields: Record = { + name: opts.name !== undefined, + targetUrl: opts.targetUrl !== undefined, + username: opts.username !== undefined, + password: passwordSupplied, + description: opts.description !== undefined, + instruction: opts.instruction !== undefined, }; - const presentFields = Object.entries(mutableFields).filter(([, v]) => v !== undefined); - if (presentFields.length === 0) { + const presentFieldNames = Object.entries(mutableFields) + .filter(([, present]) => present) + .map(([field]) => field); + if (presentFieldNames.length === 0) { throw localValidationError( 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', ); @@ -296,19 +312,36 @@ export async function runUpdate( } const sample: CliUpdateProjectResponse = { id: opts.projectId, - updatedFields: presentFields.map(([k]) => k), + updatedFields: presentFieldNames, updatedAt: '2026-05-16T00:00:00.000Z', }; out.print(sample, data => renderUpdateText(data as CliUpdateProjectResponse)); return sample; } + // Resolve password only on the real path. Dry-run must not touch the + // filesystem, even when --password-file is present. + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderr(`idempotency-key: ${idempotencyKey}`); } - const body = Object.fromEntries(presentFields) as Record; + const bodyFields: Record = { + name: opts.name, + targetUrl: opts.targetUrl, + username: opts.username, + password, + description: opts.description, + instruction: opts.instruction, + }; + const body = Object.fromEntries( + Object.entries(bodyFields).filter(([, v]) => v !== undefined), + ) as Record; const client = makeClient(opts, deps); const updated = await client.patch( `/projects/${encodeURIComponent(opts.projectId)}`, @@ -494,13 +527,9 @@ function resolveCommonOptions(command: Command): CommonOptions { requestTimeout?: string; }; // P2-8: validate --output before allowing silent fallback to 'text'. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('--output must be one of: json, text'); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -509,18 +538,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeClient(opts: CommonOptions, deps: ProjectDeps): HttpClient { return makeHttpClient(opts, { env: deps.env, diff --git a/src/commands/test.artifact.spec.ts b/src/commands/test.artifact.spec.ts index c058079..c0b9283 100644 --- a/src/commands/test.artifact.spec.ts +++ b/src/commands/test.artifact.spec.ts @@ -21,6 +21,7 @@ import { assertOutDirParentExists, createTestArtifactCommand, createTestCommand, + resolveDefaultArtifactDir, runArtifactGet, runFailureGet, } from './test.js'; @@ -322,6 +323,53 @@ describe('runArtifactGet', () => { } }); + it('rejects path-like runId before auth or fetch when default --out is used', async () => { + const fetchImpl = vi.fn(); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: '../../outside', + failedOnly: false, + }, + { fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it.each([ + '.', + '..', + '. ', + '.. ', + '...', + '.. .', + '. .', + '../outside', + '..\\outside', + 'nested/run', + 'nested\\run', + 'bad\0id', + ])('rejects unsafe default artifact runId segment %j', runId => { + expect(() => resolveDefaultArtifactDir(runId, '/repo')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'run-id' }), + }), + ); + }); + + it('keeps the documented default directory for path-safe runIds', () => { + expect(resolveDefaultArtifactDir(SAMPLE_RUN_ID, '/repo')).toBe( + join('/repo', '.testsprite', 'runs', SAMPLE_RUN_ID), + ); + }); + // ---- --failed-only passed through to writeBundle ---- it('passes --failed-only through to writeBundle (steps filtered to failed ± 1)', async () => { diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts new file mode 100644 index 0000000..0c4450e --- /dev/null +++ b/src/commands/test.flaky.spec.ts @@ -0,0 +1,389 @@ +/** + * Unit tests for `test flaky` — the repeat-run flaky-test detector. + * + * All HTTP is mocked via `makeFlakyFetch`. The polling loop's sleep is injected + * through `TestDeps.sleep` to avoid real delays. Each rerun POST returns a + * unique runId; each run GET returns a terminal status scripted per attempt, + * so a test can assert stable / flaky / failing verdicts deterministically. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { CLIError, ApiError } from '../lib/errors.js'; +import type { FlakyReport } from '../lib/flaky.js'; +import type { FetchImpl } from '../lib/http.js'; +import { runFlaky } from './test.js'; + +type FetchInput = Parameters[0]; +type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; + +function urlOf(input: FetchInput): string { + return typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +/** + * Build a fetch that: + * - GET /tests/{id} → the test record ('frontend' | 'backend') + * - POST /tests/{id}/runs/rerun → a queued rerun with runId run_ (n increments) + * - GET /runs/run_ → a terminal run with statuses[k-1] + * + * `notFoundOnTrigger` makes the rerun POST return 404 (no replayable run). + */ +function makeFlakyFetch(opts: { + statuses: RunStatus[]; + testType?: 'frontend' | 'backend'; + notFoundOnTrigger?: boolean; +}): { fetchImpl: FetchImpl; triggerCount: () => number } { + let triggers = 0; + const testType = opts.testType ?? 'frontend'; + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = urlOf(input); + const method = (init.method ?? 'GET').toUpperCase(); + + if (method === 'GET' && /\/tests\/[^/]+$/.test(url.split('?')[0]!)) { + return jsonResponse(200, { + id: 'test_x', + projectId: 'project_abc', + name: 'sample', + type: testType, + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }); + } + + if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.notFoundOnTrigger) { + return jsonResponse(404, { + error: { + code: 'NOT_FOUND', + message: 'no replayable run', + nextAction: 'run it', + requestId: 'req_1', + details: {}, + }, + }); + } + triggers += 1; + return jsonResponse(200, { + runId: `run_${triggers}`, + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + }); + } + + const runMatch = /\/runs\/(run_\d+)/.exec(url); + if (method === 'GET' && runMatch) { + const runId = runMatch[1]!; + const idx = Number(runId.replace('run_', '')) - 1; + const status = opts.statuses[idx] ?? 'passed'; + return jsonResponse(200, { + runId, + testId: 'test_x', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'rerun:prior', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }); + } + + return jsonResponse(404, { + error: { code: 'NOT_FOUND', message: 'unmatched', requestId: 'x', details: {} }, + }); + }) as FetchImpl; + + return { fetchImpl, triggerCount: () => triggers }; +} + +function makeCreds(): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-flaky-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync( + credentialsPath, + `[default]\napi_url = http://localhost:13509\napi_key = sk-user-test\n`, + { + mode: 0o600, + }, + ); + return { credentialsPath }; +} + +const instantSleep = (): Promise => Promise.resolve(); + +function makeDeps(fetchImpl: FetchImpl): { + deps: { + credentialsPath: string; + fetchImpl: FetchImpl; + sleep: () => Promise; + stdout: (l: string) => void; + stderr: (l: string) => void; + }; + stdout: string[]; + stderr: string[]; +} { + const stdout: string[] = []; + const stderr: string[] = []; + const { credentialsPath } = makeCreds(); + return { + deps: { + credentialsPath, + fetchImpl, + sleep: instantSleep, + stdout: (l: string) => stdout.push(l), + stderr: (l: string) => stderr.push(l), + }, + stdout, + stderr, + }; +} + +// --------------------------------------------------------------------------- +// Surface +// --------------------------------------------------------------------------- + +describe('createTestCommand — flaky subcommand exposed', () => { + it('exposes flaky with its flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const flaky = test.commands.find(c => c.name() === 'flaky'); + expect(flaky).toBeDefined(); + const flagNames = flaky!.options.map(o => o.long); + expect(flagNames).toContain('--runs'); + expect(flagNames).toContain('--until-fail'); + expect(flagNames).toContain('--timeout'); + }); +}); + +// --------------------------------------------------------------------------- +// Behavior +// --------------------------------------------------------------------------- + +describe('runFlaky', () => { + it('reports STABLE and exits 0 when every attempt passes', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(triggerCount()).toBe(3); + }); + + it('reports FLAKY and throws exit 1 on a mix of pass/fail', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + expect((err as CLIError).message).toContain('flaky'); + }); + + it('reports FAILING and throws exit 1 when no attempt passes', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['failed', 'failed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).message).toContain('failing'); + }); + + it('--until-fail stops at the first non-passing attempt', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'failed', 'passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 5, + untilFail: true, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + // Stopped after attempt 2 (the failure) — only 2 triggers fired. + expect(triggerCount()).toBe(2); + expect(err).toBeInstanceOf(CLIError); + }); + + it('prints a backend credit advisory to stderr', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'passed'], testType: 'backend' }); + const { deps, stderr } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ); + expect(stderr.some(l => l.includes('backend test') && l.includes('credits'))).toBe(true); + }); + + it('emits a machine-readable JSON stability report', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps, stdout } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch(() => undefined); // swallow the exit-1 throw; we only assert stdout + const parsed = JSON.parse(stdout.join('\n')) as FlakyReport; + expect(parsed.testId).toBe('test_x'); + expect(parsed.runs).toBe(3); + expect(parsed.passed).toBe(2); + expect(parsed.verdict).toBe('flaky'); + expect(parsed.failures).toHaveLength(1); + expect(parsed.failures[0]!.failureKind).toBe('assertion'); + }); + + it('throws exit 4 when the test has no replayable run', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [], notFoundOnTrigger: true }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + }); + + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 0, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); + + it('rejects --runs above the cap (11) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 11, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..a6f68d3 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -280,6 +280,126 @@ describe('runTestRerun — validation', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('exit 5 (VALIDATION_ERROR) when explicit test IDs are combined with --all', async () => { + // The --all branch resolves the FULL project test set and overwrites the + // listed ids, so 'rerun test_abc --all' would silently rerun the ENTIRE + // project instead of test_abc — burning rerun/auto-heal credits. The + // guard throws BEFORE any network/dispatch. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all guard.) + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_abc'], + all: true, + projectId: 'proj_1', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'test-ids' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) when --status is passed WITHOUT --all', async () => { + // --status is an --all-only narrowing filter. With explicit ids it was + // silently ignored (both tests dispatched, filter dropped) — same + // failure mode as the --filter guard above. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a', 'test_b'], + all: false, + statusFilter: 'failed', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'status' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) for an INVALID --status value without --all (was silently accepted)', async () => { + // Before the guard, an invalid --status token without --all was never + // even validated: 'rerun test_a --status notastatus' exited 0 while the + // same flag on delete-batch exits 5. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + statusFilter: 'notastatus', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 (VALIDATION_ERROR) when --skip-terminal is passed WITHOUT --all', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + skipTerminal: true, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'skip-terminal' }), + }); + }); + it('exit 5 when --all without --project', async () => { const creds = makeCreds(); try { @@ -1637,6 +1757,74 @@ describe('--idempotency-key passthrough', () => { expect(receivedKey).toBe('my-custom-key-abc'); }); + + it('emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + + it('does NOT emit an idempotency-key line in default text mode', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.includes('idempotency-key:'))).toBe(false); + }); }); // --------------------------------------------------------------------------- @@ -4630,3 +4818,78 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// TimeoutError on single FE rerun --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }); +}); diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index e3fb963..a2d2018 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -169,6 +169,24 @@ describe('parseDuration', () => { it('case-insensitive day suffix', () => { expect(parseDuration('7D', NOW)).toBe('2026-05-27T12:00:00.000Z'); }); + + it('overflow hours throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999h', NOW)).toThrow(); + try { + parseDuration('99999999999h', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); + + it('overflow days throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999d', NOW)).toThrow(); + try { + parseDuration('99999999999d', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); }); // --------------------------------------------------------------------------- @@ -535,6 +553,32 @@ describe('runResultHistory — pagination', () => { expect(capturedUrl).toContain('pageSize=5'); }); + + it('rejects fractional --page-size before making a request', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('should not be called'); + }); + + await expect( + runResultHistory( + { + output: 'json', + testId: 'test_abc', + pageSize: 1.5, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'page-size' }), + }); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7e2b55e..c0ff083 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -1935,6 +1935,74 @@ describe('runTestRun --wait: Fix 3 — RequestTimeoutError writes partial JSON t }); }); +// --------------------------------------------------------------------------- +// TimeoutError on --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + let dateCallCount = 0; + let fetchCallCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (++dateCallCount > 6 ? base + 2000 : base); + + try { + const fetchImpl: typeof globalThis.fetch = async () => { + ++fetchCallCount; + if (fetchCallCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + const runningRun: RunResponse = { ...makePassedRun(), status: 'running' }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // Fix 5 — B2(c): --timeout hint fires on default, not on explicit timeout // --------------------------------------------------------------------------- @@ -2487,6 +2555,61 @@ describe('runTestRunAll — batch fresh run', () => { expect(payload.accepted.every(r => r.status === 'passed')).toBe(true); }); + it('run --all --wait: does not start a fresh poll for a queued run after the shared deadline expired', async () => { + const { credentialsPath } = makeCreds(); + const baseNow = new Date('2026-06-09T10:00:00.000Z').getTime(); + let nowMs = baseNow; + const runFetches: string[] = []; + const stdoutLines: string[] = []; + let caughtError: unknown; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => nowMs); + + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + runFetches.push(runId); + if (runId === 'run_fresh_01') { + nowMs = baseNow + 2000; + return { body: makePassedRun(runId, 'test_be_01') }; + } + return { body: makePassedRun(runId, 'test_be_02') }; + }); + + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 1, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + } catch (err) { + caughtError = err; + } finally { + nowSpy.mockRestore(); + } + + const payload = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + expect(runFetches).toEqual(['run_fresh_01']); + expect(payload.accepted.find(r => r.runId === 'run_fresh_02')?.status).toBe('timeout'); + expect((caughtError as { exitCode?: number } | undefined)?.exitCode).toBe(7); + }); + it('--wait with a failed run → exit 1', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch((url, init) => { @@ -3470,6 +3593,39 @@ describe('dashboardUrl on run completion', () => { ); }); + it('run --all: emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + it('run --all --wait (prod endpoint): summary items carry dashboardUrl + stderr Dashboard line', async () => { const { credentialsPath } = makeCreds('sk-user-test', PROD_API); const batchResp: BatchRunFreshResponse = { @@ -3536,3 +3692,59 @@ describe('dashboardUrl on run completion', () => { ).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- + +describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const { credentialsPath } = makeCreds(); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: batchResp }; + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_all'); + } + return errorBody('NOT_FOUND'); + }); + const stdoutLines: string[] = []; + + const err = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_fresh_01', 'run_fresh_02']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 4d205e0..6ae831c 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -27,13 +27,17 @@ import { runCreateBatch, runCreateFromPlan, runDelete, + runDiff, runFailureGet, runFailureSummary, runGet, + runLint, runList, runPlanPut, runResult, + runScaffold, runSteps, + runTestWaitMany, runUpdate, } from './test.js'; @@ -121,13 +125,17 @@ describe('createTestCommand — surface', () => { 'create-batch', 'delete', 'delete-batch', + 'diff', 'failure', + 'flaky', 'get', + 'lint', 'list', 'plan', 'rerun', 'result', 'run', + 'scaffold', 'steps', 'update', 'wait', @@ -475,6 +483,34 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', details: { field: 'page-size' } }); }); + it('rejects invalid --status before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-list-status-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'notastatus', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => undefined, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'status' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('forwards a server-side VALIDATION_ERROR envelope as ApiError exit 5', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -1522,6 +1558,31 @@ describe('runCodeGet', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('--out rejects an existing directory path with VALIDATION_ERROR (exit 5) before any network I/O', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-dir-')); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: dir, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchCalls).toBe(0); + expect(readdirSync(dir)).toEqual([]); + }); + // Regression: a parent dir that doesn't exist used to surface as exit 1 // (TRANSPORT_ERROR) — `createWriteStream` opens lazily and ENOENT fires // mid-write. Synchronous parent stat keeps every `--out` user-input @@ -1591,22 +1652,66 @@ describe('runCodeGet', () => { expect(leftovers).toEqual([]); }); - // Regression: the "no code generated yet" branch writes nothing but - // previously still closed (and thus truncated) the opened file. + it('--out (text mode) rejects empty inline code with VALIDATION_ERROR and leaves no artifact', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-empty-out-')); + const target = join(dir, 'empty.ts'); + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: target, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); + expect(existsSync(target)).toBe(false); + }); + + // Regression: empty inline code with --out must reject (exit 5) without + // truncating or replacing a pre-existing destination file. it('--out: "no code generated yet" leaves a pre-existing file untouched', async () => { const { credentialsPath } = makeCreds(); const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-empty-')); const target = join(dir, 'existing.ts'); writeFileSync(target, 'PRE-EXISTING CONTENT', 'utf8'); const fetchImpl = makeFetch(() => ({ body: { ...TEST_CODE_INLINE, code: '' } })); - await runCodeGet( - { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, - { credentialsPath, fetchImpl, stderr: () => undefined }, - ); + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, + { credentialsPath, fetchImpl, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); expect(readFileSync(target, 'utf-8')).toBe('PRE-EXISTING CONTENT'); const leftovers = readdirSync(dir).filter(f => f !== 'existing.ts'); expect(leftovers).toEqual([]); }); + + it('text mode without --out still hints on stderr when inline code is empty', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const stderr: string[] = []; + const got = await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stderr: line => stderr.push(line) }, + ); + expect(got.code).toBe(''); + expect(stderr.join('\n')).toContain('no code generated yet'); + }); }); describe('runCodePut', () => { @@ -1661,6 +1766,30 @@ describe('runCodePut', () => { expect(sent.headers.get('content-type')).toBe('application/json'); }); + it('strips a UTF-8 BOM from --code-file before uploading (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-bom-')); + const codeFile = join(dir, 'updated.py'); + writeFileSync(codeFile, '\uFEFF' + 'updated body', 'utf8'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ code: 'updated body' }); + }); + it('forwards --language in the body when set', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('print("hi")'); @@ -2198,6 +2327,94 @@ describe('runCodePut', () => { }); }); +describe('runScaffold', () => { + it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> }; + expect(plan.type).toBe('frontend'); + // Placeholder project id when TESTSPRITE_PROJECT_ID is unset. + expect(plan.projectId).toContain('testsprite project list'); + expect(plan.planSteps.length).toBeGreaterThanOrEqual(2); + // Every emitted step type must come from the real enum (no drift). + for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type); + // At least one assertion step so the scaffold is a meaningful test. + expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true); + // stdout body parses back to the same plan (`> plan.json` works). + expect(JSON.parse(out.join('\n'))).toEqual(plan); + }); + + it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => { + const result = await runScaffold( + { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false }, + { + stdout: () => undefined, + stderr: () => undefined, + env: { TESTSPRITE_PROJECT_ID: 'project_env' }, + }, + ); + expect((result as { projectId: string }).projectId).toBe('project_env'); + }); + + it('backend scaffold defines a requests test with a status assertion AND calls it', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const code = (result as { code: string }).code; + expect(code).toContain('import requests'); + expect(code).toContain('assert response.status_code == 200'); + // The onboarding rule: the function must be CALLED, not just defined. + expect(code).toContain('\ntest_health_endpoint()'); + expect(out.join('\n')).toContain('import requests'); + }); + + it('--out writes the file and refuses to overwrite without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-')); + const target = join(dir, 'plan.json'); + const opts = { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: target, + force: false, + } as const; + const deps = { stdout: () => undefined, stderr: () => undefined, env: {} }; + await runScaffold({ ...opts }, deps); + const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string }; + expect(written.type).toBe('frontend'); + // Second run without --force must not clobber the (possibly edited) file. + await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + // --force overwrites. + await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined(); + }); + + it('--out pointing at an existing path (here a directory) rejects without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-')); + await expect( + runScaffold( + { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: dir, + force: false, + }, + { stdout: () => undefined, stderr: () => undefined, env: {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); @@ -2482,6 +2699,65 @@ describe('runSteps', () => { expect(step2.outcomeContributesToFailure).toBe(true); }); + it('--run-id carries the per-step error text and stepType through to JSON (no silent drop)', async () => { + // Regression lock for the "steps discard RunStepDto.error" gap: the wire + // already returns the failure text in the same response; it must survive + // the CliTestStep mapping instead of forcing an artifact-bundle download. + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Expected heading "Order confirmed" to be visible, got hidden', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + const passing = page.items.find(s => s.stepIndex === 1)!; + const failing = page.items.find(s => s.stepIndex === 2)!; + expect(failing.error).toBe('Expected heading "Order confirmed" to be visible, got hidden'); + expect(failing.stepType).toBe('assertion'); + expect(passing.error).toBeNull(); + expect(passing.stepType).toBe('action'); + }); + + it('--run-id text mode prints an indented error: sub-line under the failed row only', async () => { + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Locator resolved to hidden element\n at assert heading', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const out: string[] = []; + await runSteps( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + // Newlines in the wire error collapse to one displayable line. + expect(block).toContain('error: Locator resolved to hidden element at assert heading'); + // Exactly one sub-line: the passing step must not grow one. + expect(block.match(/error: /g)).toHaveLength(1); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the @@ -2653,6 +2929,364 @@ describe('runSteps', () => { }); }); +describe('runDiff', () => { + const baseRun = { + testId: 'test_fe', + projectId: 'project_alice', + userId: 'u1', + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + targetUrl: 'https://example.com', + createdFrom: null, + error: null, + videoUrl: null, + stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 }, + }; + const makeStep = (index: string, status: string, error: string | null = null) => ({ + stepIndex: index, + type: 'action', + action: `step ${index}`, + status, + description: `Step ${index}`, + error, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:05.000Z', + }); + const RUN_GREEN = { + ...baseRun, + runId: 'run_green', + status: 'passed', + codeVersion: 'v1', + failedStepIndex: null, + failureKind: null, + steps: [makeStep('0001', 'passed'), makeStep('0002', 'passed')], + }; + const RUN_RED = { + ...baseRun, + runId: 'run_red', + status: 'failed', + codeVersion: 'v2', + failedStepIndex: 2, + failureKind: 'assertion', + steps: [makeStep('0001', 'passed'), makeStep('0002', 'failed', 'heading not visible')], + }; + const fetchForRuns = () => + makeFetch(url => ({ body: url.includes('run_green') ? RUN_GREEN : RUN_RED })); + + it('reports the verdict flip, the changed step with its error, and code drift, then exits 1', async () => { + const { credentialsPath } = makeCreds(); + const out: string[] = []; + const rejection = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl: fetchForRuns(), stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const printed = JSON.parse(out.join('')) as { + verdictChanged: boolean; + codeVersionChanged: boolean; + crossTest: boolean; + changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string; errorB?: string }>; + }; + expect(printed.verdictChanged).toBe(true); + expect(printed.codeVersionChanged).toBe(true); + expect(printed.crossTest).toBe(false); + expect(printed.changedSteps).toHaveLength(1); + expect(printed.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + errorB: 'heading not visible', + }); + }); + + it('identical verdicts resolve with exit 0 and no step changes', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RUN_GREEN })); + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_green' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(diff.verdictChanged).toBe(false); + expect(diff.changedSteps).toHaveLength(0); + }); + + it('warns (not fails) when the runs belong to different tests', async () => { + const { credentialsPath } = makeCreds(); + const OTHER = { ...RUN_GREEN, runId: 'run_red', testId: 'test_other' }; + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_green') ? RUN_GREEN : OTHER, + })); + const errs: string[] = []; + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + ); + expect(diff.crossTest).toBe(true); + expect(errs.join('\n')).toContain('different tests'); + }); + + it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => { + // Dry-run must not require credentials or hit the network — it returns a + // canned CliRunDiff so `--dry-run` shows the shape offline. + const diff = await runDiff( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runA: 'run_aaa', + runB: 'run_bbb', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(diff.runA.runId).toBe('run_aaa'); + expect(diff.runB.runId).toBe('run_bbb'); + expect(diff.verdictChanged).toBe(true); + expect(diff.changedSteps).toHaveLength(1); + expect(diff.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + }); + }); +}); + +describe('runLint', () => { + const VALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Checkout works', + planSteps: [ + { type: 'action', description: 'Open the cart' }, + { type: 'assertion', description: 'Assert the total is visible' }, + ], + }); + const INVALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Broken', + planSteps: [{ type: 'hover', description: 'Bad step type' }], + }); + + it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-')); + writeFileSync(join(dir, 'a-valid.json'), VALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'b-invalid.json'), INVALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'c-notjson.json'), '{oops', 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + valid: number; + issues: Array<{ file: string; field: string }>; + }; + // All three files were checked (no first-error-fatal bailout). + expect(report.checked).toBe(3); + expect(report.valid).toBe(1); + expect(report.issues.map(issue => issue.file).sort()).toEqual([ + 'b-invalid.json', + 'c-notjson.json', + ]); + }); + + it('an all-valid directory resolves with exit 0 and no network/credentials needed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-ok-')); + writeFileSync(join(dir, 'a.json'), VALID_PLAN, 'utf8'); + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: () => undefined }, + ); + expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] }); + }); + + it('a JSONL file reports each bad line with its PHYSICAL line number (blank lines skipped, not renumbered)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-')); + const file = join(dir, 'plans.jsonl'); + // A blank separator line sits between entries: line 1 valid, line 2 blank, + // line 3 bad JSON, line 4 invalid plan. Reported numbers must be 3 and 4. + writeFileSync(file, `${VALID_PLAN}\n\nnot json at all\n${INVALID_PLAN}\n`, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, plans: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { checked: number; issues: Array<{ file: string }> }; + expect(report.checked).toBe(3); + expect(report.issues.some(issue => issue.file.endsWith(':3'))).toBe(true); + expect(report.issues.some(issue => issue.file.endsWith(':4'))).toBe(true); + // The blank line itself is not an entry and never reports. + expect(report.issues.some(issue => issue.file.endsWith(':2'))).toBe(false); + }); + + it('requires exactly one input source', async () => { + await expect( + runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + +describe('runTestWaitMany', () => { + const terminalRun = (runId: string, status: string) => ({ + runId, + testId: `test_of_${runId}`, + projectId: 'project_alice', + userId: 'u1', + status, + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }); + + it('polls every run, keeps input order, and exits 1 when one finished non-passed', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_bad') + ? terminalRun('run_bad', 'failed') + : terminalRun('run_ok', 'passed'), + })); + const out: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_bad'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + summary: { passed: number; failed: number }; + }; + expect(payload.results.map(row => row.runId)).toEqual(['run_ok', 'run_bad']); + expect(payload.summary).toMatchObject({ passed: 1, failed: 1 }); + }); + + it('a member whose poll errors is captured as error: and the others survive (exit 7)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('run_gone')) { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'no such run', + nextAction: 'check the id', + requestId: 'req_x', + details: {}, + }, + }, + }; + } + return { body: terminalRun('run_ok', 'passed') }; + }); + const out: string[] = []; + const errs: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_gone'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errs.push(line), + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + }; + // The failing member did not abort the pool: the passed verdict survived. + expect(payload.results[0]).toMatchObject({ runId: 'run_ok', status: 'passed' }); + expect(payload.results[1]!.status).toBe('error:NOT_FOUND'); + // The re-attach hint names the errored member (resumable) but NOT the + // already-terminal passed one. + const hint = errs.find(line => line.includes('Re-attach with:')); + expect(hint).toContain('run_gone'); + expect(hint).not.toContain('run_ok'); + }); + + it('members dequeued after the shared deadline are not granted extra poll time', async () => { + const { credentialsPath } = makeCreds(); + let fetches = 0; + const fetchImpl = makeFetch(() => { + fetches += 1; + return { body: terminalRun('run_any', 'passed') }; + }); + // timeoutSeconds 0: the shared deadline is already in the past when the + // pool starts, so every member must resolve to timeout WITHOUT polling + // (previously each dequeued member was granted a fresh 1s minimum). + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b', 'run_c'], + timeoutSeconds: 0, + maxConcurrency: 1, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + expect(fetches).toBe(0); + }); + + it('an auth error escalates the exit code to 3', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 401, + body: { + error: { + code: 'AUTH_INVALID', + message: 'bad key', + nextAction: 'run setup', + requestId: 'req_y', + details: {}, + }, + }, + })); + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 3 }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); @@ -5709,6 +6343,31 @@ describe('runUpdate', () => { expect(called).toBe(0); }); + it('rejects a whitespace-only --name before sending (parity with test create)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: ' ', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'name' }), + }); + expect(called).toBe(0); + }); + it('renders text mode with one line per updated field', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..af874e0 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,4 +1,11 @@ -import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; +import { + createWriteStream, + existsSync, + readFileSync, + readdirSync, + statSync, + type WriteStream, +} from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; @@ -6,6 +13,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { @@ -17,7 +25,16 @@ import { writeBundle, type WriteBundleResult, } from '../lib/bundle.js'; -import { findSample } from '../lib/dry-run/samples.js'; +import { findSample, sampleJUnitReportXml } from '../lib/dry-run/samples.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitReportFormat, + parseJUnitReportFormat, + type JUnitTestResult, +} from '../lib/junit-report.js'; import { ApiError, CLIError, @@ -34,7 +51,7 @@ import { import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { fetchSinglePage, paginate, @@ -65,6 +82,14 @@ import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; import { loadConfig } from '../lib/config.js'; +import { + flakyExitCode, + renderFlakyText, + summarizeFlaky, + type FlakyAttempt, + type FlakyOutcome, + type FlakyReport, +} from '../lib/flaky.js'; /** * `details` debug block per the CLI OpenAPI `Test` schema @@ -176,6 +201,19 @@ export interface CliTestStep { * that don't emit the field still type-check. */ outcomeContributesToFailure?: boolean | null; + /** + * Per-step failure text, carried from `RunStepDto.error` on the run-scoped + * endpoint (`GET /runs/{id}?includeSteps=true`). Only present on `--run-id` + * responses; the cumulative `/tests/{id}/steps` rows do not carry it, so the + * field stays optional (additive, non-breaking for existing consumers). + */ + error?: string | null; + /** + * Wire step kind from `RunStepDto.type` on the run-scoped endpoint. Same + * availability rules as `error`. Named `stepType` to avoid colliding with + * the free-form `action` label above. + */ + stepType?: 'action' | 'assertion'; } /** @@ -451,6 +489,12 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

= { projectId: opts.projectId, type: opts.type, @@ -911,7 +949,7 @@ function readCodeFileGuarded(path: string): string { function readCodeFile(path: string): string { try { - return readFileSync(resolveAbsolute(path), 'utf8'); + return stripBom(readFileSync(resolveAbsolute(path), 'utf8')); } catch (err) { const code = (err as NodeJS.ErrnoException).code; if (code === 'ENOENT') { @@ -1242,6 +1280,12 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); requireNonEmpty('test-id', opts.testId); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError( + 'name', + 'must be a non-empty string (whitespace-only is not allowed)', + ); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('name', 'must be at most 200 characters'); } @@ -3292,7 +3336,7 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro return code; } - const fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; + let fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; const out = fileSink ? makeFileOutput(opts.output, fileSink) : makeOutput(opts.output, deps); const client = makeClient(opts, deps); @@ -3317,9 +3361,20 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro } else if (code.code === '' || code.code === null) { // P2-10: draft test with no code yet — empty body would produce // silent empty stdout. Print a friendly hint to stderr instead so - // the operator knows what happened, and keep exit 0. Nothing was - // written, so the temp file is discarded below without touching - // a pre-existing `--out` file. + // the operator knows what happened, and keep exit 0 when no `--out`. + // + // With `--out`, refuse to leave a zero-byte artifact behind: agents + // and scripts that check file size would otherwise treat exit 0 as + // a successful download. Discard the temp sink without touching a + // pre-existing destination file. + if (fileSink) { + await abortOutputFile(fileSink); + fileSink = null; + throw localValidationError( + 'out', + 'test has no generated code yet — run the test first (refusing to write an empty --out file)', + ); + } const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); stderrFn('(no code generated yet — run the test first)'); } else { @@ -3615,7 +3670,449 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte // non-contributors. (Per the CliTestStep contract: null ≠ false.) outcomeContributesToFailure: run.failedStepIndex === null ? null : numericIndex === run.failedStepIndex, + // Carry the per-step failure text and the wire step kind through instead + // of dropping them: the agent asking "why did this step fail?" would + // otherwise have to download the whole artifact bundle to read a string + // this very response already contained. + error: step.error, + stepType: step.type, + }; +} + +export interface DiffOptions extends CommonOptions { + runA: string; + runB: string; +} + +/** One step whose status flipped between the two compared runs. */ +export interface CliDiffStep { + stepIndex: number; + statusA: string; + statusB: string; + /** First divergent failing side's error text, when the wire carried one. */ + errorA?: string | null; + errorB?: string | null; +} + +export interface CliRunDiff { + runA: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + runB: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + verdictChanged: boolean; + failedStepIndexChanged: boolean; + failureKindChanged: boolean; + codeVersionChanged: boolean; + /** True when the two runs belong to DIFFERENT tests (deltas may be meaningless). */ + crossTest: boolean; + changedSteps: CliDiffStep[]; +} + +/** + * `test diff ` (issue #124): isolate what regressed between two + * runs, the first question when CI goes red ("what changed since the last + * green run?"). Pure client-side composition of the existing per-run read + * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids, + * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts + * match, exit 1 when they differ, so the command is CI-scriptable. + */ +export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample: CliRunDiff = { + runA: { + runId: opts.runA, + testId: 'test_dryrun', + status: 'passed', + failureKind: null, + failedStepIndex: null, + codeVersion: 'v1', + }, + runB: { + runId: opts.runB, + testId: 'test_dryrun', + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 2, + codeVersion: 'v1', + }, + verdictChanged: true, + failedStepIndexChanged: true, + failureKindChanged: true, + codeVersionChanged: false, + crossTest: false, + changedSteps: [{ stepIndex: 2, statusA: 'passed', statusB: 'failed' }], + }; + out.print(sample, () => renderRunDiffText(sample)); + return sample; + } + + const client = makeClient(opts, deps); + const [runA, runB] = await Promise.all([ + client.getRun(opts.runA, { includeSteps: true }), + client.getRun(opts.runB, { includeSteps: true }), + ]); + + const crossTest = runA.testId !== runB.testId; + if (crossTest) { + stderrFn( + `⚠ the two runs belong to different tests (${runA.testId} vs ${runB.testId}) — step deltas may be meaningless`, + ); + } + + const stepsByIndex = ( + run: RunResponse, + ): Map => { + const map = new Map(); + for (const step of run.steps ?? []) { + const index = parseInt(step.stepIndex, 10); + if (Number.isInteger(index)) + map.set(index, { status: step.status ?? 'unknown', error: step.error }); + } + return map; + }; + const stepsA = stepsByIndex(runA); + const stepsB = stepsByIndex(runB); + const allIndexes = [...new Set([...stepsA.keys(), ...stepsB.keys()])].sort( + (left, right) => left - right, + ); + const changedSteps: CliDiffStep[] = []; + for (const index of allIndexes) { + const sideA = stepsA.get(index); + const sideB = stepsB.get(index); + const statusA = sideA?.status ?? 'absent'; + const statusB = sideB?.status ?? 'absent'; + if (statusA === statusB) continue; + changedSteps.push({ + stepIndex: index, + statusA, + statusB, + ...(sideA?.error ? { errorA: sideA.error } : {}), + ...(sideB?.error ? { errorB: sideB.error } : {}), + }); + } + + const summarize = (run: RunResponse) => ({ + runId: run.runId, + testId: run.testId, + status: run.status, + failureKind: run.failureKind ?? null, + failedStepIndex: run.failedStepIndex, + codeVersion: run.codeVersion ?? null, + }); + const diff: CliRunDiff = { + runA: summarize(runA), + runB: summarize(runB), + verdictChanged: runA.status !== runB.status, + failedStepIndexChanged: runA.failedStepIndex !== runB.failedStepIndex, + failureKindChanged: (runA.failureKind ?? null) !== (runB.failureKind ?? null), + codeVersionChanged: (runA.codeVersion ?? null) !== (runB.codeVersion ?? null), + crossTest, + changedSteps, }; + out.print(diff, () => renderRunDiffText(diff)); + + if (diff.verdictChanged) { + // Result already printed; the typed exit makes `test diff` a CI gate. + throw new CLIError( + `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`, + 1, + ); + } + return diff; +} + +function renderRunDiffText(diff: CliRunDiff): string { + const lines: string[] = []; + lines.push(`runA: ${diff.runA.runId} ${diff.runA.status} (test ${diff.runA.testId})`); + lines.push(`runB: ${diff.runB.runId} ${diff.runB.status} (test ${diff.runB.testId})`); + lines.push( + `verdict: ${diff.verdictChanged ? `${diff.runA.status} -> ${diff.runB.status}` : `unchanged (${diff.runA.status})`}`, + ); + if (diff.failureKindChanged) + lines.push( + `failureKind: ${diff.runA.failureKind ?? '(none)'} -> ${diff.runB.failureKind ?? '(none)'}`, + ); + if (diff.failedStepIndexChanged) + lines.push( + `failedStepIndex: ${diff.runA.failedStepIndex ?? '(none)'} -> ${diff.runB.failedStepIndex ?? '(none)'}`, + ); + lines.push( + `codeVersion: ${diff.codeVersionChanged ? `${diff.runA.codeVersion ?? '(none)'} -> ${diff.runB.codeVersion ?? '(none)'} (code drift)` : 'unchanged'}`, + ); + if (diff.changedSteps.length === 0) { + lines.push('steps: no per-step status changes'); + } else { + lines.push(`steps changed: ${diff.changedSteps.length}`); + for (const step of diff.changedSteps) { + lines.push(` #${step.stepIndex} ${step.statusA} -> ${step.statusB}`); + if (step.errorB) lines.push(` error(B): ${step.errorB.replace(/\s+/g, ' ').trim()}`); + else if (step.errorA) + lines.push(` error(A): ${step.errorA.replace(/\s+/g, ' ').trim()}`); + } + } + return lines.join('\n'); +} + +export interface LintOptions extends CommonOptions { + planFrom?: string; + planFromDir?: string; + plans?: string; + steps?: string; +} + +export interface CliLintIssue { + file: string; + field: string; + reason: string; +} + +export interface CliLintReport { + checked: number; + valid: number; + issues: CliLintIssue[]; +} + +/** + * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the + * SAME validators the create paths run, but collecting EVERY problem instead + * of dying on the first one, and without any network write. The create-batch + * reader is first-error-fatal and only reachable through a command that POSTs, + * so authoring a 12-plan directory meant one error per paid round-trip. Zero + * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so + * it drops into a pre-commit hook or CI step before `create-batch`. + */ +export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const sources = [opts.planFrom, opts.planFromDir, opts.plans, opts.steps].filter( + source => source !== undefined, + ); + if (sources.length !== 1) { + throw localValidationError( + 'plan-from', + 'exactly one of --plan-from, --plan-from-dir, --plans, or --steps is required', + ); + } + + const issues: CliLintIssue[] = []; + let checked = 0; + // Run one existing validator, converting its typed throw into a report row + // (same envelopes, so `details.field` pointers like planSteps[2].type + // survive verbatim). + const collect = (file: string, validate: () => void): void => { + checked += 1; + try { + validate(); + } catch (err) { + if (err instanceof ApiError) { + issues.push({ + file, + field: String(err.getDetail('field') ?? '(file)'), + reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), + }); + } else { + issues.push({ + file, + field: '(file)', + reason: err instanceof Error ? err.message : String(err), + }); + } + } + }; + + if (opts.planFrom !== undefined) { + const planFrom = opts.planFrom; + collect(planFrom, () => void readPlanFromGuarded(planFrom)); + } else if (opts.steps !== undefined) { + const steps = opts.steps; + collect(steps, () => void readPlanStepsFileGuarded(steps)); + } else if (opts.planFromDir !== undefined) { + const dir = resolveAbsolute(opts.planFromDir); + let entries: string[]; + try { + entries = readdirSync(dir) + .filter(name => name.endsWith('.json')) + .sort(); + } catch { + throw localValidationError('plan-from-dir', `cannot read directory: ${dir}`); + } + if (entries.length === 0) { + throw localValidationError('plan-from-dir', 'contains no *.json plan files'); + } + for (const entry of entries) { + collect(entry, () => void readPlanFromGuarded(join(dir, entry))); + } + } else if (opts.plans !== undefined) { + // JSONL: validate PER LINE so every bad line reports (the create path's + // reader stays throw-on-first; this is the collecting counterpart). + const absolute = resolveAbsolute(opts.plans); + let content: string; + try { + content = readFileSync(absolute, 'utf8'); + } catch { + throw localValidationError('plans', `cannot read file: ${absolute}`); + } + // Index lines BEFORE dropping blanks so every reported `file:N` points at + // the PHYSICAL line in the file (a blank separator line must not shift all + // subsequent line numbers). + const numberedLines = content + .split('\n') + .map((rawLine, physicalIndex) => ({ line: rawLine.trim(), lineNo: physicalIndex + 1 })) + .filter(entry => entry.line.length > 0); + if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines'); + for (const { line, lineNo } of numberedLines) { + collect(`${opts.plans}:${lineNo}`, () => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + throw localValidationError( + 'plans', + `line ${lineNo} is not valid JSON`, + undefined, + 'field', + ); + } + assertPlanShape(parsed, { specIndex: lineNo - 1 }); + }); + } + } + + const filesWithIssues = new Set(issues.map(issue => issue.file)).size; + const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues }; + out.print(report, () => + [ + ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, + ].join('\n'), + ); + if (issues.length > 0) { + throw new CLIError(`lint: ${issues.length} problem(s) across ${report.checked} file(s)`, 5); + } + return report; +} + +/** Flag options for `test scaffold`. */ +interface ScaffoldFlagOpts { + type?: string; + out?: string; + force?: boolean; +} + +export interface ScaffoldOptions extends CommonOptions { + scaffoldType: 'frontend' | 'backend'; + out?: string; + force: boolean; +} + +/** JSON payload `test scaffold --type backend` prints under --output json. */ +export interface CliBackendScaffold { + type: 'backend'; + language: 'python'; + code: string; +} + +/** + * `test scaffold` — emit a schema-correct starter test definition so a first + * test never starts from hand-copied JSON. Pure-local: no network, no + * credentials, no filesystem reads. The frontend template is a `CliPlanInput` + * (the exact shape `--plan-from` ingests; sourceRef: CliPlanInput / + * PLAN_STEP_TYPES above), so `scaffold | create --plan-from -`-style flows + * validate out of the box. The backend template is the minimal `requests` + * script the onboarding skill mandates: define a test function with a + * concrete status assertion, then CALL it (a defined-but-never-called test + * would pass without asserting anything). + */ +export async function runScaffold( + opts: ScaffoldOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const env = deps.env ?? process.env; + // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's + // environment carries one; otherwise a clearly-marked placeholder the user + // swaps after running `testsprite project list`. + const projectId = + typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0 + ? env.TESTSPRITE_PROJECT_ID + : ''; + + let payload: CliPlanInput | CliBackendScaffold; + let body: string; + if (opts.scaffoldType === 'frontend') { + const plan: CliPlanInput = { + projectId, + type: 'frontend', + name: 'My first frontend test', + description: 'Replace with one sentence describing what this test verifies.', + priority: 'p2', + planSteps: [ + { + type: 'action', + description: 'Navigate to /login and sign in with a seeded test account', + }, + { type: 'action', description: 'Open the first product page and click "Add to cart"' }, + { type: 'assertion', description: 'Assert that the cart badge shows 1 item' }, + ], + }; + payload = plan; + body = `${JSON.stringify(plan, null, 2)}\n`; + } else { + const code = [ + 'import requests', + '', + '# Replace with your API base URL (must be reachable from the internet).', + 'BASE_URL = "https://staging.example.com"', + '', + '', + 'def test_health_endpoint() -> None:', + ' response = requests.get(f"{BASE_URL}/health", timeout=30)', + ' assert response.status_code == 200, f"expected 200, got {response.status_code}"', + '', + '', + '# The test function MUST be called: TestSprite executes this file top to', + '# bottom, so a defined-but-never-called function would pass vacuously.', + 'test_health_endpoint()', + '', + ].join('\n'); + payload = { type: 'backend', language: 'python', code }; + body = code; + } + + if (opts.out !== undefined) { + const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); + // Never clobber silently: scaffolds are starting points the user edits, so + // an accidental re-run must not erase their work. --force opts in. + if (!opts.force && existsSync(resolved)) { + throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); + } + const sink = openOutputFile(opts.out); // reuses the directory/parent guards + const fileOut = makeFileOutput(opts.output, sink); + await fileOut.writeChunk(body); + await closeOutputFile(sink, true); + stderrFn(`Scaffold written to ${resolved}`); + return payload; + } + + // No --out: the scaffold body IS the stdout payload (`> plan.json` works). + out.print(payload, () => body.trimEnd()); + return payload; } export async function runSteps( @@ -3791,12 +4288,20 @@ export function parseDuration(raw: string, now: Date = new Date()): string { const hourMatch = /^(\d+)h$/i.exec(raw); if (hourMatch) { const hours = Number(hourMatch[1]); - return new Date(now.getTime() - hours * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - hours * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~1141552511h'); + } + return result.toISOString(); } const dayMatch = /^(\d+)d$/i.exec(raw); if (dayMatch) { const days = Number(dayMatch[1]); - return new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - days * 24 * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~47564688d'); + } + return result.toISOString(); } // Pass-through: ISO timestamp or epoch value — server validates. return raw; @@ -3836,7 +4341,10 @@ export async function runResultHistory( // configured (codex round-2), matching validatePaginationFlags ordering // in `test list` / `project list`. if (opts.pageSize !== undefined) { - if (!Number.isFinite(opts.pageSize) || opts.pageSize < 1 || opts.pageSize > 100) { + if (!Number.isFinite(opts.pageSize) || !Number.isInteger(opts.pageSize)) { + throw localValidationError('page-size', 'must be an integer between 1 and 100'); + } + if (opts.pageSize < 1 || opts.pageSize > 100) { throw localValidationError('page-size', 'must be between 1 and 100'); } } @@ -3932,6 +4440,14 @@ const RUN_HISTORY_TABLE_COL_WIDTHS = { */ const DESC_COL_MAX = 60; +/** + * Cap, in chars, for the one-line `error:` sub-line under a failed step row + * in `renderStepsText`. Long enough for a full assertion message, short + * enough that a stack-trace blob can't flood the table. Full text is in + * `--output json`. + */ +const ERROR_SUBLINE_MAX = 200; + /** Max chars to show in the TARGETURL sub-line (excess truncated with …). */ const HISTORY_TARGET_URL_MAX = 80; @@ -4281,6 +4797,12 @@ interface RunTestRerunOptions extends CommonOptions { * filters. Client-side only. */ nameFilter?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; } /** @@ -4817,6 +5339,26 @@ export async function runTestRun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${triggerResponse.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, timeoutPartial, opts.createContext, data => { + const p = data as typeof timeoutPartial; + const lines = [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + ]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + return lines.join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md @@ -4904,6 +5446,211 @@ export async function runTestRun( return finalRun; } +/** One row of the `test wait ` multi-run payload. */ +export interface CliMultiWaitResult { + runId: string; + /** Terminal run status, or 'timeout', or 'error:' when the poll failed. */ + status: string; + /** Test the run belongs to, when the poll observed it. */ + testId?: string; +} + +export interface RunTestWaitManyOptions extends CommonOptions { + runIds: string[]; + timeoutSeconds: number; + maxConcurrency: number; +} + +/** + * `test wait ` with two or more ids: attach to N already-dispatched + * runs in ONE invocation. This closes the loop the CLI itself opens: every + * batch/closure timeout prints one `testsprite test wait ` hint PER + * member, which previously meant N sequential blocking invocations. The runs + * are polled concurrently under a bounded pool with ONE shared deadline + * (`--timeout` bounds the whole invocation, not each member), each member's + * poll is total (a transient error on one run never discards the others), and + * the exit code is the worst status across members: auth errors escalate to + * exit 3, any timeout or poll error exits 7, any non-passed terminal exits 1. + * Distinct from a run journal (issue #80): no persistence, just N known ids. + */ +export async function runTestWaitMany( + opts: RunTestWaitManyOptions, + deps: TestDeps = {}, +): Promise<{ results: CliMultiWaitResult[]; summary: Record }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const results: CliMultiWaitResult[] = opts.runIds.map(runId => ({ + runId, + status: 'passed', + })); + const payload = { + results, + summary: { total: results.length, passed: results.length, failed: 0, timedOut: 0, errors: 0 }, + }; + out.print(payload, () => results.map(r => `${r.runId} ${r.status}`).join('\n')); + return payload; + } + + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + // One shared deadline across every member (the whole point of the shared + // pool: `--timeout 600` means the invocation ends within ~600s, not + // 600s x ceil(N/concurrency)). + const deadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + type WaitOutcome = + | { kind: 'result'; run: RunResponse } + | { kind: 'timeout' } + | { kind: 'error'; code: string; exitCode: number }; + + const pollOne = async (runId: string): Promise => { + // A member dequeued AFTER the shared deadline has passed must not be + // granted a fresh minimum poll window (with --max-concurrency 1 that + // would extend the invocation by ~1s per queued run past --timeout). + const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); + if (remainingSeconds <= 0) return { kind: 'timeout' }; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: run => run.testId, + resolveNotBefore: run => run.createdAt, + onResolved: () => undefined, + }); + try { + const run = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); + }, + resolveAlternate, + }); + return { kind: 'result', run }; + } catch (err) { + if (err instanceof TimeoutError) return { kind: 'timeout' }; + if (err instanceof RequestTimeoutError) throw err; + if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; + return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + } + }; + + const outcomes = new Map(); + let inFlight = 0; + let nextIdx = 0; + try { + await new Promise((resolve, reject) => { + const startNext = (): void => { + while (inFlight < opts.maxConcurrency && nextIdx < opts.runIds.length) { + const runId = opts.runIds[nextIdx++]!; + inFlight++; + pollOne(runId) + .then(outcome => { + outcomes.set(runId, outcome); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= opts.runIds.length) resolve(); + }) + // pollOne is total except for RequestTimeoutError (handled below). + .catch(reject); + } + }; + startNext(); + if (opts.runIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + if (fanOutErr instanceof RequestTimeoutError) { + // Same contract as the batch pollers: leave stdout parseable before + // exiting 7. Members that already settled keep their real status; only + // the still-unfinished ids are marked running and named in the hint + // (re-attaching to an already-terminal run would be a wasted command). + ticker.finalize('Multi-run wait — request timed out'); + const partial = { + results: opts.runIds.map((runId): CliMultiWaitResult => { + const outcome = outcomes.get(runId); + if (outcome === undefined) return { runId, status: 'running' }; + if (outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }), + summary: { total: opts.runIds.length }, + }; + out.print(partial, () => partial.results.map(r => `${r.runId} ${r.status}`).join('\n')); + const unfinished = partial.results + .filter(r => r.status === 'running' || r.status === 'timeout') + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); + } + } + throw fanOutErr; + } + ticker.finalize(); + + const results: CliMultiWaitResult[] = opts.runIds.map(runId => { + const outcome = outcomes.get(runId); + if (outcome === undefined || outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }); + const passed = results.filter(r => r.status === 'passed').length; + const timedOut = results.filter(r => r.status === 'timeout').length; + const errors = results.filter(r => r.status.startsWith('error:')).length; + const failed = results.length - passed - timedOut - errors; + const payload = { + results, + summary: { total: results.length, passed, failed, timedOut, errors }, + }; + out.print(payload, () => + [ + ...results.map(r => `${r.runId} ${r.status}`), + '', + `${passed}/${results.length} passed, ${failed} failed/blocked, ${timedOut} timed out, ${errors} poll errors`, + ].join('\n'), + ); + + // Every member that did not reach a terminal verdict is re-attachable: + // timeouts (still running server-side) and poll errors (e.g. a transient + // transport failure) both belong in the hint; terminal runs do not. + const unfinishedIds = results + .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) + .map(r => r.runId); + if (unfinishedIds.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); + } + + // Worst-status exit: auth escalates (a rejected key fails every member the + // same way), then timeout/poll-error (7, resumable), then plain failure (1). + const authError = [...outcomes.values()].find( + o => + o.kind === 'error' && + (o.code === 'AUTH_REQUIRED' || o.code === 'AUTH_INVALID' || o.code === 'AUTH_FORBIDDEN'), + ); + if (authError !== undefined && authError.kind === 'error') { + throw new CLIError( + `Multi-run wait: authentication failed (${authError.code})`, + authError.exitCode, + ); + } + if (timedOut > 0 || errors > 0) { + throw new CLIError( + `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, + 7, + ); + } + if (failed > 0) { + throw new CLIError(`Multi-run wait: ${failed} run(s) finished non-passed`, 1); + } + return payload; +} + /** * `test wait ` — M3.3 piece-3. * @@ -4977,10 +5724,22 @@ export async function runTestWait( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${opts.runId} — timed out after ${opts.timeoutSeconds}s`); - throw ApiError.fromEnvelope({ - error: { - code: 'UNSUPPORTED', // exit 7 per errors.md - message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: opts.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); + throw ApiError.fromEnvelope({ + error: { + code: 'UNSUPPORTED', // exit 7 per errors.md + message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, nextAction: `Resume polling: testsprite test wait ${opts.runId}`, requestId: 'local', details: { runId: opts.runId, timeoutSeconds: opts.timeoutSeconds }, @@ -5056,6 +5815,32 @@ interface RunTestRunAllOptions extends CommonOptions { maxConcurrency: number; /** Caller-supplied idempotency token; auto-minted if absent. */ idempotencyKey?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; +} + +async function writeBatchJUnitReportIfRequested( + opts: { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + projectId?: string; + }, + results: readonly JUnitTestResult[], +): Promise { + if (opts.report !== 'junit' || opts.reportFile === undefined) return; + const projectId = resolveBatchReportProjectId(opts, results); + const suiteName = opts.reportSuiteName ?? `testsprite:${projectId}`; + const xml = buildJUnitReport({ + suiteName, + classname: projectId, + results, + }); + await writeJUnitReportFile(opts.reportFile, xml); } /** @@ -5064,6 +5849,8 @@ interface RunTestRunAllOptions extends CommonOptions { interface CliBatchRunFreshResult { testId: string; runId: string | undefined; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; status: string; error?: { code: string; message: string; exitCode: number }; /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ @@ -5090,6 +5877,13 @@ export async function runTestRunAll( ) { throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); } + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: true, + }); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const out = makeOutput(opts.output, deps); @@ -5113,6 +5907,12 @@ export async function runTestRunAll( idempotencyKey, ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(opts.projectId, opts.reportSuiteName), + ); + } out.print(batchRunSample ?? envelope); return undefined; } @@ -5136,12 +5936,9 @@ export async function runTestRunAll( }; const idempotencyKey = opts.idempotencyKey ?? `cli-batch-run-fresh-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // Resolve testIds: fetch all BE tests in the project, apply --filter. let testIds: string[] | undefined; @@ -5421,7 +6218,20 @@ export async function runTestRunAll( async function pollFreshAccepted(entry: BatchRunFreshAccepted): Promise { const runId = entry.runId; - const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + const remainingSeconds = Math.ceil(remainingMs / 1000); const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: () => entry.testId, @@ -5442,7 +6252,12 @@ export async function runTestRunAll( }, resolveAlternate, }); - return { testId: entry.testId, runId, status: finalRun.status }; + return { + testId: entry.testId, + runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -5456,6 +6271,22 @@ export async function runTestRunAll( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, // RATE_LIMITED=11, …) instead of flattening every member failure to 1 @@ -5524,6 +6355,7 @@ export async function runTestRunAll( total: pollable.length, }, }; + await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), @@ -5586,6 +6418,8 @@ export async function runTestRunAll( interface CliRerunResult { testId: string; runId: string; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; /** Terminal status, or 'timeout' for per-run deadline exceeded. */ status: string; /** Set when the test is a closure member (not the user's named test). */ @@ -5628,6 +6462,18 @@ export async function runTestRerun( 'provide at least one , or use --all to rerun all tests in the project', ); } + // Explicit ids + --all is ambiguous: the --all branch resolves the FULL + // project test set and overwrites the listed ids, so the user's narrowing + // intent would be silently replaced by a whole-project batch rerun — + // burning rerun/auto-heal credits. Reject early. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all data-loss guard.) + if (opts.all && opts.testIds.length > 0) { + throw localValidationError( + 'test-ids', + 'pass either explicit test IDs or --all, not both — --all reruns every test in the ' + + 'project and would ignore the listed IDs. Drop the IDs, or drop --all.', + ); + } if (opts.all && !opts.projectId) { throw localValidationError( 'project', @@ -5646,6 +6492,24 @@ export async function runTestRerun( 'Remove --filter, or add --all --project .', ); } + // --status and --skip-terminal are --all-only narrowing filters with the + // same silent-ignore failure mode as --filter above: without --all the + // explicit ids get reran unfiltered (and an invalid --status value is + // never even validated). Reject both, mirroring the --filter guard. + if (opts.statusFilter !== undefined && !opts.all) { + throw localValidationError( + 'status', + '--status only applies with --all (it narrows which project tests get reran). ' + + 'Remove --status, or add --all --project .', + ); + } + if (opts.skipTerminal && !opts.all) { + throw localValidationError( + 'skip-terminal', + '--skip-terminal only applies with --all (it narrows which project tests get reran). ' + + 'Remove --skip-terminal, or add --all --project .', + ); + } if ( !Number.isInteger(opts.maxConcurrency) || opts.maxConcurrency < 1 || @@ -5655,6 +6519,13 @@ export async function runTestRerun( } const isSingle = !opts.all && opts.testIds.length === 1; + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: !isSingle, + }); // ------------------------------------------------------------------------- // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) @@ -5694,6 +6565,13 @@ export async function runTestRerun( idempotencyKey, ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + const projectKey = resolveBatchReportProjectId(opts, []); + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(projectKey, opts.reportSuiteName), + ); + } out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); } void client; @@ -5704,12 +6582,9 @@ export async function runTestRerun( // slow rerun trigger / long-poll under load isn't cut at the 120s default. const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); const idempotencyKey = opts.idempotencyKey ?? `cli-rerun-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // ------------------------------------------------------------------------- // Single rerun path @@ -6099,6 +6974,18 @@ export async function runTestRerun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${rerunResp.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', @@ -6263,7 +7150,15 @@ export async function runTestRerun( chunkResponses = []; for (let idx = 0; idx < chunks.length; idx++) { const chunk = chunks[idx]!; - const chunkKey = chunks.length === 1 ? idempotencyKey : `${idempotencyKey}:chunk${idx}`; + // Bound the per-chunk idempotency key to <=256 chars (mirrors the retry + // path). A long base key plus the `:chunkN` suffix could otherwise exceed + // the server cap and be rejected or truncated inconsistently. + const chunkSuffix = chunks.length === 1 ? '' : `:chunk${idx}`; + const chunkBase = + chunkSuffix.length > 0 && idempotencyKey.length + chunkSuffix.length > 256 + ? idempotencyKey.slice(0, 256 - chunkSuffix.length) + : idempotencyKey; + const chunkKey = `${chunkBase}${chunkSuffix}`; const chunkResp = await client.triggerBatchRerun( { source: 'cli', @@ -6594,7 +7489,12 @@ export async function runTestRerun( }, resolveAlternate, }); - return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + return { + testId: entry.testId, + runId: entry.runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -6608,12 +7508,31 @@ export async function runTestRerun( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId: entry.runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { + // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the + // batch exit-code aggregator can escalate auth failures correctly. Mirroring + // the identical fix already applied to runTestRunAll's pollFreshAccepted. return { testId: entry.testId, runId: entry.runId, status: 'error', - error: { code: err.code, message: err.message, exitCode: 1 }, + error: { code: err.code, message: err.message, exitCode: err.exitCode }, }; } throw err; @@ -6682,6 +7601,7 @@ export async function runTestRerun( total: accepted.length, }, }; + await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 @@ -6714,6 +7634,17 @@ export async function runTestRerun( } if (failed > 0) { + // Auth failure on any member is a batch-wide condition — the credential is + // bad, not the test. Propagate exit 3 so the operator fixes auth rather than + // chasing a "rerun failed" (exit 1). Mirrors the identical logic already + // applied to runTestRunAll lines 5462-5468. + const authErr = rerunResults.find(r => r.error?.exitCode === 3); + if (authErr) { + throw new CLIError( + `${failed} rerun${failed !== 1 ? 's' : ''} failed — auth error (${authErr.error?.code}): ${authErr.error?.message}`, + 3, + ); + } throw new CLIError(`${failed} rerun${failed !== 1 ? 's' : ''} failed.`, 1); } @@ -6748,6 +7679,25 @@ export interface ArtifactGetResult { bundle?: WriteBundleResult; } +export function resolveDefaultArtifactDir(runId: string, cwd: string = process.cwd()): string { + requireNonEmpty('run-id', runId); + const windowsNormalizedSegment = runId.replace(/[ .]+$/u, ''); + if ( + windowsNormalizedSegment === '' || + windowsNormalizedSegment === '.' || + windowsNormalizedSegment === '..' || + runId.includes('/') || + runId.includes('\\') || + runId.includes('\0') + ) { + throw localValidationError( + 'run-id', + 'must be a single path-safe segment for the default output directory; pass --out

to choose a custom path', + ); + } + return join(cwd, '.testsprite', 'runs', runId); +} + /** * Validate that the parent directory of `resolvedDir` exists and is a * directory. Surfaces `VALIDATION_ERROR` (exit 5) — matches the convention @@ -6797,14 +7747,11 @@ export async function runArtifactGet( deps: TestDeps = {}, ): Promise { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const { runId } = opts; // Resolve output dir: explicit --out or the default .testsprite/runs// const resolvedDir = - opts.out !== undefined - ? resolveBundleDir(opts.out) - : join(process.cwd(), '.testsprite', 'runs', runId); + opts.out !== undefined ? resolveBundleDir(opts.out) : resolveDefaultArtifactDir(runId); // --dry-run: no network, no disk write. // The client (makeClient) is already wired with createDryRunFetch() when @@ -6850,6 +7797,8 @@ export async function runArtifactGet( await assertOutDirParentExists(resolvedDir); } + const client = makeClient(opts, deps); + // Fetch the run-scoped failure bundle. const { body: context, requestId: fetchRequestId } = await client.getWithMeta( `/runs/${encodeURIComponent(runId)}/failure`, @@ -7185,6 +8134,34 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('scaffold') + .description( + 'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.', + ) + .option('--type ', 'frontend|backend (default: frontend)') + .option('--out ', 'write the scaffold to a file instead of stdout') + .option('--force', 'overwrite an existing --out file', false) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite test scaffold > first-test.plan.json\n' + + ' testsprite test scaffold --type backend --out tests/health.py\n' + + ' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => { + await runScaffold( + { + ...resolveCommonOptions(command), + scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend', + out: cmdOpts.out, + force: cmdOpts.force === true, + }, + deps, + ); + }); + test .command('steps ') .description( @@ -7212,6 +8189,47 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('diff ') + .description( + 'Compare two runs and print what regressed: verdict, failureKind, failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runA: string, runB: string, _cmdOpts: unknown, command: Command) => { + await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); + }); + + test + .command('lint') + .description( + 'Validate plan/steps files offline with the same validators `create` runs, collecting EVERY problem. No network, no credentials. Exit 0 when all valid, 5 otherwise.', + ) + .option('--plan-from ', 'single plan JSON file') + .option( + '--plan-from-dir ', + 'directory of *.json plan files (each checked, all errors reported)', + ) + .option('--plans ', 'JSONL file with one plan spec per line (each line checked)') + .option('--steps ', 'plan-steps JSON file (the shape `test plan put` ingests)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { planFrom?: string; planFromDir?: string; plans?: string; steps?: string }, + command: Command, + ) => { + await runLint( + { + ...resolveCommonOptions(command), + planFrom: cmdOpts.planFrom, + planFromDir: cmdOpts.planFromDir, + plans: cmdOpts.plans, + steps: cmdOpts.steps, + }, + deps, + ); + }, + ); + test .command('result ') .description( @@ -7406,11 +8424,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--max-concurrency ', `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, ) + .option( + '--report ', + 'with --all --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + '(see `testsprite test create --help` for details).', ) @@ -7440,6 +8468,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', ); } + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isAll, + }); if (isAll) { // --all path: wave-ordered fresh batch run. @@ -7470,6 +8506,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7493,26 +8532,53 @@ export function createTestCommand(deps: TestDeps = {}): Command { }); test - .command('wait ') + .command('wait ') .description( - 'Wait for a run to reach a terminal status.\n' + + 'Wait for one or more runs to reach a terminal status.\n' + + '\nWith several run-ids the runs are polled concurrently under one shared\n' + + '--timeout and a {results, summary} envelope is printed (worst status wins\n' + + 'the exit code), so every re-attach hint the CLI prints can be pasted as\n' + + 'ONE command.\n' + '\nExit codes:\n' + ' 0 passed\n' + ' 1 failed / blocked / cancelled\n' + ' 3 auth error\n' + - ' 4 run not found\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 4 run not found (single run-id; with several ids a per-member poll error\n' + + ' is recorded as error: in its row and folded into exit 7)\n' + + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) + .option( + '--max-concurrency ', + 'with several run-ids, max concurrent polls (1-100, default: 10)', + ) .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { - await runTestWait( + .action(async (runIds: string[], cmdOpts: WaitFlagOpts, command: Command) => { + // One id keeps the historical single-run path byte-identical (same + // output shape, same exit codes); two or more fan out. + if (runIds.length === 1) { + await runTestWait( + { + ...resolveCommonOptions(command), + runId: runIds[0]!, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + return; + } + const maxConcurrency = parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? 10; + if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + await runTestWaitMany( { ...resolveCommonOptions(command), - runId, + runIds, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + maxConcurrency, }, deps, ); @@ -7577,6 +8643,15 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--idempotency-key ', 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', ) + .option( + '--report ', + 'with batch --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -7602,10 +8677,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so // autoHealExplicit is always false in this design — the default-on value // is never a deliberate user choice to opt in. + const testIds = testIdsArg ?? []; + const isBatch = cmdOpts.all === true || testIds.length !== 1; + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isBatch, + }); await runTestRerun( { ...resolveCommonOptions(command), - testIds: testIdsArg ?? [], + testIds, all: cmdOpts.all === true, projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, @@ -7620,11 +8705,72 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); }); + // ------------------------------------------------------------------------- + // `test flaky` — repeat-run flaky-test detector + // ------------------------------------------------------------------------- + + test + .command('flaky ') + .description( + 'Repeatedly replay a test to measure stability and surface flakiness.\n' + + 'Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.\n' + + '\nExit codes:\n' + + ' 0 stable (every attempt passed)\n' + + ' 1 flaky or failing (at least one attempt did not pass)\n' + + ' 3 auth error\n' + + ' 4 test not found (no replayable run — trigger `testsprite test run ` first)\n' + + ' 5 validation error', + ) + .option( + '--runs ', + `number of replays to run (1-${MAX_FLAKY_RUNS}, default ${DEFAULT_FLAKY_RUNS})`, + ) + .option( + '--until-fail', + 'stop at the first non-passing attempt (fast "is it flaky at all?" check)', + false, + ) + .option( + '--timeout ', + `per-attempt max seconds to wait (1-${MAX_RUN_TIMEOUT_SECONDS}, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .addHelpText( + 'after', + '\nNotes:\n' + + ' • Frontend replays are free verbatim script replays (no credit); backend replays\n' + + ' re-run the dependency closure and may cost credits — a one-line advisory is printed.\n' + + ' • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;\n' + + ' this measures replay stability of the saved script against the configured URL.\n' + + ' • `--output json` emits a machine-readable stability report for CI gating.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + testIdArg: string, + cmdOpts: { runs?: string; untilFail?: boolean; timeout?: string }, + command: Command, + ) => { + await runFlaky( + { + ...resolveCommonOptions(command), + testId: testIdArg, + runs: parseNumericFlag(cmdOpts.runs, 'runs') ?? DEFAULT_FLAKY_RUNS, + untilFail: cmdOpts.untilFail === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + }, + ); + test.addCommand(createTestCodeCommand(deps)); test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); @@ -7633,6 +8779,174 @@ export function createTestCommand(deps: TestDeps = {}): Command { return test; } +// --------------------------------------------------------------------------- +// `test flaky` — repeat-run flaky-test detector +// --------------------------------------------------------------------------- + +/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */ +const MAX_FLAKY_RUNS = 10; +/** Default replay count when `--runs` is omitted. */ +const DEFAULT_FLAKY_RUNS = 5; + +interface RunTestFlakyOptions extends CommonOptions { + testId: string; + /** Number of replays to run (1..MAX_FLAKY_RUNS). */ + runs: number; + /** Stop at the first non-passing attempt. */ + untilFail: boolean; + /** Per-attempt polling deadline in seconds. */ + timeoutSeconds: number; +} + +/** + * `test flaky ` — replay a test N times and report a stability score. + * + * Each attempt is a `POST /tests/{id}/runs/rerun` with auto-heal OFF (a strict + * verbatim replay) followed by `pollRunUntilTerminal`. Frontend replays are + * free verbatim script replays; backend replays re-run the dependency closure + * (a one-line credit advisory is printed). The pure scoring lives in + * `lib/flaky.ts`; this function is the I/O orchestrator. + * + * Exit code: 0 when every observed attempt passed (stable), else 1 — so CI can + * gate a merge on flakiness. + */ +export async function runFlaky( + opts: RunTestFlakyOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (typeof opts.testId !== 'string' || opts.testId.length === 0) { + throw localValidationError('test-id', 'is required'); + } + if (!Number.isInteger(opts.runs) || opts.runs < 1 || opts.runs > MAX_FLAKY_RUNS) { + throw localValidationError('runs', `must be an integer between 1 and ${MAX_FLAKY_RUNS}`); + } + + if (opts.dryRun) { + out.print({ + dryRun: true, + command: 'test flaky', + testId: opts.testId, + runs: opts.runs, + untilFail: opts.untilFail, + method: 'POST', + path: `/api/cli/v1/tests/${opts.testId}/runs/rerun`, + note: `Would replay the test up to ${opts.runs}x with auto-heal OFF and report a stability score.`, + }); + return undefined; + } + + // Under the implicit wait, raise the per-request timeout to cover --timeout + // so a slow trigger / long-poll under load isn't cut at the 120s default. + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + + // Best-effort test-type detection for the credit advisory. A probe failure + // never blocks the run — we just skip the advisory. + let isBackend = false; + try { + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + isBackend = test.type === 'backend'; + } catch { + // best-effort — proceed without the advisory. + } + if (isBackend) { + stderrFn( + `[advisory] ${opts.testId} is a backend test — each replay re-runs its dependency closure ` + + `and may cost credits. Frontend replays are free verbatim script replays; backend replays are not.`, + ); + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const attempts: FlakyAttempt[] = []; + + for (let i = 1; i <= opts.runs; i++) { + const idempotencyKey = `cli-flaky-${randomUUID()}`; + + let rerunResp: RerunResponse; + try { + // auto-heal is intentionally OFF: flaky detection needs a strict verbatim + // replay so healed drift cannot mask a nondeterministic pass/fail. + rerunResp = await client.triggerRerun(opts.testId, { source: 'cli' }, { idempotencyKey }); + } catch (err) { + // A missing replayable run is fatal for the whole command (mirror rerun): + // there is nothing to repeat, so point the user at a fresh `test run`. + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Test ${opts.testId} has no replayable run (unknown/cross-tenant id, or it has never completed a clean run).`, + nextAction: `Trigger a fresh run first: testsprite test run ${opts.testId}`, + requestId: err.requestId ?? 'local', + details: { testId: opts.testId, reason: 'no_replayable_run' }, + }, + }); + } + // Any other trigger error is recorded as an errored attempt so a single + // transient blip doesn't abort a long stability probe. + const code = err instanceof ApiError ? err.code : 'ERROR'; + attempts.push({ attempt: i, runId: null, outcome: 'error', failureKind: code }); + ticker.update(`Attempt ${i}/${opts.runs} — error (${code})`); + if (opts.untilFail) break; + continue; + } + + const runId = rerunResp.runId; + // Backend run rows never finalize server-side; resolve the verdict from the + // testId-scoped result on non-terminal ticks (same fallback as `test rerun`). + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => opts.testId, + resolveNotBefore: () => rerunResp.enqueuedAt, + }); + + let outcome: FlakyOutcome; + let failureKind: string | null = null; + try { + const finalRun = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + resolveAlternate, + }); + outcome = finalRun.status as FlakyOutcome; + failureKind = finalRun.failureKind; + } catch (err) { + // A per-attempt deadline (poll TimeoutError) or a client-side request + // timeout both count as a non-passing "timeout" outcome for this attempt. + if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { + outcome = 'timeout'; + } else { + throw err; + } + } + + attempts.push({ attempt: i, runId, outcome, failureKind }); + const passedSoFar = attempts.filter(a => a.outcome === 'passed').length; + ticker.update(`Attempt ${i}/${opts.runs} — ${outcome} (${passedSoFar} passed so far)`); + + if (opts.untilFail && outcome !== 'passed') break; + } + + ticker.finalize(); + + const report = summarizeFlaky(opts.testId, attempts); + out.print(report, data => renderFlakyText(data as FlakyReport)); + + const exitCode = flakyExitCode(report); + if (exitCode !== 0) { + throw new CLIError( + `Test ${opts.testId} is ${report.verdict} — ${report.passed}/${report.runs} attempts passed`, + exitCode, + ); + } + return report; +} + interface RunFlagOpts { targetUrl?: string; wait?: boolean; @@ -7643,10 +8957,14 @@ interface RunFlagOpts { project?: string; filter?: string; maxConcurrency?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface WaitFlagOpts { timeout?: string; + maxConcurrency?: string; } interface RerunFlagOpts { @@ -7661,6 +8979,9 @@ interface RerunFlagOpts { skipDependencies?: boolean; maxConcurrency?: string; idempotencyKey?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface UpdateFlagOpts { @@ -7806,13 +9127,9 @@ function resolveCommonOptions(command: Command): CommonOptions { // P2-8: validate --output before allowing silent fallback to 'text'. // An invalid value (e.g. `--output yaml`) must exit 5 with a clear error // rather than silently treating the request as text mode. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), dryRun: globals.dryRun ?? false, endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, @@ -7821,18 +9138,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - /** D4: headroom added on top of `--timeout` when deriving the per-request window under `--wait`. */ const WAIT_REQUEST_TIMEOUT_CUSHION_MS = 5_000; @@ -7933,6 +9238,17 @@ function openOutputFile(rawPath: string): FileSink { if (!parentStat.isDirectory()) { throw localValidationError('out', `parent path is not a directory: ${parent}`); } + let targetStat; + try { + targetStat = statSync(resolved); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + throw localValidationError('out', `cannot stat output path: ${resolved}`); + } + } + if (targetStat?.isDirectory()) { + throw localValidationError('out', `must point to a file, not a directory: ${resolved}`); + } const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); const sink: FileSink = { stream, path: resolved, tmpPath, error: null }; @@ -7998,6 +9314,19 @@ async function closeOutputFile(sink: FileSink, commit: boolean): Promise { await rename(sink.tmpPath, sink.path); } +/** Tear down an opened `--out` sink without leaving a zero-byte artifact. */ +async function abortOutputFile(sink: FileSink): Promise { + await new Promise(resolve => { + if (sink.stream.destroyed) { + resolve(); + return; + } + sink.stream.once('close', () => resolve()); + sink.stream.destroy(); + }); + await unlink(sink.tmpPath).catch(() => undefined); +} + /** A presigned `code` body is any `https://` URL — never anything else. */ export function isPresignedCodeUrl(code: string): boolean { return code.startsWith('https://'); @@ -8202,9 +9531,9 @@ function renderStepsText(page: Page): string { ' ' + 'UPDATED'; - const rows = page.items.map(s => { + const rows = page.items.flatMap(s => { const marker = s.outcomeContributesToFailure === true ? '* ' : ' '; - return [ + const row = [ marker, pad(String(s.stepIndex), indexWidth), pad(s.action, actionWidth), @@ -8212,6 +9541,19 @@ function renderStepsText(page: Page): string { pad(descOf(s), descWidth), s.updatedAt, ].join(' '); + // Run-scoped rows carry the per-step failure text; surface it as an + // indented sub-line under failed rows (mirrors the history table's + // `targetUrl:` sub-line). Collapsed to one line and capped so a huge + // stack blob can't wreck the table; full text ships in --output json. + if (s.status === 'failed' && typeof s.error === 'string' && s.error.length > 0) { + const oneLine = s.error.replace(/\s+/g, ' ').trim(); + const shown = + oneLine.length > ERROR_SUBLINE_MAX + ? `${oneLine.slice(0, ERROR_SUBLINE_MAX - 1)}…` + : oneLine; + return [row, ` error: ${shown}`]; + } + return [row]; }); const lines: string[] = [header, ...rows, '']; diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index 68a4c99..c83195d 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -1021,6 +1021,69 @@ describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdo }); }); +// --------------------------------------------------------------------------- +// TimeoutError on test wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestWait: TimeoutError writes partial JSON to stdout', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('running') })); + + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // FIX 4 — D5-UX: text mode shows the `error` string for failed/blocked runs // --------------------------------------------------------------------------- diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d8a0f21..a97dbba 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -17,12 +17,13 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { loadConfig } from '../lib/config.js'; import { resolvePortalBase } from '../lib/facade.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; /** * Usage/balance response from `/me` (when the backend supplies it) or a future @@ -199,7 +200,12 @@ export function createUsageCommand(deps: UsageDeps = {}): Command { '\nExamples:\n' + ' testsprite usage # show balance + plan\n' + ' testsprite usage --output json # machine-readable balance\n' + + ' testsprite usage --debug # trace HTTP method/path, request id, latency\n' + ' testsprite credits # alias for usage\n' + + '\nExit codes:\n' + + ' 0 success (or --dry-run)\n' + + ' 3 auth error — run `testsprite setup` to configure credentials\n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + " check your portal's Billing page (/dashboard/settings/billing) for your balance.", ) @@ -216,7 +222,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -225,13 +231,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - function makeOutput(mode: OutputMode, deps: UsageDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/index.ts b/src/index.ts index 7842ec9..806f6e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,9 @@ #!/usr/bin/env node + import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, createSetupCommand, @@ -11,10 +13,23 @@ import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; +import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { maybeNotifyUpdate } from './lib/update-check.js'; import { VERSION } from './version.js'; +import { shouldRejectNodeVersion } from './version-guard.js'; + +// Guard: exit early with a clear message on unsupported Node.js versions, +// rather than failing later with a cryptic ESM/runtime error. +if (shouldRejectNodeVersion(process.versions.node)) { + process.stderr.write( + `Error: testsprite requires Node.js >= 20 (found ${process.versions.node}).\nInstall the latest LTS from https://nodejs.org\n`, + ); + process.exit(1); +} const program = new Command(); @@ -76,6 +91,7 @@ program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); +program.addCommand(createDoctorCommand()); // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested @@ -128,16 +144,38 @@ program.hook('preAction', (_thisCommand, actionCommand) => { profile?: string; dryRun?: boolean; }; + const commandPath = commandPathOf(actionCommand); maybeEmitSkillNudge({ - commandPath: commandPathOf(actionCommand), + commandPath, output: isOutputMode(globals.output) ? globals.output : 'text', dryRun: globals.dryRun ?? false, profile: globals.profile ?? 'default', cwd: process.cwd(), env: process.env, }); + + // Best-effort update notice (see lib/update-check.ts): self-gates on the + // opt-out env, CI, TTY, and a 24h cache; the wiring adds the flag-level + // gates the lib cannot see. Skipped for `completion` (its stdout is eval'd + // by shells), under --output json, and under --dry-run. Deliberately not + // awaited: an advisory must never delay the real command. + if (globals.output !== 'json' && globals.dryRun !== true && commandPath !== 'completion') { + void maybeNotifyUpdate(); + } }); +// Clean process lifecycle: a clear message + conventional exit code on SIGINT / +// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted +// `test run --wait` explains the run continues server-side; plus an EPIPE guard +// so piping to a reader that closes early (`| head`) exits cleanly instead of +// dumping a raw `write EPIPE` stack. +installSignalHandlers(); +installBrokenPipeGuard(); + +// Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch +// ignores them by default). No-op when no proxy variable is set. +maybeInstallProxyAgent(); + try { await program.parseAsync(process.argv); } catch (err) { diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index b049711..51bf9b5 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -1,5 +1,7 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; +import { VERSION } from '../version.js'; import { DEFAULT_SKILLS, MANAGED_SECTION_BEGIN, @@ -9,13 +11,17 @@ import { SKILL_NAME, SKILLS, TARGETS, + bodyHash12, buildCodexAggregate, + buildSkillMarker, codexContentFor, loadCodexSkillBody, loadSkillBody, loadSkillBodyFor, + parseSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, } from './agent-targets.js'; // --------------------------------------------------------------------------- @@ -74,19 +80,31 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all five required keys', () => { + it('has all eight required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor']); + expect(keys).toEqual([ + 'antigravity', + 'claude', + 'cline', + 'codex', + 'copilot', + 'cursor', + 'kiro', + 'windsurf', + ]); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, and codex are experimental', () => { + it('cursor, cline, windsurf, copilot, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); + expect(TARGETS.windsurf.status).toBe('experimental'); + expect(TARGETS.copilot.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); + expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); }); @@ -102,6 +120,9 @@ describe('TARGETS', () => { expect(TARGETS.antigravity.mode).toBe('own-file'); expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); + expect(TARGETS.kiro.mode).toBe('own-file'); + expect(TARGETS.windsurf.mode).toBe('own-file'); + expect(TARGETS.copilot.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -200,6 +221,22 @@ describe('renderForTarget("antigravity")', () => { }); }); +describe('renderForTarget("kiro")', () => { + const result = renderForTarget('kiro', 'testsprite-verify', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.kiro/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); +}); + describe('renderForTarget("claude") vs renderForTarget("antigravity")', () => { it('produce the same frontmatter lines (name + description)', () => { const claude = renderForTarget('claude', 'testsprite-verify', STUB_BODY); @@ -269,16 +306,97 @@ describe('renderForTarget("cline")', () => { }); }); +describe('renderForTarget("windsurf")', () => { + const result = renderForTarget('windsurf', 'testsprite-verify', STUB_BODY); + + it('returns the .windsurf/rules path', () => { + expect(result.path).toBe('.windsurf/rules/testsprite-verify.md'); + }); + + it('uses the Cascade frontmatter (trigger: model_decision + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain('trigger: model_decision'); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('does NOT carry the Claude/Cursor frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + }); +}); + +describe('windsurf renders within the rules-file budget', () => { + // Regression: a `.windsurf/rules/*.md` file caps at ~12 K characters and + // Cascade silently truncates beyond that. The full verify body (~22 KB) would + // be cut in half, so windsurf renders the COMPACT body for verify (its trimmed + // codex asset) and the full body for onboard (which already fits). Uses the + // REAL bodies (no stub) so the size reflects what a user receives. + for (const skill of DEFAULT_SKILLS) { + it(`${skill} fits under 12 000 characters`, () => { + const r = renderForTarget('windsurf', skill); + expect(r.content.length).toBeLessThan(12_000); + }); + } + + it('verify uses the compact body (smaller than the full claude render)', () => { + const windsurf = renderForTarget('windsurf', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(windsurf.content.length).toBeLessThan(claude.content.length); + // The full-body-only intro line is absent from the compact body... + expect(claude.content).toContain('The verification loop that flies'); + expect(windsurf.content).not.toContain('The verification loop that flies'); + // ...but the load-bearing command survives. + expect(windsurf.content).toContain('testsprite test run'); + }); +}); + +describe('renderForTarget("copilot")', () => { + const result = renderForTarget('copilot', 'testsprite-verify', STUB_BODY); + + it('returns the .github/instructions path', () => { + expect(result.path).toBe('.github/instructions/testsprite-verify.instructions.md'); + }); + + it('uses the Copilot frontmatter (applyTo + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + expect(result.content).toContain("applyTo: '**'"); + }); + + it('does NOT carry the Claude/Cursor/Windsurf frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + expect(fm).not.toContain('trigger:'); // windsurf Cascade key + }); + + it('renders the compact verify body (applyTo:** is always-on, so keep it small)', () => { + // Uses the REAL bodies (no stub): copilot always-injects, so like windsurf it + // ships the trimmed verify body while keeping the load-bearing command. + const copilot = renderForTarget('copilot', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(copilot.content.length).toBeLessThan(claude.content.length); + expect(copilot.content).not.toContain('The verification loop that flies'); + expect(copilot.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { - const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity'> = [ + // Full-body own-file targets. Compact-body targets (windsurf, copilot) are + // excluded — they render the trimmed verify body; see their dedicated tests. + const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', 'cline', 'antigravity', + 'kiro', ]; // Use the real body for these checks, since we're guarding against trimming. @@ -680,3 +798,110 @@ describe('renderForTarget for testsprite-onboard', () => { expect(() => renderForTarget('claude', 'testsprite-unknown')).toThrow('unknown skill'); }); }); + +// --------------------------------------------------------------------------- +// Install marker (issue #123): format, parsing, and render placement +// --------------------------------------------------------------------------- + +describe('buildSkillMarker / parseSkillMarker / bodyHash12', () => { + it('marker line is an HTML comment carrying name, vVERSION, and a 12-hex hash', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(marker).toBe( + ``, + ); + expect(marker).toMatch( + /^$/, + ); + }); + + it('bodyHash12 equals the first 12 hex chars of the body SHA-256', () => { + const fullHex = createHash('sha256').update(STUB_BODY, 'utf8').digest('hex'); + expect(bodyHash12(STUB_BODY)).toBe(fullHex.slice(0, 12)); + }); + + it('parseSkillMarker round-trips a built marker embedded in surrounding content', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`# heading\n${marker}\nbody text\n`); + expect(parsed).not.toBeNull(); + expect(parsed?.skill).toBe('testsprite-verify'); + expect(parsed?.version).toBe(VERSION); + expect(parsed?.hash12).toBe(bodyHash12(STUB_BODY)); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker strips a trailing CR so CRLF checkouts parse identically', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`${marker}\r\nrest\r\n`); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker returns null when no marker line is present', () => { + expect(parseSkillMarker('# Just a heading\n\nProse without any marker.\n')).toBeNull(); + }); + + it('parseSkillMarker ignores the managed-section sentinels (also HTML comments)', () => { + expect(parseSkillMarker(`${MANAGED_SECTION_BEGIN}\nbody\n${MANAGED_SECTION_END}\n`)).toBeNull(); + }); +}); + +describe('render marker placement (own-file targets)', () => { + it('claude render carries the marker on the line right after the closing frontmatter fence', () => { + const { content } = renderForTarget('claude', 'testsprite-verify', STUB_BODY); + const closingFence = '\n---\n'; + const fenceEnd = content.indexOf(closingFence) + closingFence.length; + expect(content.slice(fenceEnd).startsWith(''; + const withForeign = renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + foreignMarker, + STUB_BODY, + ); + expect(withForeign).toContain(foreignMarker); + // Marker line aside, the bytes match the canonical render exactly. + const canonical = renderForTarget('claude', 'testsprite-verify', STUB_BODY).content; + const currentMarker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(withForeign.replace(foreignMarker, currentMarker)).toBe(canonical); + }); + + it('renderOwnFileWithMarker rejects the managed-section target', () => { + expect(() => renderOwnFileWithMarker('codex', 'testsprite-verify', 'marker', 'body')).toThrow( + 'own-file', + ); + }); + + it('renderOwnFileWithMarker throws on an unknown skill', () => { + expect(() => renderOwnFileWithMarker('claude', 'testsprite-unknown', 'marker')).toThrow( + 'unknown skill', + ); + }); +}); diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 2547daf..7e6f94d 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,6 +1,16 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; +import { VERSION } from '../version.js'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex'; +export type AgentTarget = + | 'claude' + | 'cursor' + | 'cline' + | 'antigravity' + | 'codex' + | 'kiro' + | 'windsurf' + | 'copilot'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -12,11 +22,19 @@ export interface TargetSpec { */ path: string; /** - * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity). + * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity/windsurf). * 'managed-section': the CLI writes only a sentinel-delimited section inside * a potentially user-authored file (codex target, AGENTS.md). */ mode: 'own-file' | 'managed-section'; + /** + * When true, render the budget-friendly body (see {@link compactBodyFor}) + * instead of the full own-file skill body. Used for own-file targets whose + * rule files are size-capped — currently `windsurf` (`.windsurf/rules/*.md` + * files cap at ~12 K characters and Cascade silently truncates beyond that, + * which would cut the full ~22 KB verify skill in half). + */ + compactBody?: boolean; /** * Wrap a skill body in this target's frontmatter/header. Takes the skill's * `name`+`description` (own-file targets emit them as frontmatter) and the body. @@ -120,6 +138,31 @@ function wrapMdc(_name: string, description: string, body: string): string { return `---\ndescription: ${description}\nalwaysApply: false\n---\n\n${body}\n`; } +/** + * Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md` with YAML + * frontmatter. `trigger: model_decision` is the Cascade equivalent of the Cursor + * `.mdc` `alwaysApply: false` mode: only the `description` is surfaced up front, + * and Cascade pulls in the full rule body when the description shows it is + * relevant — exactly the on-demand activation these skills want. (The other + * triggers are `always_on`, `manual`, and `glob`.) + */ +function wrapWindsurf(_name: string, description: string, body: string): string { + return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; +} + +/** + * GitHub Copilot reads path-specific custom instructions from + * `.github/instructions/*.instructions.md` (VS Code / Visual Studio / GitHub + * Copilot Chat). Each file carries YAML frontmatter with `applyTo` — a glob that + * scopes when the instructions attach. `applyTo: '**'` attaches the guidance to + * every request in the repo, which is what a persistent verification skill wants + * (there is no on-demand "model decides" mode for Copilot instruction files, so + * always-apply is the correct idiom). `description` is surfaced in Copilot's UI. + */ +function wrapCopilot(_name: string, description: string, body: string): string { + return `---\ndescription: ${description}\napplyTo: '**'\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -140,6 +183,12 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.cursor/rules/${skill}.mdc`; case 'cline': return `.clinerules/${skill}.md`; + case 'kiro': + return `.kiro/skills/${skill}/SKILL.md`; + case 'windsurf': + return `.windsurf/rules/${skill}.md`; + case 'copilot': + return `.github/instructions/${skill}.instructions.md`; case 'codex': return 'AGENTS.md'; } @@ -170,6 +219,35 @@ export const TARGETS: Record = { mode: 'own-file', wrap: (_name, _description, body) => body, }, + kiro: { + status: 'experimental', + path: pathFor('kiro', SKILL_NAME), + mode: 'own-file', + // kiro reads SKILL.md files with name/description frontmatter, same as + // claude/antigravity, so it shares the wrapSkill wrapper. + wrap: wrapSkill, + }, + windsurf: { + status: 'experimental', + path: pathFor('windsurf', SKILL_NAME), + mode: 'own-file', + // Windsurf rules files are budget-capped (~12 K chars per `.windsurf/rules/*.md`), + // so render the compact body per skill (see compactBodyFor). + compactBody: true, + wrap: wrapWindsurf, + }, + copilot: { + status: 'experimental', + path: pathFor('copilot', SKILL_NAME), + mode: 'own-file', + // GitHub Copilot path-specific instructions: frontmatter carries `applyTo`. + // `applyTo: '**'` means the file is ALWAYS injected into Copilot requests + // (there is no on-demand "model decides" mode like Cursor/Windsurf), so + // render the compact body to keep the always-on context cost small — the + // same reasoning that drives windsurf's compact render. + compactBody: true, + wrap: wrapCopilot, + }, /** * codex target — managed-section mode. * @@ -201,6 +279,85 @@ export const MANAGED_SECTION_BEGIN = ''; export const MANAGED_SECTION_END = ''; +// --------------------------------------------------------------------------- +// Install marker (stale-skill detection, issue #123) +// --------------------------------------------------------------------------- + +/** + * Hex characters of the canonical body's SHA-256 kept in the install marker. + * 12 hex chars (48 bits) is ample for drift DETECTION (equality against bodies + * this CLI ships); the marker is provenance metadata, not a security boundary. + */ +const MARKER_HASH_HEX_LENGTH = 12; + +/** + * When one marker covers several skills (the codex managed section aggregates + * every installed skill), their names are joined with this separator in the + * marker's skill field. Skill names never contain '+' (see {@link SKILLS} keys). + */ +export const MARKER_SKILL_SEPARATOR = '+'; + +/** + * Marker line shape: ``. + * An HTML comment is inert in every target format (SKILL.md, .mdc, .clinerules + * markdown, AGENTS.md). Built via `new RegExp` so the hash length stays bound + * to {@link MARKER_HASH_HEX_LENGTH}. + */ +const SKILL_MARKER_LINE_RE = new RegExp( + `^$`, +); + +/** + * First {@link MARKER_HASH_HEX_LENGTH} hex chars of the SHA-256 of a canonical + * skill body. The hash covers the CANONICAL BODY ONLY (pre-wrap, pre-marker), + * so writing the marker into the rendered artifact never changes the hash the + * marker itself carries. + */ +export function bodyHash12(canonicalBody: string): string { + return createHash('sha256') + .update(canonicalBody, 'utf8') + .digest('hex') + .slice(0, MARKER_HASH_HEX_LENGTH); +} + +/** + * Build the provenance marker line for a skill (or a + * {@link MARKER_SKILL_SEPARATOR}-joined skill set) and its canonical body. + * `agent status` compares this fingerprint against the bodies the running CLI + * ships to detect silently stale installs. + */ +export function buildSkillMarker(skillName: string, canonicalBody: string): string { + return ``; +} + +/** A marker line parsed back into its fields. */ +export interface ParsedSkillMarker { + /** Skill name, or several names joined with {@link MARKER_SKILL_SEPARATOR}. */ + skill: string; + /** CLI version that wrote the artifact. */ + version: string; + /** First 12 hex chars of the canonical body's SHA-256 at install time. */ + hash12: string; + /** The exact marker line (trailing CR/whitespace stripped) as found. */ + line: string; +} + +/** + * Find the first testsprite-skill marker line in `content`, or null when the + * content carries none (a pre-marker install). Lines are matched whole with + * trailing CR/whitespace stripped, so CRLF checkouts parse identically. + */ +export function parseSkillMarker(content: string): ParsedSkillMarker | null { + for (const rawLine of content.split('\n')) { + const line = rawLine.trimEnd(); + const matched = SKILL_MARKER_LINE_RE.exec(line); + if (matched) { + return { skill: matched[1]!, version: matched[2]!, hash12: matched[3]!, line }; + } + } + return null; +} + type ReadFn = (url: URL) => string; const defaultRead: ReadFn = (url: URL) => readFileSync(url, 'utf8'); @@ -227,6 +384,24 @@ export function loadSkillBodyFor(skill: string, read: ReadFn = defaultRead): str return readSkillAsset(spec.bodyFile, read); } +/** + * Budget-friendly body for an own-file target whose rule files are size-capped + * (e.g. windsurf). For a skill that ships a trimmed codex asset (`codex.kind === + * 'full'`, e.g. `testsprite-verify` — full body ~22 KB, codex ~5 KB) we render + * that compact asset so the wrapped file stays under the cap. For skills whose + * codex contribution is only a one-liner (`'line'`/`'none'`, e.g. + * `testsprite-onboard`), the one-liner is useless as a standalone rule and the + * full own-file body (~6.5 KB) already fits the budget — so the full body is + * used. + */ +export function compactBodyFor(skill: string, read: ReadFn = defaultRead): string { + const spec = SKILLS[skill]; + if (!spec) throw new Error(`unknown skill: ${skill}`); + return spec.codex.kind === 'full' + ? readSkillAsset(spec.codex.file, read) + : loadSkillBodyFor(skill, read); +} + /** * Resolve a skill's codex (AGENTS.md) contribution as a Markdown string. * 'full' → read the `*.codex.md` asset; 'line' → the inline one-liner; 'none' → ''. @@ -275,15 +450,67 @@ export function loadCodexSkillBody(read: ReadFn = defaultRead): string { // renderForTarget // --------------------------------------------------------------------------- +/** + * Place the marker line inside a wrapped own-file render. + * + * - Wraps that emit YAML frontmatter (claude/antigravity/cursor): the marker + * lands on the line right after the closing `---` fence, before the body. + * - Wrapless targets (cline, body verbatim): the marker is appended as the + * LAST line instead. Cline surfaces the file's first heading as the rule + * title, so a leading comment would displace the body's H1. + */ +function injectMarkerLine(wrapped: string, markerLine: string): string { + if (wrapped.startsWith('---\n')) { + // The name/description frontmatter values are single-line, so the first + // `\n---\n` after the opening fence is always the closing fence. + const closingFence = '\n---\n'; + const fenceIdx = wrapped.indexOf(closingFence); + if (fenceIdx !== -1) { + const insertAt = fenceIdx + closingFence.length; + return `${wrapped.slice(0, insertAt)}${markerLine}\n${wrapped.slice(insertAt)}`; + } + } + const separator = wrapped.endsWith('\n') ? '' : '\n'; + return `${wrapped}${separator}${markerLine}\n`; +} + +/** + * Exact own-file bytes for a skill on a target, carrying the GIVEN marker line. + * `agent status` uses this to re-render the current canonical body with a + * file's own (possibly older-versioned) marker: when only the marker's version + * string lags but the body is unchanged, the artifact still compares pristine. + */ +export function renderOwnFileWithMarker( + target: AgentTarget, + skill: string, + markerLine: string, + body?: string, +): string { + const spec = TARGETS[target]; + if (spec.mode !== 'own-file') { + throw new Error(`renderOwnFileWithMarker: ${target} is not an own-file target`); + } + const skillSpec = SKILLS[skill]; + if (!skillSpec) throw new Error(`unknown skill: ${skill}`); + const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + return injectMarkerLine( + spec.wrap(skillSpec.name, skillSpec.description, resolvedBody), + markerLine, + ); +} + /** * The exact bytes to write for one skill on one target. * * - own-file targets: `body` defaults to the skill's own-file asset, wrapped in - * the target's frontmatter/header. + * the target's frontmatter/header, and carrying a provenance marker line so + * `agent status` can tell fresh, stale, and hand-edited installs apart. * - codex (managed-section): returns the skill's codex contribution unwrapped - * (plain Markdown, no frontmatter). The real install does NOT call this for - * codex — it aggregates all skills via {@link buildCodexAggregate} — but it is - * kept single-skill here for tests and parity. Pass an explicit `body` to override. + * and marker-free (plain Markdown, no frontmatter). The real install does NOT + * call this for codex: it aggregates all skills via + * {@link buildCodexAggregate} and writes ONE marker just inside the BEGIN + * sentinel. It is kept single-skill here for tests and parity. Pass an + * explicit `body` to override. */ export function renderForTarget( t: AgentTarget, @@ -298,6 +525,10 @@ export function renderForTarget( const resolvedBody = body !== undefined ? body : codexContentFor(skill); return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } - const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); - return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; + const resolvedBody = + body !== undefined ? body : spec.compactBody ? compactBodyFor(skill) : loadSkillBodyFor(skill); + return { + path, + content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), + }; } diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 5b746da..aec451a 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -8,7 +8,7 @@ * the full http+fetch path is wired against MSW). */ -import { existsSync, mkdtempSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -18,6 +18,7 @@ import { assertNoEscape, BUNDLE_SCHEMA_VERSION, buildMeta, + isBundleOwnedEntry, pickCodeExtension, resolveBundleDir, STREAM_URL_MAX_RETRIES, @@ -640,17 +641,24 @@ describe('streamUrlToFile retry', () => { calls++; throw new Error('ENETUNREACH dns lookup failed'); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + + try { + await streamUrlToFile( + 'https://example.com/x?X-Amz-Signature=secret-token', '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ name: 'TransportError', message: expect.stringContaining('ENETUNREACH'), }); + expect(caught).not.toMatchObject({ message: expect.stringContaining('secret-token') }); expect(calls).toBe(STREAM_URL_MAX_RETRIES); }); @@ -660,17 +668,46 @@ describe('streamUrlToFile retry', () => { calls++; return new Response('Forbidden', { status: 403 }); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + const presignedUrl = 'https://example.com/x?X-Amz-Signature=secret-token#download'; + + try { + await streamUrlToFile( + presignedUrl, '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ code: 'UNAVAILABLE' }); + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ + code: 'UNAVAILABLE', + details: { status: 403, artifactUrl: 'https://example.com/x' }, + }); + const details = (caught as { details?: Record }).details; + expect(details).not.toHaveProperty('url'); + expect(JSON.stringify(details)).not.toContain('secret-token'); expect(calls).toBe(1); }); + it('disables automatic redirects so unsafe redirect targets cannot bypass URL validation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stream-test-')); + const dest = join(dir, 'out.bin'); + const redirects: Array = []; + const fetchImpl = async (_url: Parameters[0], init?: RequestInit) => { + redirects.push(init?.redirect); + return new Response('hello', { status: 200 }); + }; + + await streamUrlToFile('https://example.com/x', dest, fetchImpl as typeof globalThis.fetch, { + sleep: noSleep, + }); + + expect(redirects).toEqual(['error']); + }); + it('sleeps between retries', async () => { const sleepDelays: number[] = []; const fetchImpl = async () => { @@ -694,6 +731,37 @@ describe('streamUrlToFile retry', () => { }); }); +describe('isBundleOwnedEntry', () => { + it('owns the fixed bundle file set', () => { + for (const entry of [ + 'result.json', + 'failure.json', + 'video.mp4', + 'meta.json', + 'steps', + '.tmp', + '.partial', + ]) { + expect(isBundleOwnedEntry(entry)).toBe(true); + } + }); + + it('owns code. for any single-token extension', () => { + expect(isBundleOwnedEntry('code.ts')).toBe(true); + expect(isBundleOwnedEntry('code.js')).toBe(true); + expect(isBundleOwnedEntry('code.py')).toBe(true); + }); + + it('does not own foreign entries', () => { + expect(isBundleOwnedEntry('notes.txt')).toBe(false); + expect(isBundleOwnedEntry('src')).toBe(false); + expect(isBundleOwnedEntry('.git')).toBe(false); + expect(isBundleOwnedEntry('code.tar.gz')).toBe(false); + expect(isBundleOwnedEntry('mycode.ts')).toBe(false); + expect(isBundleOwnedEntry('code.')).toBe(false); + }); +}); + describe('step artifact path validation', () => { // A fetchImpl that fails the test if called — proves validation rejects // before any write happens. @@ -807,6 +875,49 @@ describe('step artifact path validation', () => { expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); }); + describe('commit sweep ownership (data-loss guard)', () => { + it('preserves pre-existing foreign files and directories in the --out dir', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'notes.txt'), 'important notes\n', 'utf8'); + mkdirSync(join(dir, 'src')); + writeFileSync(join(dir, 'src', 'app.js'), "console.log('app')\n", 'utf8'); + + const res = await writeBundle(stepCtx(3), { + dir, + failedOnly: false, + fetchImpl: throwIfFetched, + }); + + // The bundle landed… + expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); + expect(existsSync(join(res.dir, 'result.json'))).toBe(true); + // …and the user's unrelated files survived the commit sweep. + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('important notes\n'); + expect(readFileSync(join(dir, 'src', 'app.js'), 'utf8')).toBe("console.log('app')\n"); + }); + + it('still sweeps a stale bundle-owned video.mp4 the new bundle does not write', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'video.mp4'), 'stale-bytes', 'utf8'); + + // stepCtx has videoUrl: null → the fresh bundle ships no video. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + }); + + it('sweeps a stale code file with a different extension than the new bundle writes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'code.py'), '# stale python code\n', 'utf8'); + + // baseCtx.code.language is 'typescript' → the fresh bundle writes code.ts. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'code.ts'))).toBe(true); + expect(existsSync(join(dir, 'code.py'))).toBe(false); + }); + }); + describe('assertNoEscape', () => { it('returns the resolved path for an in-bounds segment', () => { const base = mkdtempSync(join(tmpdir(), 'bundle-test-')); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 7795115..a3c0808 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -543,6 +543,30 @@ async function freshTmpDir(dir: string): Promise { * caught reading the dir during it sees no meta and refuses to consume * (per §7.3). That's what we want. */ +/** + * Whether a top-level directory entry belongs to the bundle format — + * i.e. something a prior `writeBundle` could have produced and this + * commit is therefore allowed to clean up. `code.` is matched by + * pattern (not the current run's extension) so a stale `code.py` is + * still swept when the new bundle writes `code.ts`. Everything else in + * the directory is the user's and must never be deleted (`--out` can + * point at a pre-existing, populated directory). + */ +export function isBundleOwnedEntry(entry: string): boolean { + if ( + entry === 'result.json' || + entry === 'failure.json' || + entry === 'video.mp4' || + entry === 'meta.json' || + entry === 'steps' || + entry === '.tmp' || + entry === '.partial' + ) { + return true; + } + return /^code\.[A-Za-z0-9]+$/.test(entry); +} + async function commitBundle( tmpDir: string, dir: string, @@ -553,20 +577,22 @@ async function commitBundle( // (2) Sweep stale top-level files that the new bundle won't write. // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Enumerate - // current top-level entries and remove anything that isn't being - // freshly renamed in. + // an in-place rename leaves the old video lingering. Only entries the + // bundle format OWNS are candidates: `--out` may point at a directory + // that also holds the user's unrelated files, and those must survive + // the commit (deleting them would be silent data loss). const topLevel = files.filter(f => !f.startsWith('steps/')); const newTopLevelSet = new Set(topLevel); newTopLevelSet.add('meta.json'); // about to land last, do not delete const existing = await readdir(dir).catch(() => [] as string[]); for (const entry of existing) { // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Anything else - // not-listed in the new bundle is stale. + // (we'll re-evaluate .partial at the end of commit). Any other + // bundle-owned entry not-listed in the new bundle is stale. if (entry === '.tmp' || entry === '.partial') continue; if (newTopLevelSet.has(entry)) continue; if (entry === 'steps') continue; // handled below + if (!isBundleOwnedEntry(entry)) continue; // foreign file — never touch await rm(join(dir, entry), { recursive: true, force: true }); } @@ -746,17 +772,18 @@ export async function streamUrlToFile( deps?: { sleep?: (ms: number) => Promise }, ): Promise { const sleepFn = deps?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const artifactUrl = redactArtifactUrlForDetails(url); for (let attempt = 1; attempt <= STREAM_URL_MAX_RETRIES; attempt++) { let response: Response; try { - response = await fetchImpl(url); + response = await fetchImpl(url, { redirect: 'error' }); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (attempt < STREAM_URL_MAX_RETRIES) { await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } if (!response.ok) { // Non-2xx: the URL itself is bad (expired, unauthorized, not found). @@ -768,7 +795,7 @@ export async function streamUrlToFile( nextAction: 'Re-run `testsprite test failure get`. Presigned URLs in the bundle expire after 15 minutes.', requestId: 'local', - details: { status: response.status, url }, + details: { status: response.status, artifactUrl }, }, }); } @@ -788,7 +815,7 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } } await mkdir(dirname(filePath), { recursive: true }); @@ -810,11 +837,20 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed mid-download of ${url}: ${message}`); + throw new TransportError(`Failed mid-download of ${artifactUrl}: ${message}`); } } } +function redactArtifactUrlForDetails(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return ''; + } +} + function isPresignedUrl(value: string): boolean { return value.startsWith('https://'); } diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index f9fdc44..d5135cc 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -1,10 +1,12 @@ -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_API_KEY, DRY_RUN_BANNER, + assertValidApiKeyHeaderValue, assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, resetDryRunBannerForTesting, resolveRequestTimeoutMs, } from './client-factory.js'; @@ -213,6 +215,45 @@ describe('resolveRequestTimeoutMs', () => { }); }); +// --------------------------------------------------------------------------- +// parseRequestTimeoutFlag — strict flag parsing (seconds → ms) +// --------------------------------------------------------------------------- + +describe('parseRequestTimeoutFlag', () => { + it('returns undefined when the flag is omitted (factory falls back to env/default)', () => { + expect(parseRequestTimeoutFlag(undefined)).toBeUndefined(); + }); + + it('converts a positive number of seconds to milliseconds', () => { + expect(parseRequestTimeoutFlag('30')).toBe(30_000); + expect(parseRequestTimeoutFlag('1')).toBe(1_000); + expect(parseRequestTimeoutFlag('2.5')).toBe(2_500); + }); + + it('does NOT reject positive out-of-range values — resolveRequestTimeoutMs clamps them', () => { + // 700s is above the 600s cap, but parsing succeeds; the clamp lives in + // resolveRequestTimeoutMs so a large script-supplied value still works. + expect(parseRequestTimeoutFlag('700')).toBe(700_000); + }); + + it.each(['abc', '30s', '0', '-5', 'NaN', 'Infinity', ''])( + 'throws a VALIDATION_ERROR (exit 5) on the invalid flag value %j', + bad => { + let caught: unknown; + try { + parseRequestTimeoutFlag(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('request-timeout'); + }, + ); +}); + // --------------------------------------------------------------------------- // makeHttpClient — requestTimeoutMs propagation // --------------------------------------------------------------------------- @@ -291,6 +332,77 @@ describe('makeHttpClient — real path (regression)', () => { // assertValidEndpointUrl — endpoint syntax guard (NOT an SSRF guard) // --------------------------------------------------------------------------- +describe('makeHttpClient - API key validation', () => { + it.each([ + ['newline', 'sk-user-abc\ndef'], + ['carriage return', 'sk-user-abc\rdef'], + ['smart dash', 'sk-user-abc\u2013def'], + ['smart quote', 'sk-user-\u201cabc\u201d'], + ['emoji', 'sk-user-abc\u{1f600}'], + ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); + + it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: ' ' } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + expect((caught as ApiError).code).toBe('AUTH_REQUIRED'); + }); +}); + +describe('assertValidApiKeyHeaderValue', () => { + it('accepts a normal ASCII API key value', () => { + expect(() => assertValidApiKeyHeaderValue('sk-user-abc-def_123')).not.toThrow(); + }); + + it('throws a VALIDATION_ERROR (exit 5) for a key that cannot be sent as x-api-key', () => { + let caught: unknown; + try { + assertValidApiKeyHeaderValue('sk-user-abc\u2013def'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + describe('assertValidEndpointUrl', () => { it('accepts http(s) URLs, including private / localhost hosts (self-hosted, dev, mock)', () => { for (const url of [ diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index fa1f8dc..829410d 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -180,6 +180,59 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +export function assertValidApiKeyHeaderValue(apiKey: string): void { + const reason = + 'must be a non-empty HTTP header value; paste the raw key without smart punctuation, emoji, or line breaks'; + + if (apiKey.trim().length === 0) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + + for (let i = 0; i < apiKey.length; i += 1) { + const code = apiKey.charCodeAt(i); + if (code < 0x20 || code === 0x7f || code > 0xff) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + } +} + +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * + * Returns `undefined` when the flag was omitted (the factory then falls back to + * the `TESTSPRITE_REQUEST_TIMEOUT_MS` env var, else the 120s default). + * + * A supplied-but-invalid value (non-numeric, zero, or negative) throws a typed + * VALIDATION_ERROR (exit 5) rather than being silently dropped. An explicit + * `--request-timeout 30s` typo previously resolved to `undefined` and the + * command ran with the default 120s deadline — the operator believed they had + * set a timeout but had not, with no signal. Failing loudly here is consistent + * with every other validated flag (`--page-size`, `--output`, `--type`). + * + * Out-of-range but positive values are intentionally NOT rejected — they flow + * through to {@link resolveRequestTimeoutMs}, which clamps to + * `[REQUEST_TIMEOUT_MIN_MS, REQUEST_TIMEOUT_MAX_MS]`. The env-var path stays + * lenient by design (a stray global env var should not hard-fail every + * command); only the explicit per-invocation flag is strict. + * + * This single definition replaces five byte-identical copies that previously + * lived in `auth`, `project`, `usage`, `init`, and `test` — drift between them + * would have silently changed timeout behaviour depending on the command. + */ +export function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + // Surface the offending value in the message (same as assertValidEndpointUrl) + // so the operator sees exactly what they typed. + throw localValidationError( + 'request-timeout', + `"${raw}" is not valid — must be a positive number of seconds`, + ); + } + return Math.round(n * 1000); // seconds → milliseconds +} + export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {}): HttpClient { const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const env = deps.env ?? process.env; @@ -214,6 +267,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} // VALIDATION_ERROR rather than an opaque URL throw or a retried "fetch failed". assertValidEndpointUrl(config.apiUrl); if (!config.apiKey) throw ApiError.authRequired(); + assertValidApiKeyHeaderValue(config.apiKey); return new HttpClient({ baseUrl: facadeBaseUrl(config.apiUrl), apiKey: config.apiKey, diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index 463ddcc..cf56eaf 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -81,6 +81,28 @@ describe('loadConfig', () => { const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); expect(config.apiKey).toBe('sk-dev'); }); + + it('treats empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile)', () => { + writeProfile( + 'default', + { apiKey: 'sk-file', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const config = loadConfig({ + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://api.example.com:8443'); + }); + + it('treats empty / whitespace TESTSPRITE_API_KEY as unset (falls through to profile)', () => { + writeProfile('default', { apiKey: 'sk-file' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_KEY: '' }, + credentialsPath, + }); + expect(config.apiKey).toBe('sk-file'); + }); }); describe('defaultConfigPath', () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 363c394..7f8065f 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -17,6 +17,11 @@ export interface LoadConfigOptions { const DEFAULT_API_URL = 'https://api.testsprite.com'; +/** Treat empty / whitespace-only env values as unset for `??` resolution chains. */ +export function normalizeEnvVar(value: string | undefined): string | undefined { + return value?.trim() || undefined; +} + export function defaultConfigPath(): string { return join(homedir(), '.testsprite', 'config'); } @@ -38,9 +43,15 @@ export function loadConfig(options: LoadConfigOptions = {}): Config { const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); + // Empty / whitespace-only env vars are treated as unset so they do not + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell + // profile). Matches the normalization in auth configure and init/setup. + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); + return { - apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, - apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, + apiUrl: options.endpointUrl ?? envApiUrl ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: envApiKey ?? fileEntry?.apiKey, profile, }; } diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index ac038f3..896d057 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -88,6 +88,38 @@ describe('serializeCredentials', () => { expect(text).toContain('api_key = sk'); expect(text).not.toContain('api_url'); }); + + it('strips newline characters from values to prevent INI injection', () => { + // A malicious apiUrl with embedded newlines could inject new key-value + // pairs or section headers into the credentials file. The serializer + // must strip \n and \r so the written file has exactly one value per + // field and no injected content parsed as separate keys/sections. + const malicious = 'https://evil.com\napi_key = sk-HIJACKED\n[admin]\napi_key = sk-admin'; + const text = serializeCredentials({ default: { apiKey: 'sk-real', apiUrl: malicious } }); + // The output must NOT contain a standalone [admin] section header + // (it would be on its own line if injection succeeded) + const lines = text.split('\n'); + // Only one section header exists: [default] + const sectionHeaders = lines.filter(l => /^\[.+\]$/.test(l.trim())); + expect(sectionHeaders).toEqual(['[default]']); + // Only one api_key line exists (the real one, not an injected duplicate) + const apiKeyLines = lines.filter(l => l.trim().startsWith('api_key')); + expect(apiKeyLines).toHaveLength(1); + expect(apiKeyLines[0]).toContain('sk-real'); + // Round-trip: reading back must return only the real key, not the injected one + const parsed = parseCredentials(text); + expect(parsed['default']?.apiKey).toBe('sk-real'); + expect(parsed['admin']).toBeUndefined(); + }); + + it('strips \\r\\n (CRLF) injection from values', () => { + const text = serializeCredentials({ default: { apiUrl: 'https://x.com\r\napi_key = pwned' } }); + const parsed = parseCredentials(text); + // The injected api_key must NOT be parsed as a real key + expect(parsed['default']?.apiKey).toBeUndefined(); + // The api_url value is on one line (newlines stripped) + expect(parsed['default']?.apiUrl).toContain('https://x.com'); + }); }); describe('readCredentialsFile / readProfile', () => { diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index c885c66..21e2ac0 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -116,7 +116,15 @@ export function serializeCredentials(file: CredentialsFile): string { for (const field of fields) { const value = entry[field]; if (value === undefined || value === '') continue; - lines.push(`${FIELD_TO_FILE_KEY[field]} = ${value}`); + // Guard against INI injection: a value containing newline characters + // would be serialized across multiple lines, allowing an attacker to + // inject arbitrary key-value pairs (or new section headers) into the + // credentials file. A valid API key or URL never contains \n or \r. + // Strip them so a compromised env var or MITM'd backend response + // cannot override the stored api_key on subsequent reads. + const sanitized = value.replace(/[\r\n]/g, ''); + if (sanitized === '') continue; + lines.push(`${FIELD_TO_FILE_KEY[field]} = ${sanitized}`); } lines.push(''); } diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index e1bf8ff..a1838d6 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; + +describe('sampleJUnitReportXml', () => { + it('returns well-formed JUnit XML with canned batch ids', () => { + const xml = sampleJUnitReportXml('proj_dry'); + expect(xml).toContain(''); + expect(xml).toContain(' { + const xml = sampleJUnitReportXml('proj_dry', 'custom-ci-suite'); + expect(xml).toContain(' { it('resolves /me', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 28fa1bb..4c7cbb0 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -27,6 +27,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { TriggerRunResponse, @@ -67,6 +68,37 @@ const SAMPLE_REQUEST_ID = 'req_dry-run'; export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; +/** + * Canned JUnit XML for batch `--wait --report junit --dry-run`. Mirrors the + * fresh batch-run sample ids so agents can learn the sidecar shape offline. + */ +export function sampleJUnitReportXml( + projectId: string = SAMPLE_PROJECT_ID, + reportSuiteName?: string, +): string { + return buildJUnitReport({ + suiteName: reportSuiteName ?? `testsprite:${projectId}`, + classname: projectId, + results: [ + { + testId: SAMPLE_TEST_ID_FRESH_1, + runId: SAMPLE_BATCH_FRESH_RUN_ID_1, + status: 'passed', + }, + { + testId: SAMPLE_TEST_ID_FRESH_2, + runId: SAMPLE_BATCH_FRESH_RUN_ID_2, + status: 'failed', + error: { + code: 'ASSERTION', + message: 'Expected checkout heading to be visible', + exitCode: 1, + }, + }, + ], + }); +} + const me: MeResponse = { userId: SAMPLE_USER_ID, keyId: SAMPLE_KEY_ID, diff --git a/src/lib/flaky.test.ts b/src/lib/flaky.test.ts new file mode 100644 index 0000000..a6acd9c --- /dev/null +++ b/src/lib/flaky.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { flakyExitCode, renderFlakyText, summarizeFlaky, type FlakyAttempt } from './flaky.js'; + +function pass(attempt: number): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'passed' }; +} +function fail(attempt: number, failureKind = 'assertion'): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'failed', failureKind }; +} + +describe('summarizeFlaky', () => { + it('reports STABLE when every attempt passed', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), pass(3)]); + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(report.passed).toBe(3); + expect(report.failed).toBe(0); + expect(report.stableRatio).toBe(1); + expect(report.failures).toEqual([]); + expect(flakyExitCode(report)).toBe(0); + }); + + it('reports FLAKY on a mix of pass and fail', () => { + const report = summarizeFlaky('test_x', [pass(1), fail(2), pass(3)]); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(2); + expect(report.failed).toBe(1); + expect(report.stableRatio).toBe(0.6667); + expect(report.failures).toEqual([ + { attempt: 2, runId: 'run_2', outcome: 'failed', failureKind: 'assertion' }, + ]); + expect(flakyExitCode(report)).toBe(1); + }); + + it('reports FAILING when no attempt passed', () => { + const report = summarizeFlaky('test_x', [fail(1), fail(2)]); + expect(report.verdict).toBe('failing'); + expect(report.passed).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('treats an empty attempt list as FAILING with a 0 ratio', () => { + const report = summarizeFlaky('test_x', []); + expect(report.verdict).toBe('failing'); + expect(report.runs).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('counts timeout and error outcomes as non-passing', () => { + const attempts: FlakyAttempt[] = [ + pass(1), + { attempt: 2, runId: 'run_2', outcome: 'timeout' }, + { attempt: 3, runId: null, outcome: 'error', failureKind: 'UNAVAILABLE' }, + ]; + const report = summarizeFlaky('test_x', attempts); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(1); + expect(report.failed).toBe(2); + expect(report.failures.map(f => f.outcome)).toEqual(['timeout', 'error']); + // error attempt with no runId is preserved as null in the report + expect(report.failures[1]).toEqual({ + attempt: 3, + runId: null, + outcome: 'error', + failureKind: 'UNAVAILABLE', + }); + }); + + it('rounds stableRatio to 4 decimal places', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), fail(3)]); + expect(report.stableRatio).toBe(0.6667); + }); + + it('reflects a short-circuited (--until-fail) run — fewer runs than requested', () => { + // Only two attempts were observed before the probe stopped at the failure. + const report = summarizeFlaky('test_x', [pass(1), fail(2)]); + expect(report.runs).toBe(2); + expect(report.verdict).toBe('flaky'); + }); +}); + +describe('renderFlakyText', () => { + it('summarizes a stable run on one line with no failure list', () => { + const text = renderFlakyText(summarizeFlaky('test_login', [pass(1), pass(2)])); + expect(text).toBe('Ran test_login 2x — 2 passed, 0 failed → STABLE (100% stable)'); + }); + + it('lists failing attempts with runId and failureKind', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [pass(1), fail(2, 'network_timeout')]), + ); + expect(text).toContain('→ FLAKY (50% stable)'); + expect(text).toContain('failed attempts:'); + expect(text).toContain('#2 run_2 failed failureKind=network_timeout'); + }); + + it('shows (no runId) when an errored attempt never got a runId', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [{ attempt: 1, runId: null, outcome: 'error' }]), + ); + expect(text).toContain('#1 (no runId) error'); + }); +}); diff --git a/src/lib/flaky.ts b/src/lib/flaky.ts new file mode 100644 index 0000000..402eaf3 --- /dev/null +++ b/src/lib/flaky.ts @@ -0,0 +1,133 @@ +/** + * Pure aggregation + rendering for `test flaky` — the repeat-run flaky-test + * detector. This module performs no I/O: the orchestrator in + * `commands/test.ts` feeds it the per-attempt outcomes, and it returns a + * machine-readable stability report plus the text rendering and exit code. + * + * Keeping the scoring logic pure makes it trivially unit-testable in isolation + * (deterministic, no network / credentials), matching the repo's mock-based + * test convention. + */ + +/** + * Outcome of a single flaky-detector attempt. The first four mirror the + * terminal `RunStatus` values; `timeout` and `error` are orchestration + * outcomes (per-attempt deadline exceeded, or a non-fatal trigger/transport + * error that was recorded rather than aborting the whole probe). + */ +export type FlakyOutcome = 'passed' | 'failed' | 'blocked' | 'cancelled' | 'timeout' | 'error'; + +/** Overall stability verdict across all observed attempts. */ +export type FlakyVerdict = 'stable' | 'flaky' | 'failing'; + +/** One recorded attempt in a flaky run. */ +export interface FlakyAttempt { + /** 1-based attempt index. */ + attempt: number; + /** The runId of this attempt, or `null` when the trigger never returned one. */ + runId: string | null; + outcome: FlakyOutcome; + /** + * Server `failureKind` for non-passing runs (or the error code for `error` + * outcomes). `null` / omitted when the attempt passed or the kind is unknown. + */ + failureKind?: string | null; +} + +/** A single non-passing attempt as surfaced in the report. */ +export interface FlakyFailure { + attempt: number; + runId: string | null; + outcome: FlakyOutcome; + failureKind: string | null; +} + +/** + * Machine-readable stability report. This is also the exact `--output json` + * shape, so dashboards / agents / CI can consume it directly. + */ +export interface FlakyReport { + testId: string; + /** + * Attempts actually observed. May be fewer than requested when + * `--until-fail` short-circuits on the first non-passing attempt. + */ + runs: number; + passed: number; + failed: number; + /** `passed / runs`, rounded to 4 decimal places. `0` when `runs === 0`. */ + stableRatio: number; + verdict: FlakyVerdict; + /** Non-passing attempts, in attempt order. */ + failures: FlakyFailure[]; +} + +/** + * Aggregate per-attempt outcomes into a stability report. + * + * Verdict rules: + * - every attempt passed → `stable` + * - no attempt passed → `failing` + * - a mix of pass and non-pass → `flaky` + * An empty attempt list (no runs observed) is reported as `failing` with a + * `0` ratio — there is no evidence the test is stable. + */ +export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyReport { + const runs = attempts.length; + const passed = attempts.filter(a => a.outcome === 'passed').length; + const failed = runs - passed; + const stableRatio = runs === 0 ? 0 : Math.round((passed / runs) * 10000) / 10000; + const verdict: FlakyVerdict = + runs > 0 && passed === runs ? 'stable' : passed === 0 ? 'failing' : 'flaky'; + const failures: FlakyFailure[] = attempts + .filter(a => a.outcome !== 'passed') + .map(a => ({ + attempt: a.attempt, + runId: a.runId, + outcome: a.outcome, + failureKind: a.failureKind ?? null, + })); + return { testId, runs, passed, failed, stableRatio, verdict, failures }; +} + +/** + * Exit code for the command: `0` only when the verdict is `stable` (every + * observed attempt passed). Anything else is non-zero so CI can gate a merge + * on flakiness (`testsprite test flaky --runs 5 || exit 1`). + */ +export function flakyExitCode(report: FlakyReport): number { + return report.verdict === 'stable' ? 0 : 1; +} + +/** Human-readable label for a verdict. */ +function verdictLabel(verdict: FlakyVerdict): string { + switch (verdict) { + case 'stable': + return 'STABLE'; + case 'flaky': + return 'FLAKY'; + case 'failing': + return 'FAILING'; + } +} + +/** + * Render a report to human-readable text. JSON-mode callers ship the report + * verbatim via `out.print`; this is the text-mode rendering. + */ +export function renderFlakyText(report: FlakyReport): string { + const pct = Math.round(report.stableRatio * 100); + const lines: string[] = [ + `Ran ${report.testId} ${report.runs}x — ${report.passed} passed, ${report.failed} failed → ` + + `${verdictLabel(report.verdict)} (${pct}% stable)`, + ]; + if (report.failures.length > 0) { + lines.push(' failed attempts:'); + for (const f of report.failures) { + const kind = f.failureKind ? ` failureKind=${f.failureKind}` : ''; + const rid = f.runId ?? '(no runId)'; + lines.push(` #${f.attempt} ${rid} ${f.outcome}${kind}`); + } + } + return lines.join('\n'); +} diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 3387eac..a9c1193 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -140,6 +140,53 @@ describe('HttpClient happy path', () => { }); }); +describe('HttpClient — 200 response with a non-JSON body', () => { + function htmlResponse(status = 200): Response { + return new Response('Login', { + status, + headers: { 'content-type': 'text/html' }, + }); + } + + it('throws a typed INTERNAL ApiError (not a raw SyntaxError) with the requestId and details', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + let caught: unknown; + try { + await client.get('/me'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('INTERNAL'); + expect(apiErr.exitCode).toBe(1); + expect(apiErr.requestId).toMatch(/^cli_/); + expect(apiErr.message).toContain('non-JSON response'); + expect(apiErr.nextAction).toContain('TESTSPRITE_API_URL'); + expect(apiErr.details).toMatchObject({ httpStatus: 200, contentType: 'text/html' }); + expect(String(apiErr.details.parseError)).toContain('JSON'); + }); + + it('does not retry a malformed 200 body (single fetch call)', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toBeInstanceOf(ApiError); + // A non-JSON success body is a hard config error, not a transient transport + // failure — it must not burn the retry budget. + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('handles an empty 200 body the same way', async () => { + const fetchImpl = vi.fn( + async () => + new Response('', { status: 200, headers: { 'content-type': 'application/json' } }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toMatchObject({ code: 'INTERNAL', exitCode: 1 }); + }); +}); + describe('HttpClient error mapping', () => { it('does not retry AUTH_INVALID and exits 3', async () => { const fetchImpl = vi.fn(async () => errorEnvelopeResponse(401, 'AUTH_INVALID')); @@ -402,6 +449,52 @@ describe('HttpClient per-request timeout', () => { expect(callCount).toBe(1); }); + it('clears the per-attempt timeout after a successful response body is read', async () => { + let capturedSignal: AbortSignal | undefined; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + capturedSignal = init?.signal ?? undefined; + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(capturedSignal?.aborted).toBe(false); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(capturedSignal?.aborted).toBe(false); + }); + + it('clears a failed attempt timeout before retry backoff sleeps', async () => { + const attemptSignals: AbortSignal[] = []; + let calls = 0; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + if (init?.signal) attemptSignals.push(init.signal); + calls += 1; + if (calls === 1) return errorEnvelopeResponse(500, 'INTERNAL'); + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => new Promise(resolve => setTimeout(resolve, 50)), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(attemptSignals).toHaveLength(2); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + }); + it('caller-supplied AbortSignal still propagates as AbortError (not RequestTimeoutError)', async () => { const controller = new AbortController(); // Abort immediately @@ -428,6 +521,33 @@ describe('HttpClient per-request timeout', () => { expect((err as Error).name).toBe('AbortError'); }); + it('preserves RequestTimeoutError when the caller aborts after the request timeout wins', async () => { + const controller = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const reason = init.signal?.reason; + controller.abort(new Error('caller aborted after timeout')); + const err = new Error(reason?.message ?? 'timed out'); + err.name = reason?.name ?? 'TimeoutError'; + reject(err); + }); + }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 1, + }); + const err = await client.get('/me', { signal: controller.signal }).catch(e => e); + expect(err).toBeInstanceOf(RequestTimeoutError); + expect((err as RequestTimeoutError).exitCode).toBe(7); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it('defaults to REQUEST_TIMEOUT_DEFAULT_MS when no requestTimeoutMs is supplied', () => { // Verify the default is wired without actually waiting 120s. // We test via the exported constant rather than exercising the stall. diff --git a/src/lib/http.ts b/src/lib/http.ts index 83ca0f1..af1c25a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -395,9 +395,15 @@ export class HttpClient { timeoutSignal: AbortSignal, callerSignal: AbortSignal | undefined, requestId: string, + effectiveSignal: AbortSignal = timeoutSignal, ): void { if (isAbortError(err) || isTimeoutError(err)) { - if (timeoutSignal.aborted && (callerSignal == null || !callerSignal.aborted)) { + const timeoutWon = + timeoutSignal.aborted && + (callerSignal == null || + !callerSignal.aborted || + effectiveSignal.reason === timeoutSignal.reason); + if (timeoutWon) { throw new RequestTimeoutError(this.requestTimeoutMs, requestId); } throw err; @@ -428,184 +434,200 @@ export class HttpClient { // signal. The polling path supplies its own deadline-aware signal per // iteration — this timeout (120s default) is safely larger than any single // long-poll window (<=25s via ?waitSeconds), so it never bites polling. - const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs); + const requestTimeout = createRequestTimeout(this.requestTimeoutMs); + const timeoutSignal = requestTimeout.signal; const effectiveSignal = options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; try { - response = await this.fetchImpl(url, { - method, - headers: this.buildHeaders(requestId, options), - body: options.body !== undefined ? JSON.stringify(options.body) : undefined, - signal: effectiveSignal, - }); - } catch (err) { - // Distinguish a client-side request timeout from a caller-supplied abort. - // - // Node 22 `AbortSignal.timeout()` throws a `DOMException` with - // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. - // A caller-supplied abort sets `name === 'AbortError'`. - // We treat both abort variants together: if the timeout signal fired and - // the caller hadn't already aborted, surface a clear RequestTimeoutError. - // A timeout/abort during the fetch itself: classify it (RequestTimeoutError - // when our deadline fired; otherwise rethrow the caller's abort unmodified). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // If a RequestTimeoutError already propagated from somewhere (e.g. from a - // nested call or from a test-injected fetchImpl), pass it through unchanged - // rather than re-wrapping it as a TransportError. - if (err instanceof RequestTimeoutError) throw err; - const message = err instanceof Error ? err.message : String(err); - this.debug({ - kind: 'error', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - durationMs: Date.now() - startedAt, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) throw new TransportError(message, requestId); - this.transition( - `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); - continue; - } + try { + response = await this.fetchImpl(url, { + method, + headers: this.buildHeaders(requestId, options), + body: options.body !== undefined ? JSON.stringify(options.body) : undefined, + signal: effectiveSignal, + }); + } catch (err) { + // Distinguish a client-side request timeout from a caller-supplied abort. + // + // The request-timeout controller aborts with an Error/DOMException whose + // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. + // A caller-supplied abort sets `name === 'AbortError'`. + // We treat both abort variants together: if the timeout signal fired and + // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A timeout/abort during the fetch itself: classify it (RequestTimeoutError + // when our deadline fired; otherwise rethrow the caller's abort unmodified). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // If a RequestTimeoutError already propagated from somewhere (e.g. from a + // nested call or from a test-injected fetchImpl), pass it through unchanged + // rather than re-wrapping it as a TransportError. + if (err instanceof RequestTimeoutError) throw err; + const message = err instanceof Error ? err.message : String(err); + this.debug({ + kind: 'error', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + durationMs: Date.now() - startedAt, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) throw new TransportError(message, requestId); + this.transition( + `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - const durationMs = Date.now() - startedAt; - if (response.ok) { - this.debug({ - kind: 'response', - method, - url, - attempt, - status: response.status, - requestId, - durationMs, - }); + const durationMs = Date.now() - startedAt; + if (response.ok) { + this.debug({ + kind: 'response', + method, + url, + attempt, + status: response.status, + requestId, + durationMs, + }); + try { + return { body: (await response.json()) as T, requestId, status: response.status }; + } catch (err) { + // A timeout/abort can fire mid-body-read (headers received, stream stalls). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); + } + } + + let rawBody: unknown; try { - return { body: (await response.json()) as T, requestId, status: response.status }; + rawBody = await safeReadJson(response); } catch (err) { - // A timeout/abort can fire mid-body-read (headers received, stream stalls). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); + // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse + // errors), so a timeout fired mid-body-read on a non-OK response lands here. + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); throw err; } - } - let rawBody: unknown; - try { - rawBody = await safeReadJson(response); - } catch (err) { - // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse - // errors), so a timeout fired mid-body-read on a non-OK response lands here. - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; - } + // Edge proxies / load balancers return 408/502/504 without our error + // envelope on transient outages. Per the CLI error spec §7 these are + // transport-level retries, not facade errors — fold them in here so + // we get the bounded backoff budget instead of a single INTERNAL bail. + if (rawBody === null && isTransportEdgeStatus(response.status)) { + this.debug({ + kind: 'error', + method, + url, + attempt, + status: response.status, + requestId, + errorCode: 'TRANSPORT', + durationMs, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) { + throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); + } + this.transition( + `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. - if (rawBody === null && isTransportEdgeStatus(response.status)) { + const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); + // Clamp server-directed Retry-After to [1s, 300s] and surface on the + // thrown error so outer callers (e.g. runBatchRun outer retry loop) + // can honor it without re-reading the now-consumed HTTP response. + const retryAfterMsForError = + retryAfterSec !== undefined + ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 + : undefined; + const apiError = ApiError.fromEnvelope( + rawBody, + response.status, + retryAfterMsForError, + // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing + // links) resolve the environment-correct portal domain. + this.baseUrl, + ); this.debug({ kind: 'error', method, url, attempt, status: response.status, + errorCode: apiError.code, requestId, - errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) { - throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); - } - this.transition( - `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + const retryOnConflict = options.retryOnConflict !== false; + const retryOnRateLimit = options.retryOnRateLimit !== false; + const decision = apiRetryDecision( + apiError.code, + attempt, + retryAfterSec, + this.random, + retryOnConflict, + retryOnRateLimit, ); + if (!decision.retry) throw apiError; + const delaySec = Math.round(decision.delayMs / 1000); + if (apiError.code === 'RATE_LIMITED') { + this.transition( + `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, + ); + } else if (apiError.code === 'INTERNAL') { + this.transition( + `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } else if (apiError.code === 'UNAVAILABLE') { + this.transition( + `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } this.debug({ kind: 'retry', method, url, attempt, requestId, - errorCode: 'TRANSPORT', + errorCode: apiError.code, delayMs: decision.delayMs, }); + requestTimeout.clear(); await this.sleep(decision.delayMs); - continue; - } - - const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); - // Clamp server-directed Retry-After to [1s, 300s] and surface on the - // thrown error so outer callers (e.g. runBatchRun outer retry loop) - // can honor it without re-reading the now-consumed HTTP response. - const retryAfterMsForError = - retryAfterSec !== undefined ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 : undefined; - const apiError = ApiError.fromEnvelope( - rawBody, - response.status, - retryAfterMsForError, - // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing - // links) resolve the environment-correct portal domain. - this.baseUrl, - ); - this.debug({ - kind: 'error', - method, - url, - attempt, - status: response.status, - errorCode: apiError.code, - requestId, - durationMs, - }); - const retryOnConflict = options.retryOnConflict !== false; - const retryOnRateLimit = options.retryOnRateLimit !== false; - const decision = apiRetryDecision( - apiError.code, - attempt, - retryAfterSec, - this.random, - retryOnConflict, - retryOnRateLimit, - ); - if (!decision.retry) throw apiError; - const delaySec = Math.round(decision.delayMs / 1000); - if (apiError.code === 'RATE_LIMITED') { - this.transition( - `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, - ); - } else if (apiError.code === 'INTERNAL') { - this.transition( - `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } else if (apiError.code === 'UNAVAILABLE') { - this.transition( - `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, - ); + } finally { + requestTimeout.clear(); } - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: apiError.code, - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); } } @@ -681,6 +703,39 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } +interface RequestTimeoutHandle { + signal: AbortSignal; + clear: () => void; +} + +function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { + const controller = new AbortController(); + const timer = setTimeout(() => { + controller.abort(makeTimeoutReason()); + }, timeoutMs); + unrefTimer(timer); + + return { + signal: controller.signal, + clear: () => clearTimeout(timer), + }; +} + +function makeTimeoutReason(): Error { + if (typeof DOMException !== 'undefined') { + return new DOMException('The operation timed out.', 'TimeoutError'); + } + const err = new Error('The operation timed out.'); + err.name = 'TimeoutError'; + return err; +} + +function unrefTimer(timer: ReturnType): void { + if (typeof timer !== 'object' || timer === null || !('unref' in timer)) return; + const unref = (timer as { unref?: () => void }).unref; + if (typeof unref === 'function') unref.call(timer); +} + async function safeReadJson(response: Response): Promise { try { return await response.json(); @@ -692,6 +747,53 @@ async function safeReadJson(response: Response): Promise { } } +/** + * Build a typed error for a successful response whose body could not be parsed + * as JSON. + * + * The CLI expects every API response to be a JSON envelope. When a `200 OK` + * carries a non-JSON body — a misconfigured endpoint, a proxy / captive-portal + * / SSO login page returning HTML with a success status, or an empty body — + * `response.json()` throws a raw `SyntaxError`. Left unhandled it escapes to + * the top-level handler in `index.ts`, which prints a bare + * `{"error":""}` under `--output json` (breaking the + * typed-envelope contract every other error honors) and gives the operator no + * actionable context. + * + * This wraps it in a typed `INTERNAL` `ApiError` (exit 1, unchanged) that + * carries the `requestId`, names the likely cause, and points the operator at + * their endpoint configuration. `details` includes the HTTP status, the + * response `content-type` (when present), and the underlying parse message. + */ +export function malformedResponseError( + response: Response, + requestId: string, + cause: unknown, +): ApiError { + const contentType = response.headers.get('content-type') ?? undefined; + const parseError = cause instanceof Error ? cause.message : String(cause); + const contentTypeNote = contentType ? ` (content-type: ${contentType})` : ''; + return new ApiError( + { + code: 'INTERNAL', + message: + `The server returned a non-JSON response${contentTypeNote} for an HTTP ${response.status}. ` + + `This usually means the endpoint is not the TestSprite API — a proxy, captive portal, or ` + + `login page can return HTML with a success status.`, + nextAction: + 'Check that --endpoint-url / TESTSPRITE_API_URL points at the TestSprite API ' + + '(default https://api.testsprite.com), then retry.', + requestId, + details: { + httpStatus: response.status, + ...(contentType ? { contentType } : {}), + parseError, + }, + }, + response.status, + ); +} + export function parseRetryAfter(headerValue: string | null): number | undefined { if (!headerValue) return undefined; const numeric = Number(headerValue); diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts new file mode 100644 index 0000000..1fcf8eb --- /dev/null +++ b/src/lib/interrupt.test.ts @@ -0,0 +1,119 @@ +import { EventEmitter } from 'node:events'; +import { writeSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import { + SIGINT_EXIT_CODE, + TERMINATION_EXIT_CODES, + formatInterruptMessage, + installBrokenPipeGuard, + installSignalHandlers, +} from './interrupt.js'; + +// installSignalHandlers' default stderr writes via fs.writeSync (synchronous, so +// the hint survives a piped stderr before exit); mock it to assert on that path. +vi.mock('node:fs', async importOriginal => { + const actual = (await importOriginal()) as Record; + return { ...actual, writeSync: vi.fn() }; +}); + +describe('formatInterruptMessage', () => { + it('defaults to SIGINT and explains the run continues server-side', () => { + const message = formatInterruptMessage(); + expect(message).toContain('Interrupted (SIGINT)'); + expect(message).toContain('test wait'); + expect(message).toContain('test list'); + }); + + it('names the specific signal when given one', () => { + expect(formatInterruptMessage('SIGTERM')).toContain('Interrupted (SIGTERM)'); + expect(formatInterruptMessage('SIGHUP')).toContain('Interrupted (SIGHUP)'); + }); +}); + +describe('installSignalHandlers', () => { + it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + }); + + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenLastCalledWith(130); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenLastCalledWith(143); + handlers.get('SIGHUP')!(); + expect(exit).toHaveBeenLastCalledWith(129); + + // Each handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); + expect(SIGINT_EXIT_CODE).toBe(130); + expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); + expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); + }); + + it('writes the hint synchronously via writeSync before exit (survives a piped stderr)', () => { + vi.mocked(writeSync).mockClear(); + const handlers = new Map void>(); + const exit = vi.fn(); + // No stderr dep: exercise the synchronous default path. + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + exit, + }); + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenCalledWith(130); + const written = vi + .mocked(writeSync) + .mock.calls.map(call => String(call[1])) + .join(''); + expect(written).toContain('Interrupted (SIGINT)'); + }); +}); + +describe('installBrokenPipeGuard', () => { + function makeEpipe(): NodeJS.ErrnoException { + return Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + } + + it('exits 0 on stdout EPIPE (clean SIGPIPE-equivalent for `| head`)', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + stdout.emit('error', makeEpipe()); + expect(exit).toHaveBeenCalledWith(0); + }); + + it('re-throws a non-EPIPE stdout error instead of silently swallowing it', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => + stdout.emit('error', Object.assign(new Error('boom'), { code: 'ENOSPC' })), + ).toThrow('boom'); + expect(exit).not.toHaveBeenCalled(); + }); + + it('swallows stderr EPIPE without exiting or throwing', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => stderr.emit('error', makeEpipe())).not.toThrow(); + expect(exit).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts new file mode 100644 index 0000000..cc5b4d5 --- /dev/null +++ b/src/lib/interrupt.ts @@ -0,0 +1,126 @@ +/** + * Process lifecycle hardening: graceful termination signals and broken-pipe. + * + * Termination signals: without a handler, Node terminates the process abruptly + * with no output, so a user (Ctrl+C), a CI runner or `docker stop` (SIGTERM), or + * a closed terminal/SSH session (SIGHUP) that interrupts a long + * `test run --wait` is left unsure whether the run was cancelled or is still + * executing server-side (it is: the CLI only polls; the run lives on the + * backend). The handler prints a one-line explanation plus how to resume, then + * exits with the conventional `128 + signal` code. + * + * Broken pipe: when output is piped to a reader that closes early + * (`testsprite ... | head`), the kernel raises `EPIPE` on the next stdout write. + * Node turns an `'error'` with no listener into an uncaughtException and dumps a + * raw `write EPIPE` stack (exit 1). The guard swallows it and exits 0, the + * conventional SIGPIPE-equivalent result for "the reader went away". + * + * `process` and the streams are injectable so the wiring is unit-testable + * without spawning a subprocess or sending a real signal. + */ + +import { writeSync } from 'node:fs'; + +/** + * Termination signals handled, mapped to their conventional `128 + signum` + * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + +export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { + return ( + `Interrupted (${signal}). Any run already started keeps executing on the server; ` + + 'check it with `testsprite test list` or `testsprite test wait `.' + ); +} + +export interface InterruptDeps { + /** Signal registrar. Defaults to `process.on`. */ + on?: (signal: TerminationSignal, handler: () => void) => void; + /** Line-oriented stderr writer (appends a newline). */ + stderr?: (line: string) => void; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a + * single top-level call in `index.ts`; not designed to be installed twice. + */ +export function installSignalHandlers(deps: InterruptDeps = {}): void { + const on = + deps.on ?? + ((signal: TerminationSignal, handler: () => void) => { + process.on(signal, handler); + }); + const stderr = + deps.stderr ?? + ((line: string) => { + // A signal handler calls process.exit() right after writing, which can + // truncate an async process.stderr.write() when stderr is a pipe. Write + // synchronously so the interrupt hint is flushed before the process exits. + try { + writeSync(process.stderr.fd, `${line}\n`); + } catch { + // Best-effort: if stderr is already gone (EPIPE), still exit cleanly. + } + }); + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { + on(signal, () => { + // Blank line first so the message starts on its own row rather than + // trailing the progress ticker's in-place line. + stderr(''); + stderr(formatInterruptMessage(signal)); + exit(TERMINATION_EXIT_CODES[signal]); + }); + } +} + +export interface BrokenPipeDeps { + /** stdout stream. Defaults to `process.stdout`. */ + stdout?: NodeJS.EventEmitter; + /** stderr stream. Defaults to `process.stderr`. */ + stderr?: NodeJS.EventEmitter; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Guard against `EPIPE` on stdout/stderr so piping to a reader that closes + * early (`testsprite ... | head`) exits cleanly instead of crashing with an + * unhandled `write EPIPE` stack. Only `EPIPE` is swallowed; any other stream + * error is left to surface normally. + */ +export function installBrokenPipeGuard(deps: BrokenPipeDeps = {}): void { + const stdout = deps.stdout ?? process.stdout; + const stderr = deps.stderr ?? process.stderr; + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + stdout.on('error', (error: NodeJS.ErrnoException) => { + // Reader went away (`| head`, `| less` then q): exit cleanly like SIGPIPE + // rather than dumping an unhandled `write EPIPE` stack. Any other stdout + // error is a genuine, actionable failure, so re-throw it (Node's default). + if (error.code === 'EPIPE') { + exit(0); + return; + } + throw error; + }); + stderr.on('error', (error: NodeJS.ErrnoException) => { + // stderr closed: nothing can be reported over it, so swallow EPIPE. Any + // other error re-throws so a genuine failure is not silently hidden. + if (error.code === 'EPIPE') return; + throw error; + }); +} diff --git a/src/lib/junit-report.test.ts b/src/lib/junit-report.test.ts new file mode 100644 index 0000000..d504dce --- /dev/null +++ b/src/lib/junit-report.test.ts @@ -0,0 +1,330 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + escapeXml, + parseJUnitReportFormat, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitTestResult, +} from './junit-report.js'; + +function makeResult(overrides: Partial & { testId: string }): JUnitTestResult { + return { + status: 'passed', + ...overrides, + }; +} + +describe('escapeXml', () => { + it('escapes XML special characters', () => { + expect(escapeXml(`a&bd"e'f`)).toBe('a&b<c>d"e'f'); + }); + + it('leaves plain text unchanged', () => { + expect(escapeXml('test_abc123')).toBe('test_abc123'); + }); +}); + +describe('parseJUnitReportFormat', () => { + it('accepts junit', () => { + expect(parseJUnitReportFormat('junit')).toBe('junit'); + }); + + it('returns undefined for absent value', () => { + expect(parseJUnitReportFormat(undefined)).toBeUndefined(); + expect(parseJUnitReportFormat('')).toBeUndefined(); + }); + + it('rejects unknown formats', () => { + expect(() => parseJUnitReportFormat('html')).toThrowError(ApiError); + try { + parseJUnitReportFormat('html'); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('assertJUnitReportOptions', () => { + it('allows absent report flags', () => { + expect(() => assertJUnitReportOptions({ wait: false, batchPath: true })).not.toThrow(); + }); + + it('rejects report-file without report', () => { + expect(() => + assertJUnitReportOptions({ reportFile: './out.xml', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); + + it('rejects report-suite-name without report', () => { + expect(() => + assertJUnitReportOptions({ + reportSuiteName: 'my-suite', + wait: true, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report on non-batch paths', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: true, + batchPath: false, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without wait', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: false, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without report-file', () => { + expect(() => + assertJUnitReportOptions({ report: 'junit', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); +}); + +describe('resolveBatchReportProjectId', () => { + it('prefers explicit projectId', () => { + expect(resolveBatchReportProjectId({ projectId: 'proj_a' }, [])).toBe('proj_a'); + }); + + it('infers from polled run rows', () => { + expect(resolveBatchReportProjectId({}, [{ projectId: 'proj_from_run' }])).toBe('proj_from_run'); + }); + + it('requires --project when the project cannot be inferred', () => { + expect(() => resolveBatchReportProjectId({}, [])).toThrowError(ApiError); + try { + resolveBatchReportProjectId({}, []); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('buildJUnitReport', () => { + it('renders an empty suite', () => { + const xml = buildJUnitReport({ + suiteName: 'Dry suite', + classname: 'proj_empty', + results: [], + }); + expect(xml).toContain( + ''); + }); + + it('counts passed tests without child elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [makeResult({ testId: 'test_a', status: 'passed' })], + }); + expect(xml).toContain(''); + expect(xml).not.toContain(' { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 'test_fail', + status: 'failed', + runId: 'run_1', + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('runId: run_1'); + expect(xml).toContain('failures="1"'); + }); + + it('maps blocked and cancelled to failures', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ testId: 't_blocked', status: 'blocked' }), + makeResult({ testId: 't_cancelled', status: 'cancelled' }), + ], + }); + expect(xml).toContain('failures="2"'); + expect(xml).toContain('type="blocked"'); + expect(xml).toContain('type="cancelled"'); + }); + + it('maps timeout to failure', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_timeout', + status: 'timeout', + error: { code: 'UNSUPPORTED', message: 'Timed out', exitCode: 7 }, + }), + ], + }); + expect(xml).toContain(''); + }); + + it('maps API error status to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_err', + status: 'error', + error: { code: 'NOT_FOUND', message: 'Run missing', exitCode: 4 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('errors="1"'); + }); + + it('maps auth failures to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_auth', + status: 'failed', + error: { code: 'AUTH_INVALID', message: 'Bad key', exitCode: 3 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('failures="0" errors="1"'); + }); + + it('escapes special characters in testcase names and messages', () => { + const xml = buildJUnitReport({ + suiteName: 'Suite "A"', + classname: 'proj<&>', + results: [ + makeResult({ + testId: 'test<1>', + status: 'failed', + error: { code: 'ASSERT', message: 'expected & "ok"', exitCode: 1 }, + }), + ], + }); + expect(xml).toContain('name="test<1>"'); + expect(xml).toContain('classname="proj<&>"'); + expect(xml).toContain('message="expected <true> & "ok""'); + }); + + it('aggregates mixed outcomes', () => { + const xml = buildJUnitReport({ + suiteName: 'Mixed', + classname: 'proj_mix', + results: [ + makeResult({ testId: 'p', status: 'passed' }), + makeResult({ testId: 'f', status: 'failed' }), + makeResult({ + testId: 'e', + status: 'error', + error: { code: 'INTERNAL', message: 'boom', exitCode: 10 }, + }), + ], + }); + expect(xml).toContain('tests="3" failures="1" errors="1" skipped="0"'); + }); +}); + +describe('writeJUnitReportFile', () => { + it('writes XML atomically to the target path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-')); + const target = join(dir, 'results.xml'); + const xml = buildJUnitReport({ + suiteName: 'Suite', + classname: 'proj_write', + results: [makeResult({ testId: 't1', status: 'passed' })], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects a directory target', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-dir-')); + await expect(writeJUnitReportFile(dir, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects missing parent directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-parent-')); + const missing = join(dir, 'missing', 'out.xml'); + await expect(writeJUnitReportFile(missing, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('overwrites an existing file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-overwrite-')); + const target = join(dir, 'results.xml'); + writeFileSync(target, 'old', 'utf8'); + const xml = buildJUnitReport({ + suiteName: 'New', + classname: 'proj_new', + results: [], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects empty path', async () => { + await expect(writeJUnitReportFile('', '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('rejects parent that is a file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-file-parent-')); + const parentFile = join(dir, 'not-a-dir'); + writeFileSync(parentFile, 'x', 'utf8'); + const target = join(parentFile, 'out.xml'); + await expect(writeJUnitReportFile(target, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/lib/junit-report.ts b/src/lib/junit-report.ts new file mode 100644 index 0000000..3b11442 --- /dev/null +++ b/src/lib/junit-report.ts @@ -0,0 +1,267 @@ +import { createWriteStream } from 'node:fs'; +import { rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { localValidationError, TransportError } from './errors.js'; + +export type JUnitReportFormat = 'junit'; + +/** Minimal testcase input shared by batch run and batch rerun poll results. */ +export interface JUnitTestResult { + testId: string; + runId?: string; + status: string; + /** Observed on polled runs; used for classname when --project is omitted. */ + projectId?: string; + error?: { code: string; message: string; exitCode?: number }; +} + +export interface JUnitReportBuildOptions { + suiteName: string; + classname: string; + results: readonly JUnitTestResult[]; +} + +export interface JUnitReportFlagOptions { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + wait: boolean; + /** True when the invocation is a batch path (run --all or rerun batch). */ + batchPath: boolean; +} + +const XML_DECL = ''; + +/** + * Parse `--report `. Only `junit` is accepted in v1. + */ +export function parseJUnitReportFormat(raw: string | undefined): JUnitReportFormat | undefined { + if (raw === undefined || raw === '') return undefined; + if (raw === 'junit') return 'junit'; + throw localValidationError('report', `unsupported report format "${raw}" — accepted: junit`); +} + +/** + * Validate `--report` / `--report-file` / `--report-suite-name` combinations. + * Report export is a sidecar artifact for batch `--wait` runs only. + */ +export function assertJUnitReportOptions(opts: JUnitReportFlagOptions): void { + if (opts.report === undefined) { + if (opts.reportFile !== undefined && opts.reportFile !== '') { + throw localValidationError('report-file', '--report-file requires --report junit'); + } + if (opts.reportSuiteName !== undefined && opts.reportSuiteName !== '') { + throw localValidationError( + 'report-suite-name', + '--report-suite-name requires --report junit', + ); + } + return; + } + + if (!opts.batchPath) { + throw localValidationError( + 'report', + '--report junit only applies to batch --wait runs (test run --all, or test rerun --all / multiple test ids)', + ); + } + if (!opts.wait) { + throw localValidationError( + 'report', + '--report junit requires --wait (the report is written after batch polling completes)', + ); + } + if (opts.reportFile === undefined || opts.reportFile === '') { + throw localValidationError('report-file', '--report junit requires --report-file '); + } +} + +/** + * Resolve the project id used for JUnit classname / default suite naming. + * Prefer explicit `--project`, then ids observed on polled run rows. + */ +export function resolveBatchReportProjectId( + opts: { projectId?: string }, + results: ReadonlyArray<{ projectId?: string }>, +): string { + if (opts.projectId) return opts.projectId; + const fromPoll = results.map(r => r.projectId).find((id): id is string => !!id); + if (fromPoll) return fromPoll; + throw localValidationError( + 'project', + '--report junit requires --project when the project cannot be inferred from run results', + ); +} + +/** + * Escape text for inclusion in XML element bodies and double-quoted attributes. + */ +export function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +type JUnitOutcome = 'passed' | 'failure' | 'error' | 'skipped'; + +function classifyJUnitOutcome(status: string, error?: JUnitTestResult['error']): JUnitOutcome { + if (status === 'passed') return 'passed'; + if (status === 'skipped') return 'skipped'; + if (status === 'error' || error?.exitCode === 3) return 'error'; + return 'failure'; +} + +function failureMessage(result: JUnitTestResult): string { + if (result.error?.message) return result.error.message; + if (result.error?.code) return result.error.code; + return result.status; +} + +function renderTestcase(result: JUnitTestResult, classname: string): string { + const outcome = classifyJUnitOutcome(result.status, result.error); + const name = escapeXml(result.testId); + const cls = escapeXml(classname); + const lines = [` `]; + + if (outcome === 'failure') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.status); + const body = escapeXml( + [ + `status: ${result.status}`, + result.runId ? `runId: ${result.runId}` : undefined, + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'error') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.error?.code ?? result.status); + const body = escapeXml( + [ + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + result.runId ? `runId: ${result.runId}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'skipped') { + lines.push(` `); + } + + lines.push(' '); + return lines.join('\n'); +} + +/** + * Build a JUnit XML document from batch poll results. Duration is `0` in v1 + * because batch poll envelopes do not carry per-run timing. + */ +export function buildJUnitReport(opts: JUnitReportBuildOptions): string { + const results = opts.results; + let failures = 0; + let errors = 0; + let skipped = 0; + + for (const result of results) { + const outcome = classifyJUnitOutcome(result.status, result.error); + if (outcome === 'failure') failures++; + else if (outcome === 'error') errors++; + else if (outcome === 'skipped') skipped++; + } + + const suiteName = escapeXml(opts.suiteName); + const testcases = results.map(r => renderTestcase(r, opts.classname)).join('\n'); + + return [ + XML_DECL, + '', + ` `, + testcases, + ' ', + '', + '', + ].join('\n'); +} + +async function assertReportFileParent(rawPath: string): Promise { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('report-file', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + if (resolved.endsWith('/') || resolved.endsWith('\\')) { + throw localValidationError('report-file', 'must point to a file, not a directory'); + } + + const parent = dirname(resolved); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('report-file', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('report-file', `parent path is not a directory: ${parent}`); + } + + let targetStat; + try { + targetStat = await stat(resolved); + } catch { + return resolved; + } + if (targetStat.isDirectory()) { + throw localValidationError('report-file', `must point to a file, not a directory: ${resolved}`); + } + return resolved; +} + +/** + * Atomically write JUnit XML to `--report-file` (temp sibling + rename). + */ +export async function writeJUnitReportFile(rawPath: string, xml: string): Promise { + const resolved = await assertReportFileParent(rawPath); + const parent = dirname(resolved); + const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); + + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); + let streamError: Error | null = null; + stream.on('error', err => { + streamError = err instanceof Error ? err : new Error(String(err)); + }); + stream.write(xml, err => { + if (err) { + streamError = err instanceof Error ? err : new Error(String(err)); + } + stream.end(() => { + if (streamError) { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError(`Failed to write --report-file ${resolved}: ${streamError.message}`), + ); + return; + } + rename(tmpPath, resolved) + .then(() => resolvePromise()) + .catch(renameErr => { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError( + `Failed to write --report-file ${resolved}: ${renameErr instanceof Error ? renameErr.message : String(renameErr)}`, + ), + ); + }); + }); + }); + }); +} diff --git a/src/lib/output.test.ts b/src/lib/output.test.ts index b572bda..bab3b53 100644 --- a/src/lib/output.test.ts +++ b/src/lib/output.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { Output, isOutputMode } from './output.js'; +import { Output, isOutputMode, resolveOutputMode } from './output.js'; +import { ApiError } from './errors.js'; describe('isOutputMode', () => { it('accepts json and text', () => { @@ -15,6 +16,36 @@ describe('isOutputMode', () => { }); }); +describe('resolveOutputMode', () => { + it('returns the mode verbatim for valid values', () => { + expect(resolveOutputMode('json')).toBe('json'); + expect(resolveOutputMode('text')).toBe('text'); + }); + + it('defaults to text when the flag is omitted (undefined)', () => { + expect(resolveOutputMode(undefined)).toBe('text'); + }); + + it('throws a typed VALIDATION_ERROR (exit 5) instead of silently falling back to text', () => { + // The footgun this guards against: an agent that asks for `--output json` + // but mistypes it would otherwise receive a text payload and fail to parse + // it as JSON with no signal. Every command group must reject, not coerce. + for (const bad of ['josn', 'yaml', 'JSON', 'Text', '']) { + let caught: unknown; + try { + resolveOutputMode(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('must be one of: json, text'); + } + }); +}); + describe('Output', () => { let logSpy: ReturnType; let errorSpy: ReturnType; diff --git a/src/lib/output.ts b/src/lib/output.ts index 2b2db39..4d59cca 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -1,18 +1,40 @@ +import { localValidationError } from './errors.js'; + export type OutputMode = 'json' | 'text'; /** * Help-text footer pointing at the global options surface so users * looking at any subcommand `--help` don't miss `--dry-run`, `--output`, - * `--profile`, `--endpoint-url`, `--debug`. + * `--profile`, `--endpoint-url`, `--request-timeout`, `--debug`. */ export const GLOBAL_OPTS_HINT = - '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug):' + + '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug):' + '\n testsprite --help'; export function isOutputMode(value: unknown): value is OutputMode { return value === 'json' || value === 'text'; } +/** + * Resolve a raw `--output` flag value to a concrete {@link OutputMode}. + * + * `undefined` (flag omitted) resolves to the default `'text'`. Any other + * value that is not `'json'` or `'text'` throws a typed VALIDATION_ERROR + * (exit 5) with an actionable message. + * + * The alternative — silently falling back to `'text'` — is a footgun for the + * CLI's primary consumer (coding agents): a caller that asks for + * `--output json` but mistypes it (`--output josn`) would otherwise receive a + * human-readable text payload and fail to parse it as JSON, with no signal as + * to why. Every command group routes its global-option resolution through this + * helper so the validation is uniform. + */ +export function resolveOutputMode(raw: unknown): OutputMode { + if (raw === undefined) return 'text'; + if (isOutputMode(raw)) return raw; + throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); +} + export interface OutputStreams { /** * Line-oriented stdout writer. Each call is one logical line; the diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index 33fd735..8a20295 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; import { ApiError } from './errors.js'; -import { paginate, validatePaginationFlags, type FetchPage, type Page } from './pagination.js'; +import { + MAX_AUTO_PAGES, + paginate, + validatePaginationFlags, + type FetchPage, + type Page, +} from './pagination.js'; function makePages(pages: Page[]): { fetchPage: FetchPage; @@ -43,10 +49,18 @@ describe('validatePaginationFlags', () => { expect(() => validatePaginationFlags({ pageSize: Number.NaN })).toThrow(ApiError); }); + it('rejects fractional pageSize values', () => { + expect(() => validatePaginationFlags({ pageSize: 1.5 })).toThrow(ApiError); + }); + it('rejects maxItems=0', () => { expect(() => validatePaginationFlags({ maxItems: 0 })).toThrow(ApiError); }); + it('rejects fractional maxItems values', () => { + expect(() => validatePaginationFlags({ maxItems: 2.5 })).toThrow(ApiError); + }); + it('accepts pageSize=100 (the hard cap)', () => { expect(() => validatePaginationFlags({ pageSize: 100 })).not.toThrow(); }); @@ -113,4 +127,55 @@ describe('paginate', () => { await expect(paginate(fetchPage, { pageSize: 0 })).rejects.toBeInstanceOf(ApiError); expect(calls).toHaveLength(0); }); + + it('continues through empty cursor pages until later data', async () => { + const { fetchPage, calls } = makePages([ + { items: [], nextToken: 'cursor-1' }, + { items: [1], nextToken: null }, + ]); + + const page = await paginate(fetchPage); + + expect(page.items).toEqual([1]); + expect(page.nextToken).toBeNull(); + expect(calls).toHaveLength(2); + expect(calls[1]!.cursor).toBe('cursor-1'); + }); + + it('rejects when API repeats a non-null cursor without making progress', async () => { + const { fetchPage, calls } = makePages([{ items: [], nextToken: 'cursor-1' }]); + + await expect(paginate(fetchPage)).rejects.toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'repeated_next_token' }), + }); + + expect(calls).toHaveLength(2); + }); + + it('rejects when auto-pagination exceeds the page safety cap', async () => { + const calls: Array<{ pageSize: number; cursor: string | undefined }> = []; + const fetchPage: FetchPage = async args => { + calls.push(args); + return { + items: [], + nextToken: + args.cursor === undefined ? 'cursor-1' : `cursor-${Number(args.cursor.slice(7)) + 1}`, + }; + }; + + let thrown: unknown; + try { + await paginate(fetchPage); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect(thrown).toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'max_pages_exceeded' }), + }); + expect(calls).toHaveLength(MAX_AUTO_PAGES); + }); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index 59dbc12..cae7581 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -1,5 +1,5 @@ import type { HttpClient } from './http.js'; -import { localValidationError } from './errors.js'; +import { ApiError, localValidationError } from './errors.js'; /** * Page shape returned by every list endpoint per @@ -22,14 +22,15 @@ export interface PaginationFlags { const HARD_PAGE_SIZE_CAP = 100; const DEFAULT_PAGE_SIZE = 25; +export const MAX_AUTO_PAGES = 1000; /** * Validates and normalizes pagination flags. Per the CLI OpenAPI spec * §components.parameters.PageSize the hard cap is 100. Values above the * cap are now rejected with exit 5 rather than silently clamped, giving - * callers fast feedback that their flag value is out of range. Sub-1 / - * NaN values also throw. `maxItems` is validated but not capped (it is a - * client-side cursor, not a server parameter). + * callers fast feedback that their flag value is out of range. Fractional, + * sub-1, and NaN values also throw. `maxItems` is validated but not capped + * (it is a client-side cursor, not a server parameter). * * NOTE: `runResultHistory` previously did its own silent clamp via * `Math.min(Math.max(1, n), 100)` — that was unified to this path by the @@ -38,7 +39,7 @@ const DEFAULT_PAGE_SIZE = 25; export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags { const out: PaginationFlags = { ...flags }; if (out.pageSize !== undefined) { - if (!Number.isFinite(out.pageSize) || out.pageSize < 1) { + if (!Number.isFinite(out.pageSize) || !Number.isInteger(out.pageSize) || out.pageSize < 1) { throw localValidationError( 'page-size', `must be a positive integer between 1 and ${HARD_PAGE_SIZE_CAP}`, @@ -52,7 +53,7 @@ export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags } } if (out.maxItems !== undefined) { - if (!Number.isFinite(out.maxItems) || out.maxItems < 1) { + if (!Number.isFinite(out.maxItems) || !Number.isInteger(out.maxItems) || out.maxItems < 1) { throw localValidationError('maxItems', 'must be a positive integer'); } } @@ -91,14 +92,24 @@ export async function paginate( const items: T[] = []; let cursor: string | undefined = flags.startingToken; let lastNextToken: string | null = null; + let pagesFetched = 0; + const seenNextTokens = new Set(); + if (cursor !== undefined) seenNextTokens.add(cursor); while (true) { const remaining = maxItems !== undefined ? maxItems - items.length : Infinity; if (remaining <= 0) break; + if (pagesFetched >= MAX_AUTO_PAGES) { + throw paginationSafetyError('max_pages_exceeded', { + maxPages: MAX_AUTO_PAGES, + lastCursor: cursor ?? null, + }); + } const callPageSize = Number.isFinite(remaining) ? Math.min(pageSize, remaining) : pageSize; const page = await fetchPage({ pageSize: callPageSize, cursor }); + pagesFetched += 1; lastNextToken = page.nextToken; for (const item of page.items) { @@ -107,12 +118,30 @@ export async function paginate( } if (page.nextToken === null) break; + if (seenNextTokens.has(page.nextToken)) { + throw paginationSafetyError('repeated_next_token', { + cursor: page.nextToken, + pagesFetched, + }); + } + seenNextTokens.add(page.nextToken); cursor = page.nextToken; } return { items, nextToken: lastNextToken }; } +function paginationSafetyError(reason: string, details: Record): ApiError { + return ApiError.fromEnvelope({ + code: 'UNAVAILABLE', + message: 'Pagination did not make progress safely.', + nextAction: + 'Retry later. If the problem continues, contact TestSprite support with the cursor details.', + requestId: 'local', + details: { reason, ...details }, + }); +} + /** * Drop-in helper for commands that take a single page and surface * the cursor verbatim (no auto-follow). Used when the caller passed diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index a552882..bb8cfb5 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -39,11 +39,63 @@ describe('promptText', () => { expect(await promptText('? ', { input, output })).toBe('ok'); }); + it('preserves buffered answers for sequential prompts on the same stream', async () => { + const input = Readable.from(['first\nsecond\nthird\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + await expect(promptText('Three: ', { input, output })).resolves.toBe('third'); + }); + + it('uses buffered tail input at EOF for a following prompt', async () => { + const input = Readable.from(['first\nsecond']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + + it('preserves buffered CRLF answers for sequential prompts', async () => { + const input = Readable.from(['first\r\nsecond\r\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + it('returns the buffered input on stream end without newline', async () => { const input = Readable.from(['eof-no-newline']); const output = new CaptureStream(); expect(await promptText('? ', { input, output })).toBe('eof-no-newline'); }); + + it('writes the question to stderr by default — keeps stdout pure for --output json', async () => { + // Regression: prompts used to default to stdout, which polluted the JSON + // result on the interactive setup/configure path. Interactive UI belongs + // on stderr. Manually swap the global stream writers (prompt reads + // process.stderr/stdout at call time, so this is reliably intercepted). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptText('Q: ', { input: Readable.from(['x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Q: '); + expect(outChunks.join('')).not.toContain('Q: '); + }); }); describe('promptSecret (non-TTY behavior)', () => { @@ -63,6 +115,35 @@ describe('promptSecret (non-TTY behavior)', () => { expect(written).not.toContain('sk-hidden-12345'); }); + it('writes the prompt to stderr by default — keeps stdout pure for --output json', async () => { + // Same regression as promptText: the secret prompt is interactive UI and + // must default to stderr so stdout carries only the command result. Manual + // stream swap (vi.spyOn does not intercept process.stderr.write here). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptSecret('Key: ', { input: Readable.from(['sk-x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Key: '); + expect(outChunks.join('')).not.toContain('Key: '); + // The typed secret is never echoed to either real stream. + expect(errChunks.join('')).not.toContain('sk-x'); + expect(outChunks.join('')).not.toContain('sk-x'); + }); + it('honors DEL/backspace before submission', async () => { const DEL = String.fromCharCode(0x7f); const input = Readable.from([`abc${DEL}d\n`]); @@ -70,6 +151,16 @@ describe('promptSecret (non-TTY behavior)', () => { expect(await promptSecret('? ', { input, output })).toBe('abd'); }); + it('preserves buffered secret answers for sequential prompts', async () => { + const input = Readable.from(['sk-one\nsk-two\n']); + const output = new CaptureStream(); + + await expect(promptSecret('First key: ', { input, output })).resolves.toBe('sk-one'); + await expect(promptSecret('Second key: ', { input, output })).resolves.toBe('sk-two'); + expect(output.text()).not.toContain('sk-one'); + expect(output.text()).not.toContain('sk-two'); + }); + it('rejects on Ctrl-C input', async () => { const ETX = String.fromCharCode(0x03); const input = Readable.from([`abc${ETX}`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 96fca29..bccead9 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -11,16 +11,24 @@ interface RawModeCapable { resume?: () => unknown; } +const pendingPromptInput = new WeakMap(); + export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // Prompts are interactive UI, not data — write the question (and any echo) + // to stderr so stdout carries only the command's result. This keeps + // `--output json` stdout a single pure JSON document even on the interactive + // setup / configure path (§8.1 stdout purity). stderr is still the user's + // TTY, so the prompt remains visible. + const output = streams.output ?? process.stderr; output.write(question); return readLine(input, output, false); } export async function promptSecret(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // See promptText: interactive prompt + masking go to stderr, not stdout. + const output = streams.output ?? process.stderr; output.write(question); const inputAsTTY = input as Readable & RawModeCapable; @@ -41,18 +49,29 @@ function readLine( return new Promise((resolve, reject) => { let buffer = ''; let resolved = false; + let listening = false; const onData = (chunk: Buffer | string): void => { const str = typeof chunk === 'string' ? chunk : chunk.toString('utf-8'); - for (const ch of str) { - const code = ch.charCodeAt(0); + processText(str); + }; + + const processText = (str: string): void => { + for (let i = 0; i < str.length; i += 1) { + const code = str.charCodeAt(i); // Enter (CR or LF) if (code === 13 || code === 10) { + let nextIndex = i + 1; + if (code === 13 && str.charCodeAt(nextIndex) === 10) { + nextIndex += 1; + } + savePendingInput(str.slice(nextIndex)); finish(); return; } // Ctrl-C if (code === 3) { + savePendingInput(''); cleanup(); output.write('\n'); if (!resolved) { @@ -71,7 +90,7 @@ function readLine( } // Drop other control chars if (code < 32) continue; - buffer += ch; + buffer += str[i]; if (mask) output.write('*'); } }; @@ -89,9 +108,12 @@ function readLine( }; const cleanup = (): void => { - input.off('data', onData); - input.off('end', onEnd); - input.off('error', onError); + if (listening) { + input.off('data', onData); + input.off('end', onEnd); + input.off('error', onError); + listening = false; + } const pausable = input as { pause?: () => unknown }; if (typeof pausable.pause === 'function') pausable.pause(); }; @@ -105,12 +127,37 @@ function readLine( } }; + const savePendingInput = (text: string): void => { + if (text.length > 0) { + pendingPromptInput.set(input, text); + } else { + pendingPromptInput.delete(input); + } + }; + + const pending = pendingPromptInput.get(input); + if (pending !== undefined) { + pendingPromptInput.delete(input); + processText(pending); + if (resolved) return; + if (isInputEnded(input)) { + finish(); + return; + } + } + input.on('data', onData); input.on('end', onEnd); input.on('error', onError); + listening = true; const resumable = input as { resume?: () => unknown }; if (typeof resumable.resume === 'function') resumable.resume(); }); } +function isInputEnded(input: NodeJS.ReadableStream): boolean { + const state = input as { readableEnded?: boolean; destroyed?: boolean; closed?: boolean }; + return state.readableEnded === true || state.destroyed === true || state.closed === true; +} + export type { Writable }; diff --git a/src/lib/proxy.test.ts b/src/lib/proxy.test.ts new file mode 100644 index 0000000..5c686af --- /dev/null +++ b/src/lib/proxy.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; +import { maybeInstallProxyAgent } from './proxy.js'; + +describe('maybeInstallProxyAgent', () => { + it('installs an agent when HTTPS_PROXY is set', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); + + it.each(['https_proxy', 'HTTP_PROXY', 'http_proxy'] as const)( + 'also honors the %s spelling', + name => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { [name]: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }, + ); + + it('does nothing when no proxy variable is set (default path unchanged)', () => { + const install = vi.fn(); + expect(maybeInstallProxyAgent({ env: {}, install })).toBe(false); + expect(maybeInstallProxyAgent({ env: { HTTPS_PROXY: '' }, install })).toBe(false); + expect(install).not.toHaveBeenCalled(); + }); + + it('falls back (returns false, warns, never throws) when installing the agent fails', () => { + // A malformed/unsupported proxy value makes the agent throw at startup; the + // CLI must degrade to a proxy-less dispatcher, not crash every command. + const errs: string[] = []; + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install: () => { + throw new Error('unsupported proxy scheme'); + }, + stderr: line => errs.push(line), + }); + expect(installed).toBe(false); + expect(errs.join('\n')).toContain('ignoring proxy environment'); + }); + + it('still installs when NO_PROXY is set (exemptions are applied per request by undici)', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080', NO_PROXY: 'localhost,127.0.0.1' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/proxy.ts b/src/lib/proxy.ts new file mode 100644 index 0000000..b536761 --- /dev/null +++ b/src/lib/proxy.ts @@ -0,0 +1,58 @@ +/** + * Proxy support (issue #119): honor HTTPS_PROXY / HTTP_PROXY / NO_PROXY. + * + * Node's built-in fetch (undici) deliberately ignores the proxy environment + * variables, so behind a corporate or CI proxy every request dies with + * `fetch failed` after a full retry cycle. Installing undici's + * `EnvHttpProxyAgent` as the global dispatcher restores the conventional + * behavior (including NO_PROXY exemptions) for every fetch the CLI makes. + * + * Only active when a proxy env var is actually present, so the default path + * stays byte-identical and pays zero startup cost. Dependency note for + * reviewers: this adds `undici` as an explicit runtime dependency (the same + * engine Node already bundles); the alternative, a hand-rolled CONNECT + * tunnel, would re-implement what undici ships and maintains. + */ +import type { Dispatcher } from 'undici'; +import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'; + +export interface ProxyDeps { + env?: NodeJS.ProcessEnv; + /** Dispatcher installer. Defaults to undici's setGlobalDispatcher. */ + install?: (agent: Dispatcher) => void; + /** Warning sink. Defaults to `process.stderr`. */ + stderr?: (line: string) => void; +} + +/** + * Install the env-driven proxy dispatcher when any proxy variable is set + * (both canonical upper-case and conventional lower-case spellings). + * Returns whether an agent was installed (observable for tests/debugging). + */ +export function maybeInstallProxyAgent(deps: ProxyDeps = {}): boolean { + const env = deps.env ?? process.env; + const hasProxy = [env.HTTPS_PROXY, env.https_proxy, env.HTTP_PROXY, env.http_proxy].some( + value => typeof value === 'string' && value.length > 0, + ); + if (!hasProxy) return false; + const install = deps.install ?? setGlobalDispatcher; + // EnvHttpProxyAgent reads HTTPS_PROXY/HTTP_PROXY/NO_PROXY itself, per + // request, so NO_PROXY exemptions apply without extra plumbing here. + // + // A malformed or unsupported proxy value (e.g. `socks5://...`) makes the + // agent throw. Because this runs at startup, an unguarded throw would abort + // every command before the CLI's own error handling — so fall back to the + // default (proxy-less) dispatcher and warn instead of crashing. + try { + install(new EnvHttpProxyAgent()); + return true; + } catch (error) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `warning: ignoring proxy environment (could not initialize proxy agent): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return false; + } +} diff --git a/src/lib/render-error.test.ts b/src/lib/render-error.test.ts index 2b1ced7..9635f2b 100644 --- a/src/lib/render-error.test.ts +++ b/src/lib/render-error.test.ts @@ -37,6 +37,12 @@ describe('rephraseUnknownOption', () => { expect(result).toContain('--endpoint-url'); }); + it('rephrases --request-timeout placed after subcommand', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('--request-timeout'); + }); + it('rephrases --debug placed after subcommand', () => { const result = rephraseUnknownOption("error: unknown option '--debug'"); expect(result).not.toBeNull(); @@ -87,6 +93,12 @@ describe('rephraseUnknownOption', () => { expect(result).not.toBeNull(); expect(result).toContain('testsprite --endpoint-url '); }); + + it('value flag (--request-timeout) example DOES include a placeholder', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('testsprite --request-timeout '); + }); }); describe('renderCommanderError', () => { diff --git a/src/lib/render-error.ts b/src/lib/render-error.ts index ebb4208..1898acd 100644 --- a/src/lib/render-error.ts +++ b/src/lib/render-error.ts @@ -11,13 +11,15 @@ import type { OutputMode } from './output.js'; * * `boolean` flags (--dry-run, --debug, --verbose) take no value; emit * example without a placeholder. `value` flags (--output, --profile, - * --endpoint-url) take a single argument; emit example with ``. + * --endpoint-url, --request-timeout) take a single argument; emit example + * with ``. */ const GLOBAL_FLAG_ARITY: Record = { 'dry-run': 'boolean', output: 'value', profile: 'value', 'endpoint-url': 'value', + 'request-timeout': 'value', debug: 'boolean', verbose: 'boolean', }; diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 34f043b..c15b09d 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import type { OutputMode } from './output.js'; import { SKILL_NUDGE_COMMANDS, @@ -36,10 +36,17 @@ describe('isVerifySkillInstalled', () => { it('true when AGENTS.md exists AND carries our BEGIN sentinel', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); - const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n`; + const readFileSync = () => + `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n${MANAGED_SECTION_END}\n`; expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(true); }); + it('false when AGENTS.md has only the BEGIN sentinel without a complete managed section', () => { + const existsSync = (p: string) => p.endsWith('AGENTS.md'); + const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...partial skill...\n`; + expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(false); + }); + it('false when only a bare AGENTS.md (no sentinel) exists', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); const readFileSync = () => '# my project\nNothing TestSprite here.\n'; diff --git a/src/lib/skill-nudge.ts b/src/lib/skill-nudge.ts index 1afc863..0f67572 100644 --- a/src/lib/skill-nudge.ts +++ b/src/lib/skill-nudge.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import { defaultCredentialsPath, readProfile } from './credentials.js'; import type { OutputMode } from './output.js'; @@ -44,8 +44,8 @@ export interface SkillPresenceDeps { * True if the `testsprite-verify` skill is installed for ANY supported agent in * `dir`. own-file targets (claude/cursor/cline/antigravity): the landing file * exists. managed-section target (codex / AGENTS.md): the file exists AND - * carries our BEGIN sentinel — a user-authored AGENTS.md without the sentinel - * does NOT count as our skill. + * carries one complete managed section — a user-authored AGENTS.md without the + * sentinels, or with a truncated section, does NOT count as our skill. * * The TARGETS table is the single source of truth for landing paths, so this * stays in lockstep with `agent install` without re-listing paths. Best-effort: @@ -59,7 +59,7 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} if (!exists(full)) continue; if (spec.mode === 'managed-section') { try { - if (read(full).includes(MANAGED_SECTION_BEGIN)) return true; + if (hasCompleteManagedSection(read(full))) return true; } catch { // unreadable AGENTS.md → treat this target as absent, keep checking } @@ -70,6 +70,14 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} return false; } +/** True when a managed-section file contains an ordered TestSprite BEGIN/END pair. */ +function hasCompleteManagedSection(content: string): boolean { + const begin = content.indexOf(MANAGED_SECTION_BEGIN); + if (begin === -1) return false; + const end = content.indexOf(MANAGED_SECTION_END, begin + MANAGED_SECTION_BEGIN.length); + return end !== -1; +} + export interface SkillNudgeContext { /** Full command path, e.g. "test run" / "auth whoami". */ commandPath: string; @@ -133,6 +141,7 @@ export function maybeEmitSkillNudge(ctx: SkillNudgeContext): void { } } +/** Interpret common env-var spellings for an enabled opt-out flag. */ function isTruthyEnv(v: string | undefined): boolean { if (v === undefined) return false; const s = v.trim().toLowerCase(); diff --git a/src/lib/target-url.spec.ts b/src/lib/target-url.spec.ts index 96f705b..4998bc3 100644 --- a/src/lib/target-url.spec.ts +++ b/src/lib/target-url.spec.ts @@ -210,6 +210,31 @@ describe('assertNotLocal — IPv6 hardening (SSRF bypass guard)', () => { }); }); +describe('assertNotLocal — trailing-dot FQDN normalization (SSRF bypass guard)', () => { + // `localhost.` is the fully-qualified form of `localhost` (RFC 6761 reserves + // both to resolve to loopback). It previously bypassed the + // `host === 'localhost'` check because the WHATWG URL parser keeps the + // trailing dot on named hosts (IP literals are dot-normalized, named hosts + // are not). + it('blocks http://localhost. (trailing-dot loopback)', () => { + expectBlocked('http://localhost.'); + }); + + it('blocks http://localhost.:8080 (trailing-dot loopback with port)', () => { + expectBlocked('http://localhost.:8080'); + }); + + it('blocks http://localhost%2e (percent-encoded trailing dot)', () => { + expectBlocked('http://localhost%2e'); + }); + + // A legitimate public FQDN with a trailing dot must still be allowed + // (no false positive from the dot strip). + it('allows https://example.com. (public FQDN with trailing dot)', () => { + expectAllowed('https://example.com.'); + }); +}); + describe('assertNotLocal — allowed public URLs', () => { it('allows https://example.com', () => { expectAllowed('https://example.com'); diff --git a/src/lib/target-url.ts b/src/lib/target-url.ts index 030fc93..895c0f0 100644 --- a/src/lib/target-url.ts +++ b/src/lib/target-url.ts @@ -41,7 +41,14 @@ export function assertNotLocal(rawUrl: string): void { throw localTargetError('target-url', 'must use http or https scheme'); } - const host = parsed.hostname.toLowerCase(); + // Normalize a single trailing dot in the hostname. `localhost.` is the + // fully-qualified form of `localhost` (RFC 6761 reserves both to resolve to + // loopback), so `http://localhost.` must be rejected just like + // `http://localhost`. Without this strip, the trailing-dot form (also + // reachable via `localhost%2e`) slips past the `host === 'localhost'` check. + // IP literals are already dot-normalized by the WHATWG URL parser, so this + // only affects named hosts. + const host = parsed.hostname.toLowerCase().replace(/\.$/, ''); // Loopback / unspecified. if (host === 'localhost' || host === '0.0.0.0') { diff --git a/src/lib/ticker.spec.ts b/src/lib/ticker.spec.ts index d449900..72760a9 100644 --- a/src/lib/ticker.spec.ts +++ b/src/lib/ticker.spec.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it, vi } from 'vitest'; -import { createTicker } from './ticker.js'; +import { createTicker, isNoColor } from './ticker.js'; describe('createTicker — non-TTY (CI mode)', () => { it('update is a no-op (no writes)', () => { @@ -222,3 +222,66 @@ describe('createTicker — spy on process.stderr', () => { } }); }); + +describe('createTicker — NO_COLOR support', () => { + it('suppresses ANSI escape sequences when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, // isTTY = true + text => raw.push(text), + true, // noColor = true + ); + ticker.update('progress'); + // Should use stderrWrite (line-oriented) instead of rawWrite with ANSI + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).not.toContain('\r'); + expect(lines[0]!).toContain('progress'); + }); + + it('finalize emits plain text without ANSI when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, + text => raw.push(text), + true, // noColor = true + ); + ticker.finalize('done'); + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).toContain('done'); + }); + + it('normal ANSI output when noColor=false on TTY', () => { + const raw: string[] = []; + const ticker = createTicker( + () => {}, + true, + text => raw.push(text), + false, // noColor = false + ); + ticker.update('progress'); + expect(raw).toHaveLength(1); + expect(raw[0]!).toContain('\x1b[2K\r'); + }); +}); + +describe('isNoColor', () => { + it('returns true when NO_COLOR is set to a non-empty value', () => { + expect(isNoColor({ NO_COLOR: '1' })).toBe(true); + expect(isNoColor({ NO_COLOR: 'true' })).toBe(true); + }); + + it('returns false when NO_COLOR is absent or empty', () => { + expect(isNoColor({})).toBe(false); + expect(isNoColor({ OTHER_VAR: '1' })).toBe(false); + // Per https://no-color.org/, an empty NO_COLOR does NOT disable color. + expect(isNoColor({ NO_COLOR: '' })).toBe(false); + }); +}); diff --git a/src/lib/ticker.ts b/src/lib/ticker.ts index 4cae6b1..d9eaed2 100644 --- a/src/lib/ticker.ts +++ b/src/lib/ticker.ts @@ -8,6 +8,9 @@ * - Uses `\r` + ANSI clear-line to overwrite in place on TTY * - On terminal, emits one final line + newline then prints the result * - `--output json` disables the ticker (caller doesn't create one) + * - Respects the NO_COLOR env var (https://no-color.org/): when set, + * ANSI escape sequences are suppressed and updates are emitted as + * plain lines instead of in-place overwrites. * * Overhead: <2ms per update (no syscalls beyond a single write). * @@ -25,6 +28,15 @@ export interface Ticker { finalize(line?: string): void; } +/** + * Returns true when NO_COLOR is present in the environment and is not + * an empty string, per https://no-color.org/. + */ +export function isNoColor(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env.NO_COLOR; + return typeof value === 'string' && value.length > 0; +} + /** * Create a ticker bound to the given stderr writer. Respects * `isTTY` to silently no-op in CI environments. @@ -35,11 +47,14 @@ export interface Ticker { * @param stderrRaw - optional raw writer (no \n appended); used for * the carriage-return + clear-line trick. Defaults to * `process.stderr.write.bind(process.stderr)`. + * @param noColor - whether to suppress ANSI escape sequences. + * Defaults to checking `NO_COLOR` env var per https://no-color.org/. */ export function createTicker( stderrWrite: (line: string) => void, isTTY?: boolean, stderrRaw?: (text: string) => void, + noColor?: boolean, ): Ticker { const tty = isTTY ?? (typeof process !== 'undefined' ? process.stderr.isTTY === true : false); const rawWrite = @@ -47,6 +62,7 @@ export function createTicker( (typeof process !== 'undefined' ? (text: string) => process.stderr.write(text) : (_text: string) => undefined); + const suppressAnsi = noColor ?? isNoColor(); let lastLength = 0; @@ -58,6 +74,25 @@ export function createTicker( }; } + if (suppressAnsi) { + // TTY but NO_COLOR: emit plain-text lines without ANSI escape sequences. + return { + update(line: string): void { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + }, + finalize(line?: string): void { + if (line !== undefined) { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + } + void stderrWrite; + }, + }; + } + return { update(line: string): void { // ANSI ESC[2K clears the entire line; \r moves to column 0. diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts new file mode 100644 index 0000000..dbce172 --- /dev/null +++ b/src/lib/update-check.test.ts @@ -0,0 +1,231 @@ +/** + * Unit tests for the update notice (issue #122). Every effect is injected: + * no real network, filesystem, clock, or TTY is touched. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { UpdateCheckDeps } from './update-check.js'; +import { + UPDATE_CHECK_OPT_OUT_ENV, + UPDATE_CHECK_TTL_MS, + compareSemver, + fetchLatestVersion, + maybeNotifyUpdate, + shouldCheckForUpdate, +} from './update-check.js'; + +/** In-memory fs + deterministic clock harness for the cache round-trip. */ +function makeHarness(overrides: UpdateCheckDeps = {}) { + const files = new Map(); + const stderrLines: string[] = []; + const deps: UpdateCheckDeps = { + env: {}, + now: () => 1_000_000, + cachePath: '/fake/.testsprite/update-check.json', + readFile: path => { + const content = files.get(path); + if (content === undefined) throw new Error('ENOENT'); + return content; + }, + writeFile: (path, content) => { + files.set(path, content); + }, + mkdir: () => undefined, + isTTY: true, + stderr: line => stderrLines.push(line), + currentVersion: '0.2.0', + fetchImpl: async () => + new Response(JSON.stringify({ version: '0.2.0' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ...overrides, + }; + return { deps, files, stderrLines }; +} + +describe('shouldCheckForUpdate gates', () => { + it('opt-out env set to any non-empty value disables (even "0")', () => { + const { deps } = makeHarness({ env: { [UPDATE_CHECK_OPT_OUT_ENV]: '0' } }); + expect(shouldCheckForUpdate(deps)).toBe(false); + }); + + it('CI set disables; CI="false" re-enables', () => { + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'true' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: '' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'false' } }).deps)).toBe(true); + }); + + it('non-TTY stderr disables', () => { + expect(shouldCheckForUpdate(makeHarness({ isTTY: false }).deps)).toBe(false); + }); + + it('a fresh cache suppresses; a stale cache does not', () => { + const fresh = makeHarness(); + fresh.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS + 5_000 }), + ); + expect(shouldCheckForUpdate(fresh.deps)).toBe(false); + + const stale = makeHarness(); + stale.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS - 5_000 }), + ); + expect(shouldCheckForUpdate(stale.deps)).toBe(true); + }); + + it('missing, corrupt, wrong-shape, or future-stamped caches count as stale', () => { + expect(shouldCheckForUpdate(makeHarness().deps)).toBe(true); // missing + const corrupt = makeHarness(); + corrupt.files.set('/fake/.testsprite/update-check.json', '{not json'); + expect(shouldCheckForUpdate(corrupt.deps)).toBe(true); + const wrongShape = makeHarness(); + wrongShape.files.set('/fake/.testsprite/update-check.json', JSON.stringify({ nope: true })); + expect(shouldCheckForUpdate(wrongShape.deps)).toBe(true); + const future = makeHarness(); + future.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 9_999_999_999 }), + ); + expect(shouldCheckForUpdate(future.deps)).toBe(true); + }); +}); + +describe('fetchLatestVersion', () => { + it('returns the version from a valid registry body', async () => { + const { deps } = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '1.2.3' }), { status: 200 }), + }); + await expect(fetchLatestVersion(deps)).resolves.toBe('1.2.3'); + }); + + it('returns undefined on non-2xx, thrown fetch, and wrong-shape body', async () => { + const notOk = makeHarness({ fetchImpl: async () => new Response('nope', { status: 500 }) }); + await expect(fetchLatestVersion(notOk.deps)).resolves.toBeUndefined(); + + const throwing = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await expect(fetchLatestVersion(throwing.deps)).resolves.toBeUndefined(); + + const wrongShape = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ notVersion: 1 }), { status: 200 }), + }); + await expect(fetchLatestVersion(wrongShape.deps)).resolves.toBeUndefined(); + }); +}); + +describe('compareSemver', () => { + it('orders numerically and treats prerelease as older than its release', () => { + expect(compareSemver('0.2.0', '0.3.0')).toBe(-1); + expect(compareSemver('1.0.0', '0.9.9')).toBe(1); + expect(compareSemver('0.2.0', '0.2.0')).toBe(0); + expect(compareSemver('0.10.0', '0.9.0')).toBe(1); // numeric, not lexicographic + expect(compareSemver('1.0.0-rc.1', '1.0.0')).toBe(-1); + expect(compareSemver('1.0.0', '1.0.0-rc.1')).toBe(1); + }); + + it('unparseable input on either side compares as 0 (never a false notice)', () => { + expect(compareSemver('garbage', '1.0.0')).toBe(0); + expect(compareSemver('1.0.0', '')).toBe(0); + }); +}); + +describe('maybeNotifyUpdate', () => { + it('prints exactly one stderr line naming both versions when newer, and stamps the cache', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.3.1' }), { status: 200 }), + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(1); + expect(harness.stderrLines[0]).toContain('0.2.0 -> 0.3.1'); + expect(harness.stderrLines[0]).toContain(UPDATE_CHECK_OPT_OUT_ENV); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBe('0.3.1'); + }); + + it('stays silent on an equal or older registry version', async () => { + const equal = makeHarness(); + await maybeNotifyUpdate(equal.deps); + expect(equal.stderrLines).toHaveLength(0); + + const older = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.1.9' }), { status: 200 }), + }); + await maybeNotifyUpdate(older.deps); + expect(older.stderrLines).toHaveLength(0); + }); + + it('a failed probe stays silent but still stamps the cache (retry once per TTL)', async () => { + const harness = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(0); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBeUndefined(); + }); + + it('does nothing when a gate blocks (no fetch fired)', async () => { + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + const harness = makeHarness({ env: { CI: '1' }, fetchImpl }); + await maybeNotifyUpdate(harness.deps); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(harness.stderrLines).toHaveLength(0); + }); + + it('never rejects, even when every injected dependency throws', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + writeFile: () => { + throw new Error('EROFS'); + }, + stderr: () => { + throw new Error('broken stderr sink'); + }, + }); + await expect(maybeNotifyUpdate(harness.deps)).resolves.toBeUndefined(); + }); + + it('uses the default fs readers/writers when none are injected (real cache round-trip)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'update-check-')); + // Point cachePath at a not-yet-existing subdir so the default mkdir + // (recursive) and writeFile arrows both run, and the first read (file + // absent) exercises the default readFile arrow's ENOENT path. + const cachePath = join(dir, 'nested', 'update-check.json'); + const stderrLines: string[] = []; + try { + await maybeNotifyUpdate({ + env: {}, + now: () => 1_000_000, + isTTY: true, + currentVersion: '0.0.1', + cachePath, + stderr: line => stderrLines.push(line), + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + }); + // Cache was persisted by the default writeFile through the default mkdir. + expect(JSON.parse(readFileSync(cachePath, 'utf8'))).toMatchObject({ latestKnown: '9.9.9' }); + expect(stderrLines.join('\n')).toContain('0.0.1 -> 9.9.9'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts new file mode 100644 index 0000000..e928913 --- /dev/null +++ b/src/lib/update-check.ts @@ -0,0 +1,285 @@ +/** + * Non-blocking "new version available" notice (issue #122), following the + * pattern of the gh and npm CLIs: at most one npm-registry probe per 24 hours, + * result cached on disk, advisory printed to stderr so stdout stays parseable. + * + * Behavior (`maybeNotifyUpdate`): + * 1. Gate through `shouldCheckForUpdate`; every gate below must pass. + * 2. Probe the npm registry for the `latest` dist-tag (1.5s hard timeout). + * 3. Stamp the cache with `lastCheckMs` (plus `latestKnown` when the probe + * succeeded) so the next 24h of invocations skip the network entirely. + * The stamp happens even after a failed probe: a dead registry must not + * trigger a retry on every command. + * 4. When `latest` is strictly newer than the running version, write exactly + * one advisory line to stderr. The function never throws or rejects and + * never alters the exit status of the command it rides along with. + * + * Gates, in order (`shouldCheckForUpdate`): + * - `TESTSPRITE_NO_UPDATE_NOTIFIER` set to any non-empty value: opted out. + * Presence-style, mirroring gh's GH_NO_UPDATE_NOTIFIER: even "0" disables. + * - `CI` set to anything except the literal "false": CI logs are not the + * place for update nags. `CI=false` explicitly re-enables the notice. + * - stderr is not a TTY: piped or redirected output stays clean. + * - the on-disk cache is fresh (last probe within the TTL). A missing, + * unreadable, corrupt, or wrong-shape cache counts as stale, and so does a + * `lastCheckMs` in the future (clock rollback or corrupt data). + * + * Why not the npm `update-notifier` package: this CLI's runtime dependency + * budget is commander + valibot only (package.json). `update-notifier` would + * add a transitive dependency tree for what Node 20 already ships as + * primitives (fetch, AbortSignal.timeout, sync fs). Owning the ~100 lines + * keeps the install size flat and every effect injectable for tests. + * + * All effects (env, network, clock, fs, tty, stderr sink) are injectable via + * `UpdateCheckDeps`, the same dependency-injection style as `skill-nudge.ts`. + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import * as v from 'valibot'; +import { VERSION } from '../version.js'; +import type { FetchImpl } from './http.js'; + +/** Re-check interval: 24 hours, expressed in milliseconds. */ +export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000; + +/** + * Env var that disables the update notice entirely. Presence-style: any + * non-empty value (including "0") opts out. Deliberately stricter than the + * truthy-style `TESTSPRITE_NO_SKILL_WARNING` because it matches the + * convention users already know from gh's GH_NO_UPDATE_NOTIFIER. + */ +export const UPDATE_CHECK_OPT_OUT_ENV = 'TESTSPRITE_NO_UPDATE_NOTIFIER'; + +/** + * Hard cap for the registry probe, in milliseconds. The probe rides along a + * real command, so unlike the 120s API budget in `http.ts` it gets a tiny + * window: a slow registry means "no update info", never a visible delay. + */ +const REGISTRY_TIMEOUT_MS = 1_500; + +/** + * npm registry `latest` dist-tag endpoint for this package. Package name from + * package.json (`@testsprite/testsprite-cli`); the scope separator must be + * URL-encoded as %2F per the npm registry API. + */ +const REGISTRY_LATEST_URL = 'https://registry.npmjs.org/@testsprite%2Ftestsprite-cli/latest'; + +/** On-disk cache shape at `cachePath`; unknown keys are stripped on read. */ +const UPDATE_CHECK_CACHE_SCHEMA = v.object({ + lastCheckMs: v.number(), + latestKnown: v.optional(v.string()), +}); + +export type UpdateCheckCache = v.InferOutput; + +/** Minimal slice of the registry response the notice needs. */ +const REGISTRY_LATEST_BODY_SCHEMA = v.object({ version: v.string() }); + +export interface UpdateCheckDeps { + env?: NodeJS.ProcessEnv; + fetchImpl?: FetchImpl; + /** Clock, epoch milliseconds. */ + now?: () => number; + /** Cache file; lives next to credentials/config under ~/.testsprite. */ + cachePath?: string; + readFile?: (path: string) => string; + writeFile?: (path: string, content: string) => void; + /** Must create missing parent directories (recursive). */ + mkdir?: (dir: string) => void; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; + /** Version the running binary reports. */ + currentVersion?: string; +} + +type ResolvedUpdateCheckDeps = Required; + +function resolveUpdateCheckDeps(deps: UpdateCheckDeps): ResolvedUpdateCheckDeps { + return { + env: deps.env ?? process.env, + fetchImpl: deps.fetchImpl ?? globalThis.fetch, + now: deps.now ?? Date.now, + cachePath: deps.cachePath ?? join(homedir(), '.testsprite', 'update-check.json'), + readFile: deps.readFile ?? ((path: string) => readFileSync(path, 'utf8')), + writeFile: + deps.writeFile ?? ((path: string, content: string) => writeFileSync(path, content, 'utf8')), + mkdir: + deps.mkdir ?? + ((dir: string) => { + mkdirSync(dir, { recursive: true }); + }), + isTTY: deps.isTTY ?? process.stderr.isTTY === true, + stderr: deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)), + currentVersion: deps.currentVersion ?? VERSION, + }; +} + +/** + * Read and validate the cache file. Every failure mode (missing file, + * unreadable file, invalid JSON, wrong shape) returns undefined, which the + * caller treats as "stale, probe again". + */ +function readUpdateCheckCache(resolved: ResolvedUpdateCheckDeps): UpdateCheckCache | undefined { + try { + const raw = resolved.readFile(resolved.cachePath); + const body: unknown = JSON.parse(raw); + const parsed = v.safeParse(UPDATE_CHECK_CACHE_SCHEMA, body); + return parsed.success ? parsed.output : undefined; + } catch { + // Missing or unreadable cache: treat as stale. + return undefined; + } +} + +/** + * Persist the cache, creating the parent directory when missing. Best-effort: + * every error (read-only home, quota, fs races) is swallowed. A failed write + * only means the next invocation probes the registry again. + */ +function writeUpdateCheckCache(resolved: ResolvedUpdateCheckDeps, cache: UpdateCheckCache): void { + try { + resolved.mkdir(dirname(resolved.cachePath)); + resolved.writeFile(resolved.cachePath, `${JSON.stringify(cache)}\n`); + } catch { + // Cache persistence is optional; never surface fs errors to the command. + } +} + +/** + * True when every gate documented in the module header passes: no opt-out + * env, not CI (unless CI=false), stderr is a TTY, and the cached check is + * stale or absent. Order matters: the cheap env gates run before any fs read. + */ +export function shouldCheckForUpdate(deps: UpdateCheckDeps = {}): boolean { + const resolved = resolveUpdateCheckDeps(deps); + + const optOutValue = resolved.env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOutValue !== undefined && optOutValue !== '') return false; + + // Any set CI value except the literal "false" counts as CI. The empty + // string still signals a CI-managed environment; silence wins in doubt. + const ciValue = resolved.env.CI; + if (ciValue !== undefined && ciValue !== 'false') return false; + + if (!resolved.isTTY) return false; + + const cache = readUpdateCheckCache(resolved); + if (cache !== undefined) { + const elapsedMs = resolved.now() - cache.lastCheckMs; + // A negative elapsed (lastCheckMs in the future) means clock rollback or + // corrupt data: treat as stale instead of suppressing the check forever. + // A NaN lastCheckMs fails both comparisons and lands on stale too. + if (elapsedMs >= 0 && elapsedMs < UPDATE_CHECK_TTL_MS) return false; + } + + return true; +} + +/** + * Probe the npm registry for the `latest` dist-tag version of this package. + * Hard 1.5s timeout via AbortSignal.timeout. ANY failure (network error, + * timeout, non-2xx status, invalid JSON, wrong shape) resolves to undefined; + * this function never rejects. + */ +export async function fetchLatestVersion(deps: UpdateCheckDeps = {}): Promise { + const resolved = resolveUpdateCheckDeps(deps); + try { + const response = await resolved.fetchImpl(REGISTRY_LATEST_URL, { + signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS), + }); + if (!response.ok) return undefined; + const body: unknown = await response.json(); + const parsed = v.safeParse(REGISTRY_LATEST_BODY_SCHEMA, body); + return parsed.success ? parsed.output.version : undefined; + } catch { + // Offline, DNS failure, abort, or a non-JSON body: no update info. + return undefined; + } +} + +interface ParsedSemver { + major: number; + minor: number; + patch: number; + hasPrerelease: boolean; +} + +/** + * x.y.z with optional prerelease (after "-") and optional build metadata + * (after "+"), per the semver 2.0.0 grammar. A leading "v" is tolerated + * because humans type it; the npm registry never returns one. + */ +const SEMVER_RE = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/; + +function parseSemver(version: string): ParsedSemver | undefined { + const match = SEMVER_RE.exec(version.trim()); + if (!match) return undefined; + const [, majorRaw, minorRaw, patchRaw, prerelease] = match; + if (majorRaw === undefined || minorRaw === undefined || patchRaw === undefined) return undefined; + return { + major: Number(majorRaw), + minor: Number(minorRaw), + patch: Number(patchRaw), + hasPrerelease: prerelease !== undefined, + }; +} + +/** + * Compare two semver strings numerically. Returns -1 when versionA is older + * than versionB, 1 when newer, 0 when equal. A version carrying a prerelease + * tag sorts OLDER than the plain release with the same x.y.z core. Prerelease + * identifiers themselves are not ranked (two prereleases on the same core + * compare as 0): the registry `latest` tag points at releases, so identifier + * ordering never decides whether the notice fires. Unparseable input on + * either side compares as 0, so garbage can never produce a false notice. + */ +export function compareSemver(versionA: string, versionB: string): number { + const left = parseSemver(versionA); + const right = parseSemver(versionB); + if (left === undefined || right === undefined) return 0; + if (left.major !== right.major) return left.major < right.major ? -1 : 1; + if (left.minor !== right.minor) return left.minor < right.minor ? -1 : 1; + if (left.patch !== right.patch) return left.patch < right.patch ? -1 : 1; + if (left.hasPrerelease !== right.hasPrerelease) return left.hasPrerelease ? -1 : 1; + return 0; +} + +/** + * Fire-and-forget update notice. Gates, probes, stamps the cache, and prints + * at most one stderr line when the registry version is strictly newer than + * the running one. Never throws and never rejects: any failure in any + * injected dependency (clock, fs, network, the stderr sink itself) is + * swallowed, because an advisory must never break or delay a real command. + */ +export async function maybeNotifyUpdate(deps: UpdateCheckDeps = {}): Promise { + try { + const resolved = resolveUpdateCheckDeps(deps); + if (!shouldCheckForUpdate(resolved)) return; + + const latest = await fetchLatestVersion(resolved); + + // Stamp even on a failed probe so a dead registry is retried at most + // once per TTL window, not on every invocation. + writeUpdateCheckCache(resolved, { + lastCheckMs: resolved.now(), + ...(latest === undefined ? {} : { latestKnown: latest }), + }); + + if (latest === undefined) return; + if (compareSemver(latest, resolved.currentVersion) !== 1) return; + + // User-facing advisory copy (exact format specified by issue #122), not a + // diagnostic log line; stderr keeps stdout parseable for scripts. + resolved.stderr( + `A new version of testsprite-cli is available: ${resolved.currentVersion} -> ${latest}. ` + + `Run npm install -g @testsprite/testsprite-cli to update. ` + + `(Disable with ${UPDATE_CHECK_OPT_OUT_ENV}=1)`, + ); + } catch { + // An update notice must never break, delay, or alter the exit status of + // the command it accompanies. Swallow everything. + } +} diff --git a/src/version-guard.test.ts b/src/version-guard.test.ts new file mode 100644 index 0000000..19f0eab --- /dev/null +++ b/src/version-guard.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + MIN_SUPPORTED_NODE_MAJOR, + parseMajorVersion, + shouldRejectNodeVersion, +} from './version-guard.js'; + +// These tests exercise the REAL guard functions used by src/index.ts, +// imported here rather than re-declared, so a regression in the source is +// actually caught. + +describe('parseMajorVersion', () => { + it('extracts the leading major from a semver string', () => { + expect(parseMajorVersion('20.11.1')).toBe(20); + expect(parseMajorVersion('18.0.0')).toBe(18); + expect(parseMajorVersion('22.3.0')).toBe(22); + }); + + it('returns NaN for a non-numeric version string', () => { + expect(Number.isNaN(parseMajorVersion('not-a-version'))).toBe(true); + }); +}); + +describe('shouldRejectNodeVersion', () => { + it('rejects majors below the supported floor', () => { + expect(shouldRejectNodeVersion('18.19.1')).toBe(true); + expect(shouldRejectNodeVersion('16.20.2')).toBe(true); + expect(shouldRejectNodeVersion('14.21.3')).toBe(true); + }); + + it('accepts the supported floor and above', () => { + expect(shouldRejectNodeVersion('20.0.0')).toBe(false); + expect(shouldRejectNodeVersion('20.11.0')).toBe(false); + expect(shouldRejectNodeVersion('21.0.0')).toBe(false); + expect(shouldRejectNodeVersion('22.1.0')).toBe(false); + }); + + it(`treats exactly ${MIN_SUPPORTED_NODE_MAJOR} as supported (boundary)`, () => { + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR}.0.0`)).toBe(false); + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR - 1}.9.9`)).toBe(true); + }); + + it('does not reject an unparseable version (guard never blocks on garbage)', () => { + expect(shouldRejectNodeVersion('not-a-version')).toBe(false); + }); + + it('the running Node satisfies the guard (meta-test)', () => { + // The test suite itself runs on a supported Node, so the guard must pass. + expect(shouldRejectNodeVersion(process.versions.node)).toBe(false); + }); +}); diff --git a/src/version-guard.ts b/src/version-guard.ts new file mode 100644 index 0000000..a7fe464 --- /dev/null +++ b/src/version-guard.ts @@ -0,0 +1,41 @@ +/** + * Node.js runtime version guard. + * + * The CLI targets modern Node (see `engines.node` in package.json). Running on + * an older runtime tends to fail later with a cryptic ESM/syntax error, so the + * entrypoint (`src/index.ts`) uses {@link shouldRejectNodeVersion} to exit early + * with a clear, actionable message instead. + * + * The logic lives here (rather than inline) so it can be unit-tested against the + * real implementation the entrypoint uses — not a copy. + */ + +/** Minimum Node.js major version supported by the CLI (matches package.json `engines.node`). */ +export const MIN_SUPPORTED_NODE_MAJOR = 20; + +/** + * Parse the leading major version number from a Node.js version string. + * + * @param nodeVersion - a dot-separated version string such as `process.versions.node` + * (e.g. `"20.11.1"`). A leading `v` is not expected (Node does not include one here). + * @returns the major version as a number, or `NaN` if the string has no numeric leading segment. + */ +export function parseMajorVersion(nodeVersion: string): number { + return Number(nodeVersion.split('.')[0]); +} + +/** + * Decide whether the given Node.js version is too old to run the CLI. + * + * A version is rejected only when its major number is a real value below + * {@link MIN_SUPPORTED_NODE_MAJOR}. An unparseable string yields `NaN`, which is + * treated as "do not reject" so the guard never blocks on a version string it + * cannot understand (the runtime would surface any real incompatibility itself). + * + * @param nodeVersion - a `process.versions.node` style string (e.g. `"18.19.1"`). + * @returns `true` when the runtime is below the supported floor and should be rejected. + */ +export function shouldRejectNodeVersion(nodeVersion: string): boolean { + const major = parseMajorVersion(nodeVersion); + return !Number.isNaN(major) && major < MIN_SUPPORTED_NODE_MAJOR; +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f1f158..fcc3d85 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Antigravity, Codex) +Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) Options: -h, --help display help for command @@ -14,6 +14,9 @@ Commands: first-run onboarding) into a project for a coding agent list List supported agent targets and skills, their status, and landing paths + status [options] Check installed TestSprite skill files against this CLI + version: ok, stale, modified, unmarked, absent, or corrupt + (exits 1 when anything needs attention, so it can gate CI) help [command] display help for command " `; @@ -25,8 +28,9 @@ Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent Options: - --target Agent target(s): claude, cursor, cline, antigravity, codex - (comma-separated or repeated) (default: []) + --target Agent target(s): claude, cursor, cline, antigravity, kiro, + windsurf, copilot, codex (comma-separated or repeated) + (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -36,7 +40,7 @@ Options: destroyed. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -49,7 +53,7 @@ List supported agent targets and skills, their status, and landing paths Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -86,7 +90,7 @@ exports[`--help snapshots > auth logout 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -97,7 +101,7 @@ exports[`--help snapshots > auth whoami 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -112,7 +116,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, codex (default: claude) (default: "claude") + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -153,7 +158,7 @@ Get a project by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -175,7 +180,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -196,9 +201,23 @@ Commands: (--plan-from, FE-only, M3.2 piece-5) create-batch [options] Create multiple FE tests from a JSONL of plan specs (FE-only) + scaffold [options] Emit a schema-correct starter test + definition (frontend plan JSON by + default, or a backend Python skeleton). + Pure-local: no network, no credentials. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) + diff Compare two runs and print what + regressed: verdict, failureKind, + failedStepIndex, per-step status flips, + codeVersion drift. Exit 0 when verdicts + match, 1 when they differ. + lint [options] Validate plan/steps files offline with + the same validators \`create\` runs, + collecting EVERY problem. No network, + no credentials. Exit 0 when all valid, + 5 otherwise. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: @@ -237,14 +256,20 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get - wait [options] Wait for a run to reach a terminal status. + wait [options] Wait for one or more runs to reach a terminal status. + + With several run-ids the runs are polled concurrently under one shared + --timeout and a {results, summary} envelope is printed (worst status wins + the exit code), so every re-attach hint the CLI prints can be pasted as + ONE command. Exit codes: 0 passed 1 failed / blocked / cancelled 3 auth error - 4 run not found - 7 timeout — resume with: testsprite test wait + 4 run not found (single run-id; with several ids a per-member poll error + is recorded as error: in its row and folded into exit 7) + 7 timeout or per-member poll error — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get @@ -261,6 +286,15 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + flaky [options] Repeatedly replay a test to measure stability and surface flakiness. + Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + + Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error code Inspect and edit generated test code plan Manage FE test plan-steps (FE-only) failure Export the latest-failure agent bundle @@ -280,7 +314,7 @@ Options: source body; json mode: wire envelope) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -315,7 +349,7 @@ Options: per invocation; pin one yourself for safe retries. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -331,7 +365,39 @@ Options: --failed-only Keep only the failed step plus its immediate neighbors (±1) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + +exports[`--help snapshots > test flaky 1`] = ` +"Usage: testsprite test flaky [options] + +Repeatedly replay a test to measure stability and surface flakiness. +Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + +Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error + +Options: + --runs number of replays to run (1-10, default 5) + --until-fail stop at the first non-passing attempt (fast "is it flaky at + all?" check) (default: false) + --timeout per-attempt max seconds to wait (1-3600, default 600) + -h, --help display help for command + +Notes: + • Frontend replays are free verbatim script replays (no credit); backend replays + re-run the dependency closure and may cost credits — a one-line advisory is printed. + • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass; + this measures replay stability of the saved script against the configured URL. + • \`--output json\` emits a machine-readable stability report for CI gating. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -344,7 +410,7 @@ Get a test by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -369,7 +435,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -392,32 +458,40 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --all rerun all tests in the resolved project (requires - --project) (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --skip-terminal with --all: skip tests already in a terminal status - (passed|failed|blocked|cancelled) (default: false) - --status with --all: only dispatch tests whose status matches - one of these values (comma-separated; accepted: - draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) - --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) - --wait block until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --no-auto-heal opt out of AI heal-on-drift for this FE rerun - (default: auto-heal is ON). Costs 0.2 credits per - engage when a step has drifted. Ignored for backend - tests. - --skip-dependencies BE only: rerun only the named test without expanding - the producer/teardown closure (default: false) - --max-concurrency with --wait, max in-flight polls at once (1-100, - default: 50) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --verbose if auto-generated. - -h, --help display help for command + --all rerun all tests in the resolved project (requires + --project) (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --skip-terminal with --all: skip tests already in a terminal + status (passed|failed|blocked|cancelled) + (default: false) + --status with --all: only dispatch tests whose status + matches one of these values (comma-separated; + accepted: + draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) + --filter with --all: only rerun tests whose name contains + this substring (case-insensitive) + --wait block until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --no-auto-heal opt out of AI heal-on-drift for this FE rerun + (default: auto-heal is ON). Costs 0.2 credits per + engage when a step has drifted. Ignored for + backend tests. + --skip-dependencies BE only: rerun only the named test without + expanding the producer/teardown closure (default: + false) + --max-concurrency with --wait, max in-flight polls at once (1-100, + default: 50) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --verbose if auto-generated. + --report with batch --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Notes: • rerun replays a saved run/script and is MORE LENIENT than a fresh \`test run\` @@ -433,7 +507,7 @@ Dry-run shape notes: omit \`closure\` (or return it as null) since there is no dependency expansion. • \`autoHeal\` defaults true for FE reruns; BE reruns ignore the field entirely. -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -465,7 +539,7 @@ Options: --cursor with --history: opaque cursor from a prior page -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -490,33 +564,39 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --target-url override the project default env URL for this run - (http/https only, no localhost/private IPs) - --wait poll until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --filter with --all: only run tests whose name contains this - substring (case-insensitive) - --max-concurrency with --all --wait, max in-flight polls at once - (1-100, default: 50) - -h, --help display help for command + --target-url override the project default env URL for this run + (http/https only, no localhost/private IPs) + --wait poll until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --debug if auto-generated. + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --filter with --all: only run tests whose name contains + this substring (case-insensitive) + --max-concurrency with --all --wait, max in-flight polls at once + (1-100, default: 50) + --report with --all --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Dependency-aware fresh run (M4): testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset + testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -539,7 +619,7 @@ Options: excluded when this flag is set. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -582,9 +662,11 @@ Commands: test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, - Codex) + Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) + doctor Diagnose CLI setup: version, Node, profile, + endpoint, credentials, connectivity, skill help [command] display help for command " `; diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 21d7cfe..b0537b1 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -499,6 +499,24 @@ describe('project list subprocess', () => { const parsed = JSON.parse(result.stderr) as { error: { code: string } }; expect(parsed.error.code).toBe('VALIDATION_ERROR'); }, 30_000); + + it('--request-timeout 30s exits 5 (VALIDATION_ERROR), not a silent fallback to 120s', async () => { + // Previously an invalid flag value resolved to `undefined` and the command + // silently ran with the default 120s deadline — the operator believed they + // had set a timeout but had not. Now the explicit flag is validated like + // every other flag. + const result = await runCli( + ['--output', 'json', '--request-timeout', '30s', 'project', 'list'], + { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }, + ); + expect(result.exitCode).toBe(5); + const parsed = JSON.parse(result.stderr) as { error: { code: string; nextAction: string } }; + expect(parsed.error.code).toBe('VALIDATION_ERROR'); + expect(parsed.error.nextAction).toContain('request-timeout'); + }, 30_000); }); describe('malformed --endpoint-url is rejected (exit 5), not retried as a network error', () => { @@ -530,6 +548,33 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ }, 30_000); }); +describe('invalid --output is rejected uniformly (exit 5)', () => { + // Regression: previously only `test` and `project` validated `--output`; + // `auth`, `usage`, `agent`, and `init` silently coerced an unknown value to + // text mode. An agent that asked for `--output json` but mistyped it then + // received a text payload it could not parse, with no signal as to why. Every + // command group now routes through resolveOutputMode (exit 5 on bad input). + + // Note: when `--output` itself is the invalid value, the requested mode is + // unusable for the error envelope, so it is rendered in text mode (the catch + // block in index.ts falls back to text for an unrecognised --output). + + it('agent list --output josn exits 5 with an actionable message (offline command)', async () => { + const result = await runCli(['--output', 'josn', 'agent', 'list'], {}); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); + + it('auth status --output yaml exits 5 before any network call', async () => { + const result = await runCli(['--output', 'yaml', 'auth', 'status'], { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); +}); + describe('a malformed --profile is rejected (exit 5), not silently corrupting credentials', () => { // A profile name becomes an INI section header (`[name]`). `prod]` would // serialise to `[prod]]`, which the parser cannot read back — `setup` would @@ -896,6 +941,22 @@ describe('--dry-run subprocess smoke', () => { expect(parsed.id).toBeTruthy(); }, 30_000); + it('project update --dry-run does not read a missing --password-file', async () => { + const result = await runCli([ + 'project', + 'update', + 'proj_anything', + '--password-file', + '/tmp/definitely-not-here-testsprite', + '--dry-run', + '--output', + 'json', + ]); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout) as { updatedFields: string[] }; + expect(parsed.updatedFields).toContain('password'); + }, 30_000); + it('test list --dry-run returns canned TestList', async () => { const result = await runCli([ 'test', diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 1cbf4fc..4fb4584 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -169,11 +169,25 @@ describe('content integrity', () => { content.trimStart().startsWith('#'), `cline: should start with a markdown heading`, ).toBe(true); + } else if (target === 'windsurf') { + // Windsurf Cascade frontmatter: trigger + description (no name/alwaysApply) + expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); + } else if (target === 'copilot') { + // GitHub Copilot instructions frontmatter: applyTo glob + description + expect(content.startsWith('---'), `copilot: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); + expect(content).toContain('description:'); } - // (b) branding — the renamed H1 must be present + // (b) branding — the renamed H1 must be present in every body variant expect(content).toContain('TestSprite Verification Loop'); - expect(content).toContain('The verification loop that flies'); + // The full-body intro line lives only in the FULL body; compact-body targets + // (e.g. windsurf, budget-capped) ship the trimmed verify body and omit it. + if (!TARGETS[target].compactBody) { + expect(content).toContain('The verification loop that flies'); + } // (c) Load-bearing command strings expect(content, `${target}: missing 'testsprite test run'`).toContain('testsprite test run'); @@ -198,6 +212,13 @@ describe('content integrity', () => { expect(content).toContain('alwaysApply: false'); } else if (target === 'cline') { expect(content.startsWith('---'), `cline/onboard: must NOT start with ---`).toBe(false); + } else if (target === 'windsurf') { + expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); + } else if (target === 'copilot') { + expect(content.startsWith('---'), `copilot/onboard: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); } // Load-bearing onboard string: the skill body must reference setup @@ -430,13 +451,13 @@ describe('dry-run', () => { // --------------------------------------------------------------------------- describe('multi-target install', () => { - it('--target=claude,cursor,cline,antigravity,codex writes all targets + skills, exit 0', () => { + it('--target=claude,cursor,cline,antigravity,kiro,codex writes all targets + skills, exit 0', () => { const tmpDir = freshTmpDir(); const result = runCli([ 'agent', 'install', - '--target=claude,cursor,cline,antigravity,codex', + '--target=claude,cursor,cline,antigravity,kiro,codex', '--dir', tmpDir, '--output', @@ -449,7 +470,7 @@ describe('multi-target install', () => { action: string; path: string; }>; - const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'codex']; + const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'codex']; for (const target of allTargets) { if (TARGETS[target].mode === 'managed-section') { @@ -790,7 +811,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 5 targets × 2 skills = 10 rows + // Expected: 8 targets × 2 skills = 16 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -819,7 +840,16 @@ describe('agent list', () => { // --------------------------------------------------------------------------- describe('matrix coverage guard', () => { it('TARGETS matches the documented, e2e-covered set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'windsurf', + 'copilot', + 'codex', + ]); }); it('SKILLS matches the documented, e2e-covered set (update this list when adding a skill)', () => { diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index b5d3963..5a07434 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -222,7 +222,16 @@ describe('deprecated `init` alias', () => { describe('matrix coverage guard', () => { it('TARGETS matches the documented set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'windsurf', + 'copilot', + 'codex', + ]); }); }); diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index 2448c91..ee89b3b 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -39,6 +39,7 @@ const cases: Array<[string, string[]]> = [ ['test result', ['test', 'result', '--help']], ['test failure get', ['test', 'failure', 'get', '--help']], ['test rerun', ['test', 'rerun', '--help']], + ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], diff --git a/vitest.config.ts b/vitest.config.ts index 18ea00f..add8f0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel + // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). + fileParallelism: false, coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'json-summary', 'html'],