diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 468c8ed..0000000 --- a/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -# Check out all text files with LF on every platform. Tests compare bytes -# from checked-out files (skill templates, snapshots); a CRLF working tree -# (core.autocrlf on Windows) broke those comparisons. -* text=auto eol=lf - -*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1c165f6..c490f9b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,9 +12,7 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml deleted file mode 100644 index d34744f..0000000 --- a/.github/workflows/ci-nudge.yml +++ /dev/null @@ -1,126 +0,0 @@ -# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). -# When a PR's CI finishes red, posts ONE sticky comment listing exactly which -# jobs failed and the local one-liner that reproduces/fixes each (the automated -# version of the hand-written "run `npm run format` and you're green" review -# comments). The comment flips to a green confirmation once all checks pass. -# -# Runs in base-repo context via workflow_run — no PR code is checked out, so -# fork PRs are safe. State is recomputed from check-runs each time, so the two -# CI workflows (CI + Test Coverage) can complete in any order. -# -# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as -# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to -# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). -name: CI failure nudge - -on: - workflow_run: - workflows: ['CI', 'Test Coverage'] - types: [completed] - -permissions: - checks: read # read the head SHA's check-run state - pull-requests: write - issues: write # PR comments ride the issues API - -env: - MARKER: '' - -jobs: - nudge: - # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). - if: >- - github.repository == 'TestSprite/testsprite-cli' && - github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest - steps: - - id: app-token - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 - env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - with: - script: | - const { owner, repo } = context.repo; - const run = context.payload.workflow_run; - const marker = process.env.MARKER; - - const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) - : null; - async function write(fn) { - if (appClient) { - try { return await fn(appClient.rest); } - catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } - } - return await fn(github.rest); - } - - // Resolve the PR for this run. `workflow_run.pull_requests` is empty - // for fork PRs, so fall back to the commit→PRs lookup. - let pr = (run.pull_requests || [])[0]; - if (!pr) { - const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( - { owner, repo, commit_sha: run.head_sha }); - pr = data.find(p => p.state === 'open'); - } else { - pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; - } - if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; - - // Recompute full CI state from this SHA's github-actions check runs. - const checks = await github.paginate(github.rest.checks.listForRef, - { owner, repo, ref: run.head_sha, per_page: 100 }); - const ours = checks.filter(c => c.app && c.app.slug === 'github-actions'); - const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); - const pending = ours.filter(c => c.status !== 'completed'); - - const comments = await github.paginate(github.rest.issues.listComments, - { owner, repo, issue_number: pr.number, per_page: 100 }); - const sticky = comments.find(c => (c.body || '').includes(marker)); - - // Job name → the local command that reproduces/fixes it. - const FIX = { - 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', - 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', - 'Unit Tests': 'run `npm test` and fix the failing tests', - 'Build': 'run `npm run build` and fix the compile errors', - 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', - 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', - }; - - if (failing.length === 0) { - // Only speak up on success if we previously flagged a failure, and - // only once everything has actually finished. - if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { - await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, - body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); - } - return; - } - - const lines = failing - .sort((a, b) => a.name.localeCompare(b.name)) - .map(c => { - const fix = FIX[c.name] || 'see the logs for details'; - return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; - }); - const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` - + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` - + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` - + `flips green automatically once all checks pass.`; - - if (sticky) { - if (sticky.body !== body) { - await write(rest => rest.issues.updateComment( - { owner, repo, comment_id: sticky.id, body })); - } - } else { - await write(rest => rest.issues.createComment( - { owner, repo, issue_number: pr.number, body })); - } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index e9923dd..63323c0 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,15 +1,13 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. +# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → +# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# TESTSPRITE_HOB_APP_ID = the App's numeric App ID -# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) -# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from -# before it was renamed — same App ID + key; either naming works.) +# ALFHEIM_AGENT_APP_ID = the App's numeric App ID +# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -32,23 +30,23 @@ env: jobs: triage: # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} runs-on: ubuntu-latest steps: - # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. + # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 with: - # App token when available (→ testsprite-hob[bot]); else the default + # App token when available (→ alfheim-agent[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml deleted file mode 100644 index a9cd23d..0000000 --- a/.github/workflows/pr-triage.yml +++ /dev/null @@ -1,189 +0,0 @@ -# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). -# Enforces the issue-first workflow on community PRs: every non-docs PR must -# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR -# author (claimed via `/assign`, which issue-triage.yml handles). Violations -# get a `needs-issue` label + a sticky comment, and — for PRs opened after -# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. -# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. -# -# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; -# the comment tells the contributor to edit the PR description or push a -# commit (`edited` / `synchronize`) to re-run the gate. -# -# Runs with NO checkout — metadata-only, so it is safe under pull_request_target -# (which is required for fork PRs to get a write-capable token). -# -# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The -# App currently has Issues R/W + Metadata only — commenting/labeling a PULL -# REQUEST needs the "Pull requests: Read & write" App permission (issues-API -# endpoints are permission-checked by target type). Until an org admin adds -# that permission and re-approves the installation, every call gracefully falls -# back to the default GITHUB_TOKEN and posts as github-actions[bot]. -# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the -# original names from before the App was renamed (same App ID + key). -name: PR triage (issue-link gate) - -on: - pull_request_target: - types: [opened, edited, reopened, synchronize] - -permissions: - pull-requests: write # add/remove the needs-issue label - issues: write # create/update the nudge comment (PR comments ride the issues API) - -env: - LABEL: 'needs-issue' - MARKER: '' - # PRs created before this instant are grandfathered (nudge, no failing check). - GATE_SINCE: '2026-07-04T00:00:00Z' - -jobs: - gate: - # Public repo only — this file also lives in the private mirror, where PRs - # are internal work that never links public issues. Skip bot authors - # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING - # exempts docs/small fixes from the issue-first ask). - if: >- - github.repository == 'TestSprite/testsprite-cli' && - github.event.pull_request.state == 'open' && - github.event.pull_request.user.type != 'Bot' && - !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && - !startsWith(github.event.pull_request.title, 'docs') - runs-on: ubuntu-latest - steps: - # Mint an App token so actions post as testsprite-hob[bot]. Until the App + - # secrets + Pull-requests permission exist this no-ops / gets 403 and the - # script below falls back to the default token per-call. - - id: app-token - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 - env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - with: - # `github` client = default GITHUB_TOKEN: used for all reads (the App - # can't read PRs without the Pull-requests permission) and as the - # write fallback. App client (when mintable) is preferred for writes. - script: | - const { owner, repo } = context.repo; - const pr = context.payload.pull_request; - const label = process.env.LABEL; - const marker = process.env.MARKER; - const author = pr.user.login; - - const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) - : null; - // Prefer the App identity; fall back to github-actions[bot] when the - // App is absent or lacks the Pull-requests permission (403/404). - async function write(fn) { - if (appClient) { - try { return await fn(appClient.rest); } - catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } - } - return await fn(github.rest); - } - - // Linked issues: GitHub-computed closing references carry number + - // assignees in one query. - const gql = await github.graphql( - `query($owner:String!,$repo:String!,$num:Int!){ - repository(owner:$owner,name:$repo){ - pullRequest(number:$num){ - closingIssuesReferences(first:10){ - nodes{ number assignees(first:10){ nodes{ login } } } - } - } - } - }`, { owner, repo, num: pr.number }); - const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes - .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); - - // Body-text fallback for the brief window before GitHub computes the - // link (same-repo bare "#N" references only). - const bodyRefs = [...(pr.body || '') - .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] - .map(m => Number(m[3])) - .filter(n => !linkedIssues.some(i => i.number === n)) - .slice(0, 5); - for (const num of bodyRefs) { - try { - const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); - if (data.pull_request) continue; // "#N" pointed at a PR, not an issue - linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); - } catch (e) { /* unknown number — ignore */ } - } - - const linked = linkedIssues.length > 0; - const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); - const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); - - const hasLabel = (pr.labels || []).some(l => l.name === label); - const comments = await github.paginate(github.rest.issues.listComments, - { owner, repo, issue_number: pr.number, per_page: 100 }); - const nudge = comments.find(c => (c.body || '').includes(marker)); - const stateLine = ``; - - const contributingUrl = - `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; - const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; - - let body; - if (state === 'ok') { - body = `${marker}\n${stateLine}\n` - + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` - + `The \`${label}\` label has been removed.`; - } else if (state === 'unassigned') { - const list = linkedIssues.map(i => { - const holders = i.assignees.filter(a => a !== author); - return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); - }).join(', '); - body = `${marker}\n${stateLine}\n` - + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` - + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` - + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` - + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` - + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; - } else { - body = `${marker}\n${stateLine}\n` - + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` - + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` - + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` - + `so it is not review-ready. ${rerunHint} ` - + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; - } - - // Sticky comment: create once, update when the state changes. - if (!nudge) { - if (state !== 'ok') { - await write(rest => rest.issues.createComment( - { owner, repo, issue_number: pr.number, body })); - } - } else if (!nudge.body.includes(stateLine)) { - await write(rest => rest.issues.updateComment( - { owner, repo, comment_id: nudge.id, body })); - } - - // Label tracks the gate state. - if (state === 'ok' && hasLabel) { - await write(rest => rest.issues.removeLabel( - { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); - } - if (state !== 'ok' && !hasLabel) { - await write(rest => rest.issues.addLabels( - { owner, repo, issue_number: pr.number, labels: [label] })); - } - - // Hard gate for PRs opened after the cutoff; older PRs are nudged only. - const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); - if (state !== 'ok' && gated) { - core.setFailed(state === 'unlinked' - ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' - : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); - } else if (state !== 'ok') { - core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); - } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index fbaa607..0000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,64 +0,0 @@ -# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). -# -# Generous windows for an open-source repo (first-response SLA is 5 business days, -# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single -# reply resets the clock. Newcomer- and security-relevant work is exempt from -# closing. `good first issue` / `help wanted` stay open for whoever picks them up. -# -# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike -# the event-gated triage bot it is fenced to the PUBLIC repo with a repository -# guard — on private atlas it is a no-op (no nagging internal issues/PRs). -name: Stale - -on: - schedule: - - cron: '30 1 * * *' # daily 01:30 UTC - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - stale: - if: ${{ github.repository == 'TestSprite/testsprite-cli' }} - runs-on: ubuntu-latest - permissions: - issues: write # comment + (un)label + close stale issues - pull-requests: write # comment + (un)label + close stale PRs - steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 - with: - # ── issues ────────────────────────────────────────────────────── - days-before-issue-stale: 60 - days-before-issue-close: 14 - stale-issue-label: stale - stale-issue-message: > - This issue has had no activity for 60 days, so it's been marked - `stale`. If it's still relevant, just leave a comment (or remove the - `stale` label) and we'll keep it open — otherwise it will be closed - in 14 days. Thanks for helping us keep the tracker tidy! - close-issue-message: > - Closing as `stale` after no activity. This isn't a judgement on the - idea — please reopen or open a fresh issue if it's still relevant. - - # ── pull requests (more grace — external contributors may be slow) ─ - days-before-pr-stale: 45 - days-before-pr-close: 21 - stale-pr-label: stale - stale-pr-message: > - This PR has had no activity for 45 days, so it's been marked `stale`. - Push a commit or leave a comment to keep it open — otherwise it will - be closed in 21 days. We'd still love to merge it; ping a maintainer - if you're blocked on a review. - close-pr-message: > - Closing as `stale` after no activity. Reopen any time you can pick it - back up — your work isn't lost. - - # ── shared behaviour ──────────────────────────────────────────── - exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' - exempt-pr-labels: 'pinned,security,in-progress,hackathon' - exempt-draft-pr: true - remove-stale-when-updated: true - ascending: true # oldest first — fairest under the per-run op cap - operations-per-run: 60 - enable-statistics: true diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index ebb3331..4eb15a9 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -73,7 +73,7 @@ jobs: fi - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' with: recreate: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bac6f2..b510b9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,48 +4,11 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] -## [0.3.0] - 2026-07-08 - ### Added -- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. -- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. -- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. -- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. -- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. -- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. -- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. -- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. -- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. -- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. -- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. -- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. -- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. - -### Changed - -- **Node.js 20 is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. -- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). -- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. -- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. -- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). -- The `usage` command no longer implies backend test runs are free. -- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. - -### Fixed - -- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). -- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. -- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. -- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. -- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. -- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. -- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. - -### Security - -- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. -- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. +- **`test failure triage --project `** — groups all failed tests in a project into root-cause clusters using existing M2.1 analysis fields (`failureKind`, `recommendedFixTarget.reference`, `rootCauseHypothesis`). Returns a representative test per cluster, affected test ids, confidence score, and fix priority — without downloading failure bundles. Supports `--type`, `--filter`, and `--max-concurrency`. Client-side Phase-0 triage until native backend clustering ships. +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. +- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. ## [0.2.0] - 2026-06-29 @@ -187,9 +150,6 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD -[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 -[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 -[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e016c2..2110f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,53 +20,14 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Docs and small fixes** (typos, doc corrections, comment-only changes): - just open a pull request. No issue required (linking one is still - appreciated). -- **Features and behavior changes** (new commands or flags, changed output, - new dependencies, refactors) follow **issue-first**: - 1. Find an existing issue, or open a new one describing the change. - 2. Claim it by commenting `/assign` on the issue — the literal slash - command on its own line. The triage bot assigns you automatically; - free-text requests ("can I take this?") are **not** detected. - 3. If the issue is new, wait for triage — we check proposals against - [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) - below before any code is written, so you don't invest in something - we'd ask you to rework or decline. - 4. Open your PR with a closing link (e.g. `Closes #123`) in the - description. -- **PR gate:** a bot checks every non-docs community PR for a closing-linked - issue that is **assigned to the PR author**. PRs that don't meet this get - the `needs-issue` label and a failing `PR triage` check, and **are not - reviewed** until it's fixed — file or claim the issue, add the closing - link, then edit the PR description (or push a commit) to re-run the check. -- Suspected **security vulnerabilities** are the exception to "file an - issue": report them privately per [SECURITY.md](./SECURITY.md) instead. +- **Small fixes and improvements:** just open a pull request. No issue required. +- **Large or breaking changes** (new commands, changed flags/output, new + dependencies, refactors): **open an issue first** to discuss the design. This + avoids wasted work on something we'd ask you to rework or that's out of scope. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. -### Standing scope policies - -Pre-decided policies, so proposals don't have to relitigate them: - -- **Runtime dependencies are budgeted.** The CLI ships with a deliberately - tiny runtime dependency set (`commander`, `valibot`, plus `undici` — - approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new - runtime dependency needs explicit maintainer sign-off **in the issue, - before the PR**. Utility modules land only together with the consumer - that uses them — standalone libraries are declined. -- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, - `cline`, `codex`. Accepted and in progress: `windsurf`, `gemini`, `kiro`. - A proposal for a new target needs (1) the editor's official rules/skill - file mechanism, documented, and (2) the proposer prepared to maintain the - target going forward. -- **Outbound network calls.** The CLI talks only to the configured - TestSprite API endpoint. The one approved exception is an opt-out-able - npm registry version check (at most once per 24h, carrying nothing but - the package name, fully silenced by its env opt-out and in CI/JSON/dry-run - modes). Any other outbound call is out of scope. - We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6686d52..66712cb 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,7 +15,6 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) - - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -116,20 +115,14 @@ testsprite agent install cline # .clinerules/testsprite-verify.md testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md -testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md -testsprite agent list # list all 8 targets with status + mode + path -testsprite agent status # check installed skills against this CLI version +testsprite agent list # list all 7 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). - -Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). - -`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf) backs up the existing file to `.bak` first. ## Command reference @@ -226,15 +219,6 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` -#### `testsprite test diff ` - -Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. - -```bash -testsprite test diff run_aaaa run_bbbb --output json -testsprite test diff run_aaaa run_bbbb --dry-run --output json -``` - #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -266,31 +250,35 @@ testsprite test failure summary test_xxxxxxxx --output json testsprite test failure summary test_xxxxxxxx --dry-run --output json ``` -### Write commands +#### `testsprite test failure triage --project ` -Require the `write:tests` scope (project commands require `write:projects`). +When many tests fail in the same project, triage them into a few root-cause clusters before downloading bundles. The CLI lists all failed tests, fetches a lightweight `failure/summary` per test (no screenshots or video), and groups them client-side by: -#### `testsprite test scaffold` +1. shared `recommendedFixTarget.reference` +2. env-wide `failureKind` (`infra`, `network`, `network_timeout`, `routing_404`) +3. normalized `rootCauseHypothesis` prefix +4. singleton (one test per cluster when no shared signal exists) -Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. +Each cluster includes a `representativeTestId`, `memberTestIds`, `confidence`, and `fixPriority` (lower = fix first). After triage, pull one bundle from the representative test: ```bash -testsprite test scaffold > first-test.plan.json -testsprite test scaffold --type backend --out tests/health.py -testsprite test scaffold --out plan.json --force # overwrite an existing file -``` +# Triage all failed tests in a project +testsprite test failure triage --project proj_xxxxxxxx --output json -#### `testsprite test lint` +# Limit to backend tests whose name contains "checkout" +testsprite test failure triage --project proj_xxxxxxxx --type backend --filter checkout --output json -Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. +# Then investigate the highest-priority cluster's representative test +testsprite test failure get --out ./.testsprite/failure -```bash -testsprite test lint --plan-from ./checkout.plan.json -testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported -testsprite test lint --plans ./plans.jsonl # one plan spec per line -testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests +# Learn the JSON shape offline +testsprite test failure triage --project proj_xxxxxxxx --dry-run --output json ``` +### Write commands + +Require the `write:tests` scope. + #### `testsprite test create` Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. @@ -328,7 +316,7 @@ testsprite test update test_xxxxxxxx --dry-run --output json #### `testsprite test delete ` / `test delete-batch` -Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -358,66 +346,20 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` -#### `testsprite project credential ` - -Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. - -```bash -testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt -testsprite project credential proj_xxxxxxxx --type public -testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json -``` - -`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. - -#### `testsprite project auto-auth ` - -Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. - -```bash -# Password login: POST the login endpoint, extract the token, inject as a Bearer header -testsprite project auto-auth proj_xxxxxxxx \ - --method password --inject bearer \ - --login-url https://api.example.com/login --login-method POST \ - --login-content-type application/json \ - --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ - --username ci@example.com --password-file ./pw.txt \ - --token-path '$.data.accessToken' - -# OAuth refresh-token flow -testsprite project auto-auth proj_xxxxxxxx \ - --method refresh_token --inject header --inject-key X-Auth-Token \ - --token-endpoint https://auth.example.com/oauth/token \ - --client-id my-client --client-secret-file ./secret.txt \ - --refresh-token-file ./refresh.txt --scope api.read - -# AWS Cognito refresh -testsprite project auto-auth proj_xxxxxxxx \ - --method aws_cognito_refresh --inject bearer \ - --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 - -# Turn it off (stored config is kept) -testsprite project auto-auth proj_xxxxxxxx --disable -``` - -Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. - ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. - -`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). ```bash # Trigger and return immediately @@ -430,7 +372,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json -# Batch run with JUnit XML for CI (sidecar; --output json unchanged) +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --output json @@ -509,17 +451,16 @@ Flags: `--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. -#### `testsprite test wait ` +#### `testsprite test wait ` -Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. +Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json -testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. #### `testsprite test artifact get ` @@ -534,30 +475,6 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). -### Account & diagnostics - -#### `testsprite usage` (alias: `testsprite credits`) - -Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. - -```bash -testsprite usage --output json -testsprite credits -testsprite usage --dry-run --output json -``` - -#### `testsprite doctor` - -One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: - -```bash -testsprite doctor -testsprite doctor --output json -testsprite doctor && testsprite test run test_xxxxxxxx --wait -``` - -Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. - ## Configuration ### Profiles & credentials @@ -580,17 +497,14 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ----------------------------------------- | ---------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | -| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | -| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | -| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | ### Update notice @@ -607,14 +521,13 @@ only outbound call the CLI makes besides your configured API endpoint. API-key scopes gate the write and run surfaces: -| Scope | Required by | -| ---------------- | -------------------------------------------------------------------- | -| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `write:projects` | `project create / update / credential / auto-auth` | -| `run:tests` | `test run / rerun / flaky / wait / artifact get` | +| Scope | Required by | +| --------------- | -------------------------------------------------------------------- | +| `read:me` | `auth whoami` | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `run:tests` | `test run / rerun / wait / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -632,25 +545,20 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| --------------------- | --------------------------------------------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | -| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | - -### Signals & pipes - -On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. +| Code | Meaning | +| ---- | --------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | ## Design principles diff --git a/README.md b/README.md index 78d293a..4f6d26c 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,6 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. -> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). - From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -91,34 +89,29 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | -| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | -| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | -| | `test wait` | Block on one or more `runId`s until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test failure triage` | Group all failed tests in a project into root-cause clusters (no bundle download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package.json b/package.json index e78ce17..341e713 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.2.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index df04c18..cc9342c 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,27 +91,6 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. -**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This -covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, -`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block -(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the -project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for -the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; -basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a -literal `Bearer …` / `sk-…` / key value into the script: - -```python -r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) -``` - -Configure the credential once on the project (ask the user for the value — never invent it or -reuse a key you happened to see): a static credential with -`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, -or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat -runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded -key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it -as a must-fix. - **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index fe7c661..26bab39 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -165,39 +165,6 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. -**Authentication — read the injected credential, NEVER hardcode any credential.** This -applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth -blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any -other credential into the test. Before your script runs, TestSprite prepends a managed -credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` -already contains the right header(s) for the configured auth type: - -```python -# Auto-injected credentials — do not modify -__AUTH_CREDENTIAL__ = "..." -__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" -__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} -``` - -Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type -the project is configured for, so the same line works for Bearer, API-key, or basic auth: - -```python -r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) -``` - -Configure the credential **once on the project** (ask the user for the value — never invent -or reuse a key you happened to see), and the block stays correct + refreshable: - -- **Bearer / API key / basic (static):** - `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` -- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` - -A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so -the test breaks on later runs; the managed block is rewritten with a fresh value each run. -`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as -a must-fix, not a nuisance. - **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer @@ -449,7 +416,22 @@ testing agent's observation; don't auto-fix on the recommendation alone. If you genuinely can't tell: report `inconclusive` with the signal that triggered the call and ask. -## 5. On failure → download the artifact +## 5. On failure → triage first, then download one bundle + +When **multiple tests failed** in the same project (batch run, regression, or +`test list --status failed` shows more than one red row), triage before pulling +every bundle: + +```bash +testsprite test failure triage --project --output json +``` + +Read the clusters: each has a `representativeTestId`, `memberTestIds`, +`confidence`, and `fixPriority` (lower = fix first). Investigate the +representative test from the highest-priority cluster — not an arbitrary failed +test. After a fix, rerun that representative before rerunning the full suite. + +For a **single** failed test, skip triage and go straight to the artifact: ```bash testsprite test artifact get --out ./.testsprite/runs// diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 24d8ec6..7da57cf 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -661,39 +661,29 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target defaults to claude and installs the skill file', async () => { - const { store, fs: agentFs } = makeMemFs(); - const { capture, deps } = makeCapture(); - - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - - const claudeAbs = path.resolve(CWD, TARGETS.claude.path); - expect(store.has(claudeAbs)).toBe(true); - expect(capture.stderr.join('\n')).toContain('defaulting to claude'); - }); - - it('non-TTY default writes the canonical claude content', async () => { - const { store, fs: agentFs } = makeMemFs(); + it('non-TTY with no target throws exit 5', async () => { + const { fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - await runInstall( - { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + } catch (err) { + thrown = err; + } - const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); - const abs = path.resolve(CWD, relPath); - expect(store.get(abs)).toBe(content); + expect(thrown).toBeInstanceOf(ApiError); + expect((thrown as ApiError).exitCode).toBe(5); }); it('TTY with injected prompt returning "claude" installs claude', async () => { @@ -779,14 +769,13 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 8 targets × 2 default skills = 16 rows - expect(json).toHaveLength(16); + // 7 targets × 2 default skills = 14 rows + expect(json).toHaveLength(14); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); expect(targets).toContain('windsurf'); - expect(targets).toContain('copilot'); expect(targets).toContain('antigravity'); expect(targets).toContain('kiro'); expect(targets).toContain('codex'); @@ -960,11 +949,11 @@ describe('runInstall — all own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all seven own-file targets +// Dry-run for all six own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the seven own-file targets (default 2 skills = 14 would-write lines)', async () => { + it('writes nothing for any of the six own-file targets (default 2 skills = 12 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -974,7 +963,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf', 'copilot'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -984,9 +973,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 7 targets × 2 default skills = 14 would-write lines + // 6 targets × 2 default skills = 12 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(14); + expect(wouldWriteLines.length).toBe(12); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 7d7e7c8..0e80a91 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -371,19 +371,18 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - stderrFn( - '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', + throw localValidationError( + 'target', + `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, ); - resolvedTargetStrings = ['claude']; - } else { - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } @@ -1062,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Windsurf, Antigravity, Codex)', ); agent @@ -1072,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index c0b4e00..1605f58 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -850,20 +850,6 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); - it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { - const { capture, deps } = makeCapture(); - await runWhoami( - { profile: 'default', output: 'text', debug: false, dryRun: true }, - { - ...deps, - env: { TESTSPRITE_API_URL: ' ' }, - credentialsPath, - }, - ); - const out = capture.stdout.join('\n'); - expect(out).toContain('endpoint: https://api.testsprite.com'); - }); - it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 59cd43f..bf2eb88 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -17,7 +17,7 @@ import { readProfile, writeProfile, } from '../lib/credentials.js'; -import { loadConfig, normalizeEnvVar } from '../lib/config.js'; +import { loadConfig } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -85,7 +85,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a // shell profile) is non-nullish and would short-circuit the `??` chains below to // an empty endpoint instead of falling through to the profile / prod default. - const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; // Dry-run: do not prompt, do not read env, do not write credentials. // Print the canned success shape so an agent sees exactly the JSON it @@ -208,8 +208,7 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi // displayed URL always matches where requests actually go (dogfood L1788). let resolvedEndpoint: string; if (opts.dryRun) { - resolvedEndpoint = - opts.endpointUrl ?? normalizeEnvVar(env.TESTSPRITE_API_URL) ?? 'https://api.testsprite.com'; + resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; } else { const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const config = loadConfig({ diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts deleted file mode 100644 index 230398a..0000000 --- a/src/commands/doctor.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Unit tests for `testsprite doctor`. - * - * The command reuses the real resolution helpers (loadConfig, makeHttpClient, - * isVerifySkillInstalled), so these tests inject env/credentials/fetch/fs and - * assert on the rendered report + the exit-on-failure contract. - */ - -import { mkdtempSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { CLIError } from '../lib/errors.js'; -import { writeProfile } from '../lib/credentials.js'; -import type { DoctorDeps, DoctorReport } from './doctor.js'; -import { createDoctorCommand, runDoctor } from './doctor.js'; - -interface CapturedOutput { - stdout: string[]; - stderr: string[]; -} - -function makeCapture(): { capture: CapturedOutput; deps: Pick } { - const capture: CapturedOutput = { stdout: [], stderr: [] }; - return { - capture, - deps: { - stdout: line => capture.stdout.push(line), - stderr: line => capture.stderr.push(line), - }, - }; -} - -function makeFetch(body: unknown, status = 200): DoctorDeps['fetchImpl'] { - return vi.fn( - async () => - new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' }, - }), - ) as unknown as DoctorDeps['fetchImpl']; -} - -const OK_ME = { userId: 'u-doc', keyId: 'k-doc' }; - -/** Base deps shared by the healthy-path tests: node OK, skill installed, empty env. */ -function healthyDeps(credentialsPath: string, extra: Partial = {}): DoctorDeps { - return { - env: {}, - credentialsPath, - cwd: '/project', - nodeVersion: '22.9.0', - existsSync: () => true, // skill landing file present - fetchImpl: makeFetch(OK_ME), - ...extra, - }; -} - -let credentialsPath: string; - -beforeEach(() => { - credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-doctor-')), 'credentials'); -}); - -describe('runDoctor — healthy environment', () => { - it('returns an all-passing report and does not throw', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const report = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath), ...deps }, - ); - expect(report.failures).toBe(0); - expect(report.warnings).toBe(0); - const out = capture.stdout.join('\n'); - expect(out).toContain('[OK]'); - expect(out).toContain('All checks passed.'); - expect(out).toContain('reached GET /me'); - }); - - it('never prints the API key anywhere in the report', async () => { - writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath), ...deps }, - ); - const all = capture.stdout.join('\n') + capture.stderr.join('\n'); - expect(all).not.toContain('sk-super-secret-value'); - }); - - it('emits a machine-readable report under --output json without leaking the API key', async () => { - writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - await runDoctor( - { profile: 'default', output: 'json', debug: false }, - { ...healthyDeps(credentialsPath), ...deps }, - ); - const raw = capture.stdout.join(''); - // Security: the JSON serialization path is distinct from the text renderer, - // so assert the key never leaks here either. - expect(raw).not.toContain('sk-json-secret-value'); - const parsed = JSON.parse(raw) as DoctorReport; - expect(parsed.failures).toBe(0); - expect(Array.isArray(parsed.checks)).toBe(true); - expect( - parsed.checks.some(check => check.name === 'Connectivity' && check.status === 'ok'), - ).toBe(true); - }); -}); - -describe('runDoctor — failing checks exit non-zero', () => { - it('missing API key fails Credentials and throws CLIError (exit 1)', async () => { - const { capture, deps } = makeCapture(); - const rejection = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath), ...deps }, // no profile written => no key - ).catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(CLIError); - expect(rejection).toMatchObject({ exitCode: 1 }); - const out = capture.stdout.join('\n'); - expect(out).toContain('[FAIL]'); - expect(out).toContain('Credentials'); - }); - - it('invalid endpoint URL fails the API endpoint check', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const rejection = await runDoctor( - { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, - { ...healthyDeps(credentialsPath), ...deps }, - ).catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(CLIError); - const out = capture.stdout.join('\n'); - expect(out).toContain('API endpoint'); - expect(out).toContain('not a valid'); - }); - - it('rejected API key surfaces as a Connectivity failure', async () => { - writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const authError = { - error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, - }; - const rejection = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(authError, 401) }), ...deps }, - ).catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(CLIError); - const out = capture.stdout.join('\n'); - expect(out).toContain('Connectivity'); - expect(out).toContain('API key rejected (AUTH_INVALID)'); - }); - - it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const notFound = { - error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, - }; - const rejection = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(notFound, 404) }), ...deps }, - ).catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(CLIError); - expect(capture.stdout.join('\n')).toContain('GET /me failed (NOT_FOUND)'); - }); - - it('an outdated Node runtime fails the Node.js check', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const rejection = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath, { nodeVersion: '18.0.0' }), ...deps }, - ).catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(CLIError); - const out = capture.stdout.join('\n'); - expect(out).toContain('Node.js'); - expect(out).toContain('below the required Node 20'); - }); -}); - -describe('runDoctor — warnings do not fail', () => { - it('missing verify skill is a warning, not a failure', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); - const { capture, deps } = makeCapture(); - const report = await runDoctor( - { profile: 'default', output: 'text', debug: false }, - { ...healthyDeps(credentialsPath, { existsSync: () => false }), ...deps }, - ); - expect(report.failures).toBe(0); - expect(report.warnings).toBeGreaterThanOrEqual(1); - const out = capture.stdout.join('\n'); - expect(out).toContain('[WARN]'); - expect(out).toContain('Verify skill'); - }); - - it('--dry-run skips connectivity and never calls fetch, missing key is a warning', async () => { - const fetchImpl = vi.fn(async () => { - throw new Error('fetch must not be called under --dry-run'); - }) as unknown as DoctorDeps['fetchImpl']; - const { capture, deps } = makeCapture(); - const report = await runDoctor( - { profile: 'default', output: 'text', debug: false, dryRun: true }, - { - env: {}, - credentialsPath, - cwd: '/project', - nodeVersion: '22.9.0', - existsSync: () => true, - fetchImpl, - ...deps, - }, - ); - expect(report.failures).toBe(0); - expect(fetchImpl).not.toHaveBeenCalled(); - expect(capture.stdout.join('\n')).toContain('skipped under --dry-run'); - }); -}); - -describe('createDoctorCommand wiring', () => { - it('exposes the doctor command name', () => { - expect(createDoctorCommand().name()).toBe('doctor'); - }); - - it('--help describes the diagnostic', () => { - expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); - }); -}); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts deleted file mode 100644 index 2fa0c57..0000000 --- a/src/commands/doctor.ts +++ /dev/null @@ -1,285 +0,0 @@ -/** - * `testsprite doctor` — one-shot environment diagnostic. - * - * Runs a fixed checklist (CLI version, Node.js runtime, active profile, API - * endpoint, credentials, live connectivity + key validity, and whether the - * verify skill is installed in the current project) and prints an OK/WARN/FAIL - * report. Exits non-zero when any check FAILS so it can gate a CI step or an - * agent preflight (`testsprite doctor && testsprite test run ...`). Warnings - * (e.g. skill not installed) do not fail the process. - * - * Every check is reused from the same helpers the real commands use, so the - * report reflects exactly what a subsequent command would resolve: `loadConfig` - * for profile/endpoint/key, `assertValidEndpointUrl` for the endpoint gate, - * `makeHttpClient` + `GET /me` for connectivity, and `isVerifySkillInstalled` - * for the skill check. - */ - -import { Command } from 'commander'; -import { - assertValidEndpointUrl, - makeHttpClient, - type CommonOptions as FactoryCommonOptions, -} from '../lib/client-factory.js'; -import { loadConfig } from '../lib/config.js'; -import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; -import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; -import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; -import { VERSION } from '../version.js'; -import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; - -export type DoctorStatus = 'ok' | 'warn' | 'fail'; - -export interface DoctorCheck { - /** Short, stable label (also the JSON key-ish name). */ - name: string; - status: DoctorStatus; - /** Human-readable one-line result. Never contains the API key. */ - detail: string; -} - -export interface DoctorReport { - checks: DoctorCheck[]; - failures: number; - warnings: number; -} - -/** Minimal projection of `GET /me` we read for the connectivity detail. */ -interface MeIdentity { - userId?: string; - keyId?: string; -} - -export interface DoctorDeps { - env?: NodeJS.ProcessEnv; - credentialsPath?: string; - fetchImpl?: FetchImpl; - stdout?: (line: string) => void; - stderr?: (line: string) => void; - /** Project dir for the skill check. Defaults to `process.cwd()`. */ - cwd?: string; - /** Runtime version string (e.g. "22.9.0"). Defaults to `process.versions.node`. */ - nodeVersion?: string; - existsSync?: (p: string) => boolean; - readFileSync?: (p: string) => string; -} - -type CommonOptions = FactoryCommonOptions; - -export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Promise { - const out = makeOutput(opts.output, deps); - const env = deps.env ?? process.env; - const cwd = deps.cwd ?? process.cwd(); - const nodeVersion = deps.nodeVersion ?? process.versions.node; - - const config = loadConfig({ - profile: opts.profile, - endpointUrl: opts.endpointUrl, - env, - credentialsPath: deps.credentialsPath, - }); - const endpointCheck = checkEndpoint(config.apiUrl); - const hasKey = Boolean(config.apiKey); - - const checks: DoctorCheck[] = [ - { name: 'CLI version', status: 'ok', detail: VERSION }, - checkNodeVersion(nodeVersion), - { name: 'Profile', status: 'ok', detail: config.profile }, - endpointCheck, - checkCredentials(hasKey, config.profile, opts.dryRun ?? false), - await checkConnectivity(opts, deps, { - hasKey, - endpointOk: endpointCheck.status === 'ok', - }), - checkSkill(cwd, deps), - ]; - - const failures = checks.filter(check => check.status === 'fail').length; - const warnings = checks.filter(check => check.status === 'warn').length; - const report: DoctorReport = { checks, failures, warnings }; - - out.print(report, () => renderDoctor(report)); - - if (failures > 0) { - // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent - // preflight. The full report already printed above; this line is the stderr - // summary index.ts renders before exiting 1. - throw new CLIError(`doctor: ${failures} check(s) failed, ${warnings} warning(s)`, 1); - } - return report; -} - -function checkNodeVersion(nodeVersion: string): DoctorCheck { - // Reuse the CLI's own runtime guard so the verdict matches exactly what the - // entrypoint enforces at startup, rather than a divergent hardcoded check. - // The precise engines floor (20.19+/22.13+/24+) is enforced by npm at install - // time via .npmrc engine-strict. sourceRef: src/version-guard.ts. - const rejected = shouldRejectNodeVersion(nodeVersion); - return { - name: 'Node.js', - status: rejected ? 'fail' : 'ok', - detail: rejected - ? `v${nodeVersion} is below the required Node ${MIN_SUPPORTED_NODE_MAJOR}; upgrade Node.js` - : `v${nodeVersion} (>=${MIN_SUPPORTED_NODE_MAJOR} required)`, - }; -} - -function checkEndpoint(apiUrl: string): DoctorCheck { - try { - assertValidEndpointUrl(apiUrl); - return { name: 'API endpoint', status: 'ok', detail: apiUrl }; - } catch { - return { - name: 'API endpoint', - status: 'fail', - detail: `"${apiUrl}" is not a valid http(s) URL`, - }; - } -} - -function checkCredentials(hasKey: boolean, profile: string, dryRun: boolean): DoctorCheck { - if (hasKey) { - // Never print any part of the key (security). Confirm presence only. - return { - name: 'Credentials', - status: 'ok', - detail: `API key configured (profile "${profile}")`, - }; - } - // Under --dry-run no key is expected, so a missing key is not a failure. - return { - name: 'Credentials', - status: dryRun ? 'warn' : 'fail', - detail: dryRun - ? 'no API key (not needed under --dry-run)' - : 'no API key found; run `testsprite setup` (or set TESTSPRITE_API_KEY)', - }; -} - -function checkSkill(cwd: string, deps: DoctorDeps): DoctorCheck { - const installed = isVerifySkillInstalled(cwd, { - existsSync: deps.existsSync, - readFileSync: deps.readFileSync, - }); - return { - name: 'Verify skill', - status: installed ? 'ok' : 'warn', - detail: installed - ? 'installed in this project' - : 'not installed here; run `testsprite setup` so your agent verifies its changes', - }; -} - -async function checkConnectivity( - opts: CommonOptions, - deps: DoctorDeps, - ctx: { hasKey: boolean; endpointOk: boolean }, -): Promise { - const name = 'Connectivity'; - if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; - if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; - if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; - - try { - const client = makeHttpClient(opts, { - env: deps.env, - credentialsPath: deps.credentialsPath, - fetchImpl: deps.fetchImpl, - stderr: deps.stderr, - }); - const me = await client.get('/me'); - const who = me.userId ? ` (userId ${me.userId})` : ''; - return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; - } catch (error) { - if (error instanceof ApiError) { - if ( - error.code === 'AUTH_REQUIRED' || - error.code === 'AUTH_INVALID' || - error.code === 'AUTH_FORBIDDEN' - ) { - return { name, status: 'fail', detail: `API key rejected (${error.code})` }; - } - return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; - } - return { - name, - status: 'fail', - detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, - }; - } -} - -const STATUS_LABEL: Record = { - ok: '[OK] ', - warn: '[WARN]', - fail: '[FAIL]', -}; - -function renderDoctor(report: DoctorReport): string { - const nameWidth = Math.max(...report.checks.map(check => check.name.length)); - const lines: string[] = ['TestSprite doctor', '']; - for (const check of report.checks) { - lines.push(` ${STATUS_LABEL[check.status]} ${check.name.padEnd(nameWidth)} ${check.detail}`); - } - lines.push(''); - lines.push( - report.failures === 0 && report.warnings === 0 - ? 'All checks passed.' - : `${report.failures} failure(s), ${report.warnings} warning(s).`, - ); - return lines.join('\n'); -} - -export function createDoctorCommand(deps: DoctorDeps = {}): Command { - const cmd = new Command('doctor') - .description( - 'Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .addHelpText( - 'after', - '\nExamples:\n' + - ' testsprite doctor # run all checks (exit 1 if any fails)\n' + - ' testsprite doctor --output json # machine-readable report\n' + - ' testsprite doctor && testsprite test run # gate a command on a healthy setup', - ) - .action(async (_cmdOpts, command: Command) => { - await runDoctor(resolveCommonOptions(command), deps); - }); - - return cmd; -} - -function resolveCommonOptions(command: Command): CommonOptions { - const globals = command.optsWithGlobals() as Partial & { - requestTimeout?: string; - }; - return { - profile: globals.profile ?? 'default', - output: globals.output ?? 'text', - endpointUrl: globals.endpointUrl, - debug: globals.debug ?? false, - verbose: globals.verbose ?? false, - dryRun: globals.dryRun ?? false, - requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), - }; -} - -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const seconds = Number(raw); - if (!Number.isFinite(seconds) || seconds <= 0) { - // Match the other commands: a malformed --request-timeout is a validation - // error, not a silently-ignored default. - throw localValidationError( - 'request-timeout', - `must be a positive number of seconds (got "${raw}")`, - ); - } - return Math.round(seconds * 1000); -} - -function makeOutput(mode: OutputMode, deps: DoctorDeps): Output { - return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); -} diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 617fd6f..75fdff0 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -199,14 +199,8 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); - // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite project create --type frontend'); - expect(stdout).toContain('testsprite test run --all --project '); - expect(stdout).toContain('the testsprite-onboard skill is installed'); - // No "current project" wording, no bare test list. - expect(stdout).not.toContain('current project'); - expect(stdout).not.toContain('testsprite test list'); + expect(stdout).toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -312,15 +306,6 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); - // --no-agent points at manual test creation; must not claim the skill is installed. - expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite project create --type frontend'); - expect(stdout).toContain('testsprite test create --project '); - expect(stdout).toContain('testsprite test run --all --project '); - expect(stdout).not.toContain('skill is installed'); - // No "current project" wording, no bare test list. - expect(stdout).not.toContain('current project'); - expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { diff --git a/src/commands/init.ts b/src/commands/init.ts index c2d5678..e01215a 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -15,7 +15,6 @@ import { parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; -import { normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -40,7 +39,7 @@ const DEFAULT_API_URL = 'https://api.testsprite.com'; */ function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { const env = deps.env ?? process.env; - const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; let existing: string | undefined; try { existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; @@ -407,33 +406,12 @@ function renderInitText(data: unknown): string { } lines.push(''); lines.push('Next steps:'); - lines.push(' # 1. Create your first project (frontend example) — prints a projectId'); - lines.push( - ' testsprite project create --type frontend --name "My App" --url https://your-app.com', - ); - lines.push(''); + lines.push(' testsprite test list # list tests in the current project'); + lines.push(' testsprite agent list # check installed agent targets'); if (s.agent) { lines.push( - ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', - ); - lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); - lines.push(' testsprite test run --all --project '); - lines.push(''); - lines.push(' # Manage installed agent skills'); - lines.push(' testsprite agent list'); - lines.push( - ' testsprite agent install --target= # re-install or install additional targets', - ); - } else { - lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); - lines.push(' testsprite test create --project ...'); - lines.push(' testsprite test run --all --project '); - lines.push( - ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', + ' testsprite agent install --target= # re-install or install additional targets', ); - lines.push(''); - lines.push(' # Manage installed agent skills'); - lines.push(' testsprite agent list'); } return lines.join('\n'); diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index daea851..f63928c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -8,9 +8,7 @@ import { type CliProject, type CliUpdateProjectResponse, createProjectCommand, - runAutoAuth, runCreate, - runCredential, runGet, runList, runUpdate, @@ -74,10 +72,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create, update, credential and auto-auth subcommands', () => { + it('exposes list, get, create and update subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); + expect(names).toEqual(['create', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -329,21 +327,6 @@ describe('runList', () => { }); }); -describe('DEV-244 — project update no longer accepts the dead --description flag', () => { - it('rejects --description on `project update` as an unknown option', async () => { - const project = createProjectCommand(); - const update = project.commands.find(c => c.name() === 'update')!; - project.exitOverride(); - update.exitOverride(); - - await expect( - project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { - from: 'user', - }), - ).rejects.toThrow(/unknown option.*--description/i); - }); -}); - describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -881,6 +864,7 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', + description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -945,249 +929,3 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); - -describe('runCredential', () => { - interface Captured { - url: string; - method: string; - body: unknown; - headers: Headers; - } - function captureFetch(captured: Captured[], body: unknown) { - return makeFetch((url, init) => { - captured.push({ - url, - method: init.method ?? 'GET', - body: init.body ? JSON.parse(init.body as string) : undefined, - headers: new Headers(init.headers as Record), - }); - return { status: 200, body }; - }); - } - - it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured, { - projectId: 'p1', - authType: 'Bearer token', - rewroteCount: 2, - }); - const res = await runCredential( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - authType: 'Bearer token', - credential: 'tok-123', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(res.rewroteCount).toBe(2); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.url).toContain('/projects/p1/credential'); - expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); - expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); - }); - - it('public clears the credential (no credential in body, none required)', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured, { - projectId: 'p1', - authType: 'public', - rewroteCount: 0, - }); - await runCredential( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.body).toEqual({ authType: 'public' }); - }); - - it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runCredential( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); - - it('rejects an unknown --type locally (no fetch)', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runCredential( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - authType: 'jwt', - credential: 'x', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); -}); - -describe('runAutoAuth', () => { - interface Captured { - url: string; - method: string; - body: Record; - headers: Headers; - } - function captureFetch(captured: Captured[]) { - return makeFetch((url, init) => { - captured.push({ - url, - method: init.method ?? 'GET', - body: init.body ? JSON.parse(init.body as string) : {}, - headers: new Headers(init.headers as Record), - }); - return { - status: 200, - body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, - }; - }); - } - - it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'aws_cognito_refresh', - inject: 'bearer', - region: 'us-east-1', - clientId: 'abc', - refreshToken: 'rt-xyz', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.url).toContain('/projects/p1/auto-auth'); - expect(put.body).toEqual({ - enabled: true, - method: 'aws_cognito_refresh', - inject: 'bearer', - region: 'us-east-1', - clientId: 'abc', - refreshToken: 'rt-xyz', - }); - expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); - }); - - it('--disable sends enabled:false', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - disable: true, - method: 'password', - inject: 'bearer', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); - }); - - it('reads a secret from --refresh-token-file', async () => { - const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); - const rtFile = join(dir, 'rt.txt'); - writeFileSync(rtFile, ' rt-from-file\n'); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'refresh_token', - inject: 'bearer', - tokenEndpoint: 'https://idp.example.com/token', - refreshTokenFile: rtFile, - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); - }); - - it('rejects an unknown --method / --inject locally (no fetch)', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'magic', - inject: 'bearer', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); -}); - -describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { - const noNetwork = () => { - throw new Error('network should not be hit'); - }; - - it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { - const { credentialsPath } = makeCreds(); - await expect( - runCreate( - { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, - { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { - const { credentialsPath } = makeCreds(); - await expect( - runUpdate( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, - { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); -}); diff --git a/src/commands/project.ts b/src/commands/project.ts index 96f2fc8..ef4d5e8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -142,16 +142,20 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); - // P1-3: client-side length checks matching server limits. - // Whitespace-only / empty rejection (parity with `test create`'s requireString; - // a truthy `--name " "` otherwise creates a blank-named project on the backend). - if (opts.name === undefined || opts.name.trim().length === 0) { - throw localValidationError('--name is required and must not be empty or whitespace-only'); + // Reject empty / whitespace-only names so a junk record never reaches the + // backend — matches the `requireString` whitespace guard `test create` uses + // (dogfood P1 fix #1). Without this, `--name " "` passes the action + // handler's `if (!name)` check (a non-empty string is truthy) and is sent + // verbatim, creating a blank-named project. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); } if (opts.password !== undefined && opts.password.trim().length === 0) { throw localValidationError('--password must not be empty or whitespace-only'); } - if (opts.name.length > 200) { + + // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } if (opts.description !== undefined && opts.description.length > 2000) { @@ -244,6 +248,7 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; + description?: string; instruction?: string; idempotencyKey?: string; } @@ -259,8 +264,6 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. - // Reject a whitespace-only `--name` on update too (parity with create); name - // stays optional here, so only validate when the flag is supplied. if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } @@ -270,6 +273,10 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('--description must be at most 2000 characters'); + } + // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); @@ -281,6 +288,7 @@ export async function runUpdate( targetUrl: opts.targetUrl !== undefined, username: opts.username !== undefined, password: passwordSupplied, + description: opts.description !== undefined, instruction: opts.instruction !== undefined, }; const presentFieldNames = Object.entries(mutableFields) @@ -288,7 +296,7 @@ export async function runUpdate( .map(([field]) => field); if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', ); } @@ -328,6 +336,7 @@ export async function runUpdate( targetUrl: opts.targetUrl, username: opts.username, password, + description: opts.description, instruction: opts.instruction, }; const body = Object.fromEntries( @@ -346,228 +355,6 @@ export async function runUpdate( return updated; } -// --------------------------------------------------------------------------- -// project credential — set the static backend credential -// --------------------------------------------------------------------------- - -const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; - -export interface CliProjectCredentialResponse { - projectId: string; - authType: string; - rewroteCount: number; -} - -interface CredentialOptions extends CommonOptions { - projectId: string; - authType: string; - credential?: string; - credentialFile?: string; - idempotencyKey?: string; -} - -export async function runCredential( - opts: CredentialOptions, - deps: ProjectDeps = {}, -): Promise { - const out = makeOutput(opts.output, deps); - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - assertIdempotencyKey(opts.idempotencyKey); - - if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { - throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); - } - - // Resolve the credential value (flag or file). Required for every type - // except `public` (which clears it). - let credential = opts.credential; - if (credential === undefined && opts.credentialFile !== undefined) { - credential = readFileSync(opts.credentialFile, 'utf8').trim(); - } - if (opts.authType !== 'public' && (credential === undefined || credential === '')) { - throw localValidationError( - '--credential (or --credential-file) is required unless --type is "public"', - ); - } - - const body: Record = { authType: opts.authType }; - if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; - - const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; - if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { - stderr(`idempotency-key: ${idempotencyKey}`); - } - - if (opts.dryRun) { - const sample: CliProjectCredentialResponse = { - projectId: opts.projectId, - authType: opts.authType, - rewroteCount: 0, - }; - out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); - return sample; - } - - const client = makeClient(opts, deps); - const res = await client.put( - `/projects/${encodeURIComponent(opts.projectId)}/credential`, - { body, headers: { 'idempotency-key': idempotencyKey } }, - ); - out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); - return res; -} - -function renderCredentialText(r: CliProjectCredentialResponse): string { - return [ - `projectId ${r.projectId}`, - `authType ${r.authType}`, - `rewroteCount ${r.rewroteCount}`, - ].join('\n'); -} - -// --------------------------------------------------------------------------- -// project auto-auth — configure the recurring-token (auto-refresh) login -// --------------------------------------------------------------------------- - -const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; -const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; - -export interface CliProjectAutoAuthResponse { - projectId: string; - enabled: boolean; - method: string; - inject: string; - /** - * Present when the server's trial refresh failed: `enabled` is then `false` - * and this carries the reason (e.g. a bad refresh token). The config is still - * stored, but auto-auth won't run until the login succeeds. - */ - lastRefreshError?: string; -} - -interface AutoAuthOptions extends CommonOptions { - projectId: string; - disable?: boolean; - method: string; - inject: string; - injectKey?: string; - // password method - loginUrl?: string; - loginMethod?: string; - loginContentType?: string; - loginBodyTemplate?: string; - username?: string; - password?: string; - passwordFile?: string; - tokenPath?: string; - // refresh_token method - tokenEndpoint?: string; - clientId?: string; - clientSecret?: string; - clientSecretFile?: string; - refreshToken?: string; - refreshTokenFile?: string; - scope?: string; - // aws_cognito_refresh method - region?: string; - idempotencyKey?: string; -} - -export async function runAutoAuth( - opts: AutoAuthOptions, - deps: ProjectDeps = {}, -): Promise { - const out = makeOutput(opts.output, deps); - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - assertIdempotencyKey(opts.idempotencyKey); - - if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { - throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); - } - if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { - throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); - } - - // Resolve secrets from --*-file variants so they stay out of shell history. - const password = - opts.password ?? - (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); - const clientSecret = - opts.clientSecret ?? - (opts.clientSecretFile !== undefined - ? readFileSync(opts.clientSecretFile, 'utf8').trim() - : undefined); - const refreshToken = - opts.refreshToken ?? - (opts.refreshTokenFile !== undefined - ? readFileSync(opts.refreshTokenFile, 'utf8').trim() - : undefined); - - const enabled = opts.disable !== true; - const body: Record = { enabled, method: opts.method, inject: opts.inject }; - const maybe = (k: string, v: string | undefined): void => { - if (v !== undefined) body[k] = v; - }; - maybe('injectKey', opts.injectKey); - maybe('loginUrl', opts.loginUrl); - maybe('loginMethod', opts.loginMethod); - maybe('loginContentType', opts.loginContentType); - maybe('loginBodyTemplate', opts.loginBodyTemplate); - maybe('username', opts.username); - maybe('password', password); - maybe('tokenPath', opts.tokenPath); - maybe('tokenEndpoint', opts.tokenEndpoint); - maybe('clientId', opts.clientId); - maybe('clientSecret', clientSecret); - maybe('refreshToken', refreshToken); - maybe('scope', opts.scope); - maybe('region', opts.region); - - const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; - if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { - stderr(`idempotency-key: ${idempotencyKey}`); - } - - if (opts.dryRun) { - const sample: CliProjectAutoAuthResponse = { - projectId: opts.projectId, - enabled, - method: opts.method, - inject: opts.inject, - }; - out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); - return sample; - } - - const client = makeClient(opts, deps); - const res = await client.put( - `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, - { body, headers: { 'idempotency-key': idempotencyKey } }, - ); - out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); - return res; -} - -function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { - const lines = [ - `projectId ${r.projectId}`, - `enabled ${r.enabled}`, - `method ${r.method}`, - `inject ${r.inject}`, - ]; - if (r.lastRefreshError) { - lines.push(`lastRefreshError ${r.lastRefreshError}`); - } - // A disabled result after a write means the trial login failed — call it out - // so the user doesn't assume auto-auth is live. - if (!r.enabled) { - lines.push( - 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', - ); - } - return lines.join('\n'); -} - export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -661,6 +448,7 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') + .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -677,6 +465,7 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, + description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -684,99 +473,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); - project - .command('credential ') - .description( - 'Set the static backend credential injected into every backend test\n' + - '(Bearer token / API key / Basic token / public). Free tier.', - ) - .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') - .option('--credential ', 'credential value (required unless --type public)') - .option('--credential-file ', 'read the credential value from a file') - .option( - '--idempotency-key ', - 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { - await runCredential( - { - ...resolveCommonOptions(command), - projectId, - authType: cmdOpts.type, - credential: cmdOpts.credential, - credentialFile: cmdOpts.credentialFile, - idempotencyKey: cmdOpts.idempotencyKey, - }, - deps, - ); - }); - - project - .command('auto-auth ') - .description( - 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + - 'A fresh token is fetched on each run and injected into every backend test.', - ) - .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') - .requiredOption('--inject ', 'bearer | header | cookie') - .option('--disable', 'turn auto-auth off (keeps stored config)') - .option('--inject-key ', 'header/cookie name when --inject is header/cookie') - // password method - .option('--login-url ', 'login endpoint (method=password)') - .option('--login-method ', 'POST | PUT (method=password)') - .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') - .option('--login-body-template ', 'login body template with {{username}}/{{password}}') - .option('--username ', 'login username (method=password)') - .option('--password ', 'login password (method=password)') - .option('--password-file ', 'read login password from a file') - .option('--token-path ', 'JSONPath to the token in the login response') - // refresh_token method - .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') - .option('--client-id ', 'OAuth client id') - .option('--client-secret ', 'OAuth client secret') - .option('--client-secret-file ', 'read OAuth client secret from a file') - .option('--refresh-token ', 'OAuth/Cognito refresh token') - .option('--refresh-token-file ', 'read the refresh token from a file') - .option('--scope ', 'OAuth scope') - // aws_cognito_refresh method - .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") - .option( - '--idempotency-key ', - 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { - await runAutoAuth( - { - ...resolveCommonOptions(command), - projectId, - disable: cmdOpts.disable, - method: cmdOpts.method, - inject: cmdOpts.inject, - injectKey: cmdOpts.injectKey, - loginUrl: cmdOpts.loginUrl, - loginMethod: cmdOpts.loginMethod, - loginContentType: cmdOpts.loginContentType, - loginBodyTemplate: cmdOpts.loginBodyTemplate, - username: cmdOpts.username, - password: cmdOpts.password, - passwordFile: cmdOpts.passwordFile, - tokenPath: cmdOpts.tokenPath, - tokenEndpoint: cmdOpts.tokenEndpoint, - clientId: cmdOpts.clientId, - clientSecret: cmdOpts.clientSecret, - clientSecretFile: cmdOpts.clientSecretFile, - refreshToken: cmdOpts.refreshToken, - refreshTokenFile: cmdOpts.refreshTokenFile, - scope: cmdOpts.scope, - region: cmdOpts.region, - idempotencyKey: cmdOpts.idempotencyKey, - }, - deps, - ); - }); - return project; } @@ -804,41 +500,11 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; + description?: string; instruction?: string; idempotencyKey?: string; } -interface CredentialFlagOpts { - type: string; - credential?: string; - credentialFile?: string; - idempotencyKey?: string; -} - -interface AutoAuthFlagOpts { - disable?: boolean; - method: string; - inject: string; - injectKey?: string; - loginUrl?: string; - loginMethod?: string; - loginContentType?: string; - loginBodyTemplate?: string; - username?: string; - password?: string; - passwordFile?: string; - tokenPath?: string; - tokenEndpoint?: string; - clientId?: string; - clientSecret?: string; - clientSecretFile?: string; - refreshToken?: string; - refreshTokenFile?: string; - scope?: string; - region?: string; - idempotencyKey?: string; -} - function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 9f16c4b..129d859 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4820,64 +4820,483 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { }); // --------------------------------------------------------------------------- -// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// [fix-exitcode] pollAccepted preserves ApiError exit codes (not hardcoded 1) // --------------------------------------------------------------------------- -describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { - it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { +describe('[fix-exitcode] polling error exit codes preserved in batch rerun results', () => { + it('AUTH_REQUIRED during polling → batch escalates to exitCode 3', async () => { const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a1', 'passed'); const batchResp: BatchRerunResponse = { accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_1', runId: 'run_auth_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, ], deferred: [], conflicts: [], closure: { byProject: [] }, }; + const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) { - return { status: 202, body: batchResp }; - } - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); - } + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth_fail')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_pass_a1')) return { body: passRun }; return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; const err = await runTestRerun( { testIds: ['test_1', 'test_2'], all: false, wait: true, - timeoutSeconds: 60, + timeoutSeconds: 10, autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 10, + maxConcurrency: 5, output: 'json', profile: 'default', dryRun: false, debug: false, verbose: false, }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e as { exitCode?: number; message?: string }); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('RATE_LIMITED during polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a2', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_rl', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_rl')) return errorBody('RATE_LIMITED'); + if (url.includes('/runs/run_pass_a2')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), - stderr: () => undefined, + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, ).catch(e => e); - expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); + + it('NOT_FOUND during run polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a3', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_nf', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_nf')) return errorBody('NOT_FOUND'); + if (url.includes('/runs/run_pass_a3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-auth-escalation] batch auth failure escalates to exit 3 +// --------------------------------------------------------------------------- + +describe('[fix-auth-escalation] auth error in batch rerun polling escalates to exit 3', () => { + it('auth failure in batch poll → batch exits 3, not 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_other', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_auth', runId: 'run_auth', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_other', runId: 'run_other', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_other')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_auth', 'test_other'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('mixed batch: one pass, one auth failure → exits 3 (auth wins)', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_pass', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_auth2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const passRun = makeTerminalRun('run_pass', 'passed'); + passRun.testId = 'test_1'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_pass')) return { body: passRun }; + if (url.includes('/runs/run_auth2')) return errorBody('AUTH_REQUIRED'); + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + expect((err as { message?: string }).message).toMatch(/auth error/i); + }); + + it('non-auth failure → exits 1 (no escalation)', async () => { + const creds = makeCreds(); + const failRun = makeTerminalRun('run_fail', 'failed'); + failRun.testId = 'test_1'; + const passRun = makeTerminalRun('run_pass_c3', 'passed'); + passRun.testId = 'test_2'; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_c3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_fail')) return { body: failRun }; + if (url.includes('/runs/run_pass_c3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-D4] initial chunk idempotency key bounded to ≤256 chars +// --------------------------------------------------------------------------- + +describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', () => { + it('short key with multiple chunks passes through unchanged', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // 51 test IDs forces 2 chunks (MAX_BATCH_RERUN_IDS = 50) + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + const key = h.get('idempotency-key') ?? ''; + receivedKeys.push(key); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: 'short-key', + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + expect(receivedKeys[0]).toBe('short-key:chunk0'); + expect(receivedKeys[1]).toBe('short-key:chunk1'); + expect(receivedKeys[0]!.length).toBeLessThanOrEqual(256); + expect(receivedKeys[1]!.length).toBeLessThanOrEqual(256); + }); + + it('249-char key + :chunk0 suffix would exceed 256 → key truncated to keep total ≤256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // key is 249 chars; `:chunk0` is 7 chars → 256 total (edge case, fits exactly) + const longKey = 'k'.repeat(249); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: longKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + // suffix must be preserved + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); + }); + + it('256-char key + :chunk0 suffix → base truncated so total is exactly 256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // Max-length user key: 256 chars. `:chunk0` = 7 chars → need to truncate base to 249. + const maxKey = 'x'.repeat(256); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: maxKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); }); }); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index c0ff083..2f2ea9d 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3693,33 +3693,60 @@ describe('dashboardUrl on run completion', () => { }); }); -// --------------------------------------------------------------------------- -// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty -// --------------------------------------------------------------------------- +describe('runTestRunAll — JUnit report export', () => { + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; -describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { - it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { - const { credentialsPath } = makeCreds(); - const batchResp: BatchRunFreshResponse = { - accepted: [ - { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, - { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, - ], - conflicts: [], - deferred: [], - skippedFrontend: [], - skippedIntegration: [], + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, }; + } + + it('--wait --report junit writes XML after polling', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); + const reportPath = join(dir, 'results.xml'); const fetchImpl = makeFetch((url, init) => { - if ((init.method ?? 'GET') === 'POST') return { body: batchResp }; - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_all'); - } + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; - const err = await runTestRunAll( + await runTestRunAll( { profile: 'default', output: 'json', @@ -3728,23 +3755,158 @@ describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out p wait: true, timeoutSeconds: 60, maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, }, { credentialsPath, fetchImpl, - stdout: line => stdoutLines.push(line), + stdout: () => undefined, stderr: () => undefined, sleep: instantSleep, }, - ).catch(e => e); + ); - expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; - }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_fresh_01', 'run_fresh_02']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain(' { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; + return errorBody('NOT_FOUND'); + }); + + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('failures="1"'); + expect(xml).toContain('name="test_be_02"'); + }); + + it('rejects --report without --wait', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: './results.xml', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('rejects --report-suite-name without --report', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + reportSuiteName: 'orphan-suite', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--dry-run --report junit writes canned sample XML', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('name="test_fresh_wave_01"'); + expect(xml).toContain('failures="1"'); + }); + + it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + reportSuiteName: 'ci-checkout-suite', + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain(' { 'rerun', 'result', 'run', - 'scaffold', 'steps', 'update', 'wait', @@ -155,7 +153,7 @@ describe('createTestCommand — surface', () => { expect(failure).toBeDefined(); // M2.1 piece 3 adds `summary`. `get` is the bundle entry point; // `summary` is the lightweight analysis-only triage card. - expect(failure!.commands.map(c => c.name()).sort()).toEqual(['get', 'summary']); + expect(failure!.commands.map(c => c.name()).sort()).toEqual(['get', 'summary', 'triage']); }); it('list exposes the documented filter and pagination flags (including --cursor alias)', () => { @@ -291,6 +289,16 @@ describe('createTestCommand — surface', () => { expect(help).toContain('--dry-run'); }); + it('test failure triage --help includes GLOBAL_OPTS_HINT and --project', () => { + const test = createTestCommand(); + const failure = test.commands.find(c => c.name() === 'failure')!; + const failureTriage = failure.commands.find(c => c.name() === 'triage')!; + const help = captureHelp(failureTriage); + expect(help).toContain('testsprite --help'); + expect(help).toContain('--project'); + expect(help).toContain('--max-concurrency'); + }); + it('M2 sweep: all remaining leaf subcommands include GLOBAL_OPTS_HINT', () => { // Covers list, get, create, create-batch, steps, result, update, delete, // code get, code put, plan put — the full M2 surface that the dogfood @@ -2327,94 +2335,6 @@ describe('runCodePut', () => { }); }); -describe('runScaffold', () => { - it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => { - const out: string[] = []; - const result = await runScaffold( - { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false }, - { stdout: line => out.push(line), stderr: () => undefined, env: {} }, - ); - const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> }; - expect(plan.type).toBe('frontend'); - // Placeholder project id when TESTSPRITE_PROJECT_ID is unset. - expect(plan.projectId).toContain('testsprite project list'); - expect(plan.planSteps.length).toBeGreaterThanOrEqual(2); - // Every emitted step type must come from the real enum (no drift). - for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type); - // At least one assertion step so the scaffold is a meaningful test. - expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true); - // stdout body parses back to the same plan (`> plan.json` works). - expect(JSON.parse(out.join('\n'))).toEqual(plan); - }); - - it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => { - const result = await runScaffold( - { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false }, - { - stdout: () => undefined, - stderr: () => undefined, - env: { TESTSPRITE_PROJECT_ID: 'project_env' }, - }, - ); - expect((result as { projectId: string }).projectId).toBe('project_env'); - }); - - it('backend scaffold defines a requests test with a status assertion AND calls it', async () => { - const out: string[] = []; - const result = await runScaffold( - { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false }, - { stdout: line => out.push(line), stderr: () => undefined, env: {} }, - ); - const code = (result as { code: string }).code; - expect(code).toContain('import requests'); - expect(code).toContain('assert response.status_code == 200'); - // The onboarding rule: the function must be CALLED, not just defined. - expect(code).toContain('\ntest_health_endpoint()'); - expect(out.join('\n')).toContain('import requests'); - }); - - it('--out writes the file and refuses to overwrite without --force', async () => { - const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-')); - const target = join(dir, 'plan.json'); - const opts = { - profile: 'default', - output: 'text', - debug: false, - scaffoldType: 'frontend', - out: target, - force: false, - } as const; - const deps = { stdout: () => undefined, stderr: () => undefined, env: {} }; - await runScaffold({ ...opts }, deps); - const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string }; - expect(written.type).toBe('frontend'); - // Second run without --force must not clobber the (possibly edited) file. - await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({ - code: 'VALIDATION_ERROR', - exitCode: 5, - }); - // --force overwrites. - await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined(); - }); - - it('--out pointing at an existing path (here a directory) rejects without --force', async () => { - const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-')); - await expect( - runScaffold( - { - profile: 'default', - output: 'text', - debug: false, - scaffoldType: 'frontend', - out: dir, - force: false, - }, - { stdout: () => undefined, stderr: () => undefined, env: {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); -}); - describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); @@ -3133,160 +3053,6 @@ describe('runLint', () => { }); }); -describe('runTestWaitMany', () => { - const terminalRun = (runId: string, status: string) => ({ - runId, - testId: `test_of_${runId}`, - projectId: 'project_alice', - userId: 'u1', - status, - source: 'cli', - createdAt: '2026-06-01T10:00:00.000Z', - startedAt: '2026-06-01T10:00:01.000Z', - finishedAt: '2026-06-01T10:00:30.000Z', - codeVersion: 'v1', - targetUrl: 'https://example.com', - createdFrom: null, - failedStepIndex: null, - failureKind: null, - error: null, - videoUrl: null, - stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, - }); - - it('polls every run, keeps input order, and exits 1 when one finished non-passed', async () => { - const { credentialsPath } = makeCreds(); - const fetchImpl = makeFetch(url => ({ - body: url.includes('run_bad') - ? terminalRun('run_bad', 'failed') - : terminalRun('run_ok', 'passed'), - })); - const out: string[] = []; - const rejection = await runTestWaitMany( - { - profile: 'default', - output: 'json', - debug: false, - runIds: ['run_ok', 'run_bad'], - timeoutSeconds: 30, - maxConcurrency: 2, - }, - { credentialsPath, fetchImpl, stdout: line => out.push(line) }, - ).catch((error: unknown) => error); - expect(rejection).toMatchObject({ exitCode: 1 }); - const payload = JSON.parse(out.join('')) as { - results: Array<{ runId: string; status: string }>; - summary: { passed: number; failed: number }; - }; - expect(payload.results.map(row => row.runId)).toEqual(['run_ok', 'run_bad']); - expect(payload.summary).toMatchObject({ passed: 1, failed: 1 }); - }); - - it('a member whose poll errors is captured as error: and the others survive (exit 7)', async () => { - const { credentialsPath } = makeCreds(); - const fetchImpl = makeFetch(url => { - if (url.includes('run_gone')) { - return { - status: 404, - body: { - error: { - code: 'NOT_FOUND', - message: 'no such run', - nextAction: 'check the id', - requestId: 'req_x', - details: {}, - }, - }, - }; - } - return { body: terminalRun('run_ok', 'passed') }; - }); - const out: string[] = []; - const errs: string[] = []; - const rejection = await runTestWaitMany( - { - profile: 'default', - output: 'json', - debug: false, - runIds: ['run_ok', 'run_gone'], - timeoutSeconds: 30, - maxConcurrency: 2, - }, - { - credentialsPath, - fetchImpl, - stdout: line => out.push(line), - stderr: line => errs.push(line), - }, - ).catch((error: unknown) => error); - expect(rejection).toMatchObject({ exitCode: 7 }); - const payload = JSON.parse(out.join('')) as { - results: Array<{ runId: string; status: string }>; - }; - // The failing member did not abort the pool: the passed verdict survived. - expect(payload.results[0]).toMatchObject({ runId: 'run_ok', status: 'passed' }); - expect(payload.results[1]!.status).toBe('error:NOT_FOUND'); - // The re-attach hint names the errored member (resumable) but NOT the - // already-terminal passed one. - const hint = errs.find(line => line.includes('Re-attach with:')); - expect(hint).toContain('run_gone'); - expect(hint).not.toContain('run_ok'); - }); - - it('members dequeued after the shared deadline are not granted extra poll time', async () => { - const { credentialsPath } = makeCreds(); - let fetches = 0; - const fetchImpl = makeFetch(() => { - fetches += 1; - return { body: terminalRun('run_any', 'passed') }; - }); - // timeoutSeconds 0: the shared deadline is already in the past when the - // pool starts, so every member must resolve to timeout WITHOUT polling - // (previously each dequeued member was granted a fresh 1s minimum). - const rejection = await runTestWaitMany( - { - profile: 'default', - output: 'json', - debug: false, - runIds: ['run_a', 'run_b', 'run_c'], - timeoutSeconds: 0, - maxConcurrency: 1, - }, - { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, - ).catch((error: unknown) => error); - expect(rejection).toMatchObject({ exitCode: 7 }); - expect(fetches).toBe(0); - }); - - it('an auth error escalates the exit code to 3', async () => { - const { credentialsPath } = makeCreds(); - const fetchImpl = makeFetch(() => ({ - status: 401, - body: { - error: { - code: 'AUTH_INVALID', - message: 'bad key', - nextAction: 'run setup', - requestId: 'req_y', - details: {}, - }, - }, - })); - const rejection = await runTestWaitMany( - { - profile: 'default', - output: 'json', - debug: false, - runIds: ['run_a', 'run_b'], - timeoutSeconds: 30, - maxConcurrency: 2, - }, - { credentialsPath, fetchImpl, stdout: () => undefined }, - ).catch((error: unknown) => error); - expect(rejection).toMatchObject({ exitCode: 3 }); - }); -}); - describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); @@ -3871,6 +3637,268 @@ describe('runFailureSummary', () => { }); }); +// ---------- runFailureTriage ---------- + +describe('runFailureTriage', () => { + const FAILED_TEST_A = { + id: 'test_a', + projectId: 'proj_1', + name: 'Checkout submit', + type: 'frontend' as const, + createdFrom: 'cli' as const, + status: 'failed' as const, + createdAt: '2026-06-26T10:00:00.000Z', + updatedAt: '2026-06-26T12:00:00.000Z', + }; + const FAILED_TEST_B = { + ...FAILED_TEST_A, + id: 'test_b', + name: 'Checkout validation', + updatedAt: '2026-06-26T12:01:00.000Z', + }; + const FAILED_TEST_C = { + ...FAILED_TEST_A, + id: 'test_c', + name: 'Health check', + type: 'backend' as const, + updatedAt: '2026-06-26T12:02:00.000Z', + }; + + const SHARED_REF = 'src/components/CheckoutForm.tsx:412'; + + function summaryFor(testId: string, overrides: Record = {}) { + return { + testId, + status: 'failed' as const, + failureKind: 'assertion' as const, + snapshotId: `snap_${testId}`, + rootCauseHypothesis: 'Submit button is disabled.', + recommendedFixTarget: { + kind: 'code' as const, + reference: SHARED_REF, + rationale: 'Fix validation predicate.', + }, + ...overrides, + }; + } + + it('JSON mode clusters failed tests by shared fix target', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + if (url.includes('/tests?') && url.includes('status=failed')) { + return { body: { items: [FAILED_TEST_A, FAILED_TEST_B, FAILED_TEST_C], nextToken: null } }; + } + if (url.includes('/tests/test_a/failure/summary')) { + return { body: summaryFor('test_a') }; + } + if (url.includes('/tests/test_b/failure/summary')) { + return { body: summaryFor('test_b') }; + } + if (url.includes('/tests/test_c/failure/summary')) { + return { + body: summaryFor('test_c', { + failureKind: 'network_timeout', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }), + }; + } + throw new Error(`unexpected url: ${url}`); + }); + const out: string[] = []; + const got = await runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_1', + maxConcurrency: 5, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(seen.some(u => u.includes('status=failed'))).toBe(true); + expect(got.summary.totalFailed).toBe(3); + expect(got.clusters).toHaveLength(2); + + const codeCluster = got.clusters.find(c => c.groupReason === 'fix_target'); + expect(codeCluster?.memberTestIds).toEqual(['test_a', 'test_b']); + // test_b is fresher (updatedAt) and both members have a hypothesis + expect(codeCluster?.representativeTestId).toBe('test_b'); + + const envCluster = got.clusters.find(c => c.groupReason === 'failure_kind'); + expect(envCluster?.memberTestIds).toEqual(['test_c']); + + expect(JSON.parse(out[0]!).clusters).toHaveLength(2); + }); + + it('text mode renders cluster summary lines', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests?')) { + return { body: { items: [FAILED_TEST_A], nextToken: null } }; + } + return { body: summaryFor('test_a') }; + }); + const out: string[] = []; + await runFailureTriage( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_1', + maxConcurrency: 5, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('projectId: proj_1'); + expect(block).toContain('representative: test_a'); + expect(block).toContain('Shared fix target:'); + }); + + it('dry-run emits canned clusters without network', async () => { + const out: string[] = []; + const got = await runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + maxConcurrency: 5, + }, + { stdout: line => out.push(line) }, + ); + expect(got.summary.clusterCount).toBe(2); + expect(got.clusters[0]?.groupReason).toBe('failure_kind'); + expect(JSON.parse(out[0]!).projectId).toBe('proj_dry'); + }); + + it('returns empty clusters when no failed tests match', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); + const out: string[] = []; + const got = await runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_empty', + maxConcurrency: 5, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(got.clusters).toEqual([]); + expect(got.summary.totalFailed).toBe(0); + expect(JSON.parse(out[0]!).clusters).toEqual([]); + }); + + it('skips tests whose failure summary returns NOT_FOUND', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests?')) { + return { body: { items: [FAILED_TEST_A, FAILED_TEST_B], nextToken: null } }; + } + if (url.includes('/tests/test_a/failure/summary')) { + return { body: summaryFor('test_a') }; + } + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'Test has no failing run.', + nextAction: 'No failing run.', + requestId: 'req_test', + details: { resource: 'test', id: 'test_b', reason: 'no_failing_run' }, + }, + }, + }; + }); + const got = await runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_1', + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(got.summary.totalFailed).toBe(1); + expect(got.summary.skipped).toBe(1); + expect(got.skipped?.[0]).toEqual({ testId: 'test_b', reason: 'no_failing_run' }); + expect(stderrLines.some(l => l.includes('skipped'))).toBe(true); + }); + + it('rejects missing projectId with VALIDATION_ERROR (exit 5)', async () => { + await expect( + runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: '', + maxConcurrency: 5, + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('rejects invalid --max-concurrency with VALIDATION_ERROR (exit 5)', async () => { + await expect( + runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_1', + maxConcurrency: 0, + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--filter keeps only tests whose name matches (case-insensitive)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests?')) { + return { + body: { + items: [FAILED_TEST_A, { ...FAILED_TEST_B, name: 'Profile update flow' }], + nextToken: null, + }, + }; + } + return { body: summaryFor('test_a') }; + }); + const got = await runFailureTriage( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_1', + nameFilter: 'checkout', + maxConcurrency: 5, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(got.summary.totalFailed).toBe(1); + expect(got.clusters[0]?.memberTestIds).toEqual(['test_a']); + }); +}); + // ---------- §6.7 runFailureGet ---------- const FAILED_STEPS: CliTestStep[] = [ @@ -4631,48 +4659,6 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); - it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { - const { credentialsPath } = makeCreds(); - const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); - const fetchImpl = makeFetch((url, init) => { - const method = init.method ?? 'GET'; - if (method === 'GET') return { status: 200, body: { items: [] } }; - return { - status: 200, - body: { - ...SAMPLE_RESPONSE, - type: 'backend', - warnings: [ - 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', - ], - }, - }; - }); - const out: string[] = []; - const err: string[] = []; - await runCreate( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_alice', - type: 'backend', - name: 'hardcoded be', - codeFile, - }, - { - credentialsPath, - fetchImpl, - stdout: line => out.push(line), - stderr: line => err.push(line), - }, - ); - // Warning lands on stderr, prefixed `[warn]`. - expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); - // stdout stays the parseable wire object — no warning noise. - expect(out.join('\n')).not.toContain('[warn]'); - }); - it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5716,36 +5702,6 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); - it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { - const { credentialsPath } = makeCreds(); - const codeFile = writeCodeFile('// fe code'); - const err = (await runCreate( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_fe', - type: 'frontend', - name: 'fe test', - codeFile, - produces: ['some_var'], - }, - { - credentialsPath, - fetchImpl: () => Promise.resolve(new Response('{}')), - stdout: () => undefined, - stderr: () => undefined, - }, - ).catch((e: unknown) => e)) as ApiError; - expect(err).toBeInstanceOf(ApiError); - // details.field is the BARE flag name (was '--produces' → double-dashed subject). - expect(err.details).toMatchObject({ field: 'produces' }); - expect(err.nextAction).toContain('--produces'); - expect(err.nextAction).not.toContain('----'); // regression: '----produces' - expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag - expect(err.nextAction).not.toContain('backend..'); // regression: double period - }); - it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 0602afc..e5cabb8 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,11 +1,4 @@ -import { - createWriteStream, - existsSync, - readFileSync, - readdirSync, - statSync, - type WriteStream, -} from 'node:fs'; +import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; @@ -25,7 +18,11 @@ import { writeBundle, type WriteBundleResult, } from '../lib/bundle.js'; -import { findSample, sampleJUnitReportXml } from '../lib/dry-run/samples.js'; +import { + findSample, + sampleFailureTriageResult, + sampleJUnitReportXml, +} from '../lib/dry-run/samples.js'; import { assertJUnitReportOptions, buildJUnitReport, @@ -82,6 +79,12 @@ import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; import { loadConfig } from '../lib/config.js'; +import { + buildFailureClusters, + renderFailureTriageText, + type FailureTriageInput, + type FailureTriageResult, +} from '../lib/failure-triage.js'; import { flakyExitCode, renderFlakyText, @@ -543,12 +546,6 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; - /** - * Non-fatal advisories from the backend (e.g. the BE auth guardrail - * flagging a hardcoded credential). Rendered on stderr; the create - * still succeeded. - */ - warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -758,19 +755,14 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); - if (opts.category !== undefined) depFlags.push('category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); + if (opts.category !== undefined) depFlags.push('--category'); if (depFlags.length > 0) { - // Pass the BARE flag name to localValidationError — its kind:'flag' branch - // adds the `--` prefix, so '--produces' would render as '----produces'. - const flagList = depFlags.map(f => `--${f}`); - const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; - // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + - `Remove ${flagList.join('/')} or use --type backend`, + `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + + `Remove ${depFlags.join('/')} or use --type backend.`, ); } } @@ -847,11 +839,6 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); - // Surface backend advisories (e.g. a hardcoded-credential warning for BE - // tests) on stderr so they reach the agent without polluting stdout JSON. - // Emitted before the --run early return so they always show. - emitResponseWarnings(response.warnings, deps); - // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -1014,16 +1001,6 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } -/** - * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), - * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. - */ -function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { - if (!warnings || warnings.length === 0) return; - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - for (const w of warnings) stderrFn(`[warn] ${w}`); -} - /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -1883,6 +1860,9 @@ export const DEFAULT_BATCH_RUN_CONCURRENCY = 50; /** Hard upper bound for --max-concurrency. Values above this are rejected with exit 5 (VALIDATION_ERROR). */ export const MAX_BATCH_CONCURRENCY = 100; +/** Default fan-out when fetching per-test failure summaries during triage. */ +export const DEFAULT_TRIAGE_CONCURRENCY = 5; + /** Client-side run-trigger throttle: 50 triggers per 60-second rolling window per key (sits just under the server's 60/min/key cap). */ export const BATCH_RUN_RATE_LIMIT = 50; /** Rolling window duration (ms) for the client-side trigger rate throttle. */ @@ -3426,12 +3406,6 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; - /** - * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded - * credential in the replaced code). Rendered on stderr; the update still - * succeeded. - */ - warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3609,7 +3583,6 @@ export async function runCodePut( }, }, ); - emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -4040,114 +4013,6 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { - const out = makeOutput(opts.output, deps); - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - const env = deps.env ?? process.env; - // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's - // environment carries one; otherwise a clearly-marked placeholder the user - // swaps after running `testsprite project list`. - const projectId = - typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0 - ? env.TESTSPRITE_PROJECT_ID - : ''; - - let payload: CliPlanInput | CliBackendScaffold; - let body: string; - if (opts.scaffoldType === 'frontend') { - const plan: CliPlanInput = { - projectId, - type: 'frontend', - name: 'My first frontend test', - description: 'Replace with one sentence describing what this test verifies.', - priority: 'p2', - planSteps: [ - { - type: 'action', - description: 'Navigate to /login and sign in with a seeded test account', - }, - { type: 'action', description: 'Open the first product page and click "Add to cart"' }, - { type: 'assertion', description: 'Assert that the cart badge shows 1 item' }, - ], - }; - payload = plan; - body = `${JSON.stringify(plan, null, 2)}\n`; - } else { - const code = [ - 'import requests', - '', - '# Replace with your API base URL (must be reachable from the internet).', - 'BASE_URL = "https://staging.example.com"', - '', - '', - 'def test_health_endpoint() -> None:', - ' response = requests.get(f"{BASE_URL}/health", timeout=30)', - ' assert response.status_code == 200, f"expected 200, got {response.status_code}"', - '', - '', - '# The test function MUST be called: TestSprite executes this file top to', - '# bottom, so a defined-but-never-called function would pass vacuously.', - 'test_health_endpoint()', - '', - ].join('\n'); - payload = { type: 'backend', language: 'python', code }; - body = code; - } - - if (opts.out !== undefined) { - const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); - // Never clobber silently: scaffolds are starting points the user edits, so - // an accidental re-run must not erase their work. --force opts in. - if (!opts.force && existsSync(resolved)) { - throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); - } - const sink = openOutputFile(opts.out); // reuses the directory/parent guards - const fileOut = makeFileOutput(opts.output, sink); - await fileOut.writeChunk(body); - await closeOutputFile(sink, true); - stderrFn(`Scaffold written to ${resolved}`); - return payload; - } - - // No --out: the scaffold body IS the stdout payload (`> plan.json` works). - out.print(payload, () => body.trimEnd()); - return payload; -} - export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -4569,6 +4434,146 @@ interface FailureSummaryOptions extends CommonOptions { testId: string; } +interface FailureTriageOptions extends CommonOptions { + projectId: string; + type?: 'frontend' | 'backend'; + nameFilter?: string; + maxConcurrency: number; +} + +/** + * `test failure triage --project ` — groups failed tests in a + * project into root-cause clusters using existing M2.1 analysis + * fields (`failureKind`, `recommendedFixTarget.reference`, + * `rootCauseHypothesis`). Lightweight: one `failure/summary` call + * per failed test, no bundle downloads. + * + * Client-side Phase-0 triage — deterministic heuristics only. When + * the backend ships native clustering, this command becomes a thin + * wrapper over the new read API. + */ +export async function runFailureTriage( + opts: FailureTriageOptions, + deps: TestDeps = {}, +): Promise { + requireProjectId(opts.projectId); + + if ( + !Number.isInteger(opts.maxConcurrency) || + opts.maxConcurrency < 1 || + opts.maxConcurrency > MAX_BATCH_CONCURRENCY + ) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + const dryRunResult = sampleFailureTriageResult(opts.projectId); + out.print(dryRunResult, data => renderFailureTriageText(data as FailureTriageResult)); + return dryRunResult; + } + + const client = makeClient(opts, deps); + + const failedPage = await paginate( + async ({ pageSize, cursor }) => + client.get>('/tests', { + query: { + projectId: opts.projectId, + status: 'failed', + type: opts.type, + pageSize, + cursor, + }, + }), + {}, + ); + + let failedTests = failedPage.items.filter(t => t.status === 'failed'); + if (opts.nameFilter !== undefined && opts.nameFilter !== '') { + const needle = opts.nameFilter.toLowerCase(); + failedTests = failedTests.filter(t => t.name.toLowerCase().includes(needle)); + } + + if (failedTests.length === 0) { + const empty: FailureTriageResult = { + projectId: opts.projectId, + clusters: [], + summary: { totalFailed: 0, clusterCount: 0, skipped: 0 }, + }; + out.print(empty, data => renderFailureTriageText(data as FailureTriageResult)); + return empty; + } + + stderrFn( + `Fetching failure summaries for ${failedTests.length} failed test${failedTests.length !== 1 ? 's' : ''}…`, + ); + + const triageInputs: FailureTriageInput[] = []; + const skipped: Array<{ testId: string; reason: string }> = []; + const concurrencyLimit = opts.maxConcurrency; + let nextIdx = 0; + let inFlight = 0; + + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && nextIdx < failedTests.length) { + const test = failedTests[nextIdx++]!; + inFlight++; + client + .get(`/tests/${encodeURIComponent(test.id)}/failure/summary`) + .then(summary => { + triageInputs.push({ + testId: test.id, + testName: test.name, + testType: test.type, + updatedAt: test.updatedAt, + summary: { + status: summary.status, + failureKind: summary.failureKind, + snapshotId: summary.snapshotId, + rootCauseHypothesis: summary.rootCauseHypothesis, + recommendedFixTarget: summary.recommendedFixTarget, + }, + }); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= failedTests.length) resolve(); + }) + .catch(err => { + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + skipped.push({ testId: test.id, reason: 'no_failing_run' }); + if (opts.verbose) { + stderrFn(`[triage] skipped ${test.id} — no failing run (race or stale list row)`); + } + } else { + reject(err); + } + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= failedTests.length) resolve(); + }); + } + } + startNext(); + if (failedTests.length === 0) resolve(); + }); + + const result = buildFailureClusters(opts.projectId, triageInputs); + if (skipped.length > 0) { + result.summary.skipped = skipped.length; + result.skipped = skipped; + stderrFn( + `[advisory] ${skipped.length} test${skipped.length !== 1 ? 's' : ''} skipped — listed as failed but had no failure summary (stale status or in-flight run).`, + ); + } + + out.print(result, data => renderFailureTriageText(data as FailureTriageResult)); + return result; +} + /** * `test failure summary ` — M2.1 piece 3. * @@ -5479,211 +5484,6 @@ export async function runTestRun( return finalRun; } -/** One row of the `test wait ` multi-run payload. */ -export interface CliMultiWaitResult { - runId: string; - /** Terminal run status, or 'timeout', or 'error:' when the poll failed. */ - status: string; - /** Test the run belongs to, when the poll observed it. */ - testId?: string; -} - -export interface RunTestWaitManyOptions extends CommonOptions { - runIds: string[]; - timeoutSeconds: number; - maxConcurrency: number; -} - -/** - * `test wait ` with two or more ids: attach to N already-dispatched - * runs in ONE invocation. This closes the loop the CLI itself opens: every - * batch/closure timeout prints one `testsprite test wait ` hint PER - * member, which previously meant N sequential blocking invocations. The runs - * are polled concurrently under a bounded pool with ONE shared deadline - * (`--timeout` bounds the whole invocation, not each member), each member's - * poll is total (a transient error on one run never discards the others), and - * the exit code is the worst status across members: auth errors escalate to - * exit 3, any timeout or poll error exits 7, any non-passed terminal exits 1. - * Distinct from a run journal (issue #80): no persistence, just N known ids. - */ -export async function runTestWaitMany( - opts: RunTestWaitManyOptions, - deps: TestDeps = {}, -): Promise<{ results: CliMultiWaitResult[]; summary: Record }> { - const out = makeOutput(opts.output, deps); - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - - if (opts.dryRun) { - emitDryRunBanner(stderrFn); - const results: CliMultiWaitResult[] = opts.runIds.map(runId => ({ - runId, - status: 'passed', - })); - const payload = { - results, - summary: { total: results.length, passed: results.length, failed: 0, timedOut: 0, errors: 0 }, - }; - out.print(payload, () => results.map(r => `${r.runId} ${r.status}`).join('\n')); - return payload; - } - - const client = makeClient( - { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, - deps, - ); - const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); - - // One shared deadline across every member (the whole point of the shared - // pool: `--timeout 600` means the invocation ends within ~600s, not - // 600s x ceil(N/concurrency)). - const deadlineMs = Date.now() + opts.timeoutSeconds * 1000; - - type WaitOutcome = - | { kind: 'result'; run: RunResponse } - | { kind: 'timeout' } - | { kind: 'error'; code: string; exitCode: number }; - - const pollOne = async (runId: string): Promise => { - // A member dequeued AFTER the shared deadline has passed must not be - // granted a fresh minimum poll window (with --max-concurrency 1 that - // would extend the invocation by ~1s per queued run past --timeout). - const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); - if (remainingSeconds <= 0) return { kind: 'timeout' }; - const resolveAlternate = makeBackendWaitFallback({ - client, - resolveTestId: run => run.testId, - resolveNotBefore: run => run.createdAt, - onResolved: () => undefined, - }); - try { - const run = await pollRunUntilTerminal(client, runId, { - timeoutSeconds: remainingSeconds, - sleep: deps.sleep, - onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, - onTick: (run, elapsedMs) => { - const elapsed = Math.round(elapsedMs / 1000); - ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); - }, - resolveAlternate, - }); - return { kind: 'result', run }; - } catch (err) { - if (err instanceof TimeoutError) return { kind: 'timeout' }; - if (err instanceof RequestTimeoutError) throw err; - if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; - return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; - } - }; - - const outcomes = new Map(); - let inFlight = 0; - let nextIdx = 0; - try { - await new Promise((resolve, reject) => { - const startNext = (): void => { - while (inFlight < opts.maxConcurrency && nextIdx < opts.runIds.length) { - const runId = opts.runIds[nextIdx++]!; - inFlight++; - pollOne(runId) - .then(outcome => { - outcomes.set(runId, outcome); - inFlight--; - startNext(); - if (inFlight === 0 && nextIdx >= opts.runIds.length) resolve(); - }) - // pollOne is total except for RequestTimeoutError (handled below). - .catch(reject); - } - }; - startNext(); - if (opts.runIds.length === 0) resolve(); - }); - } catch (fanOutErr) { - if (fanOutErr instanceof RequestTimeoutError) { - // Same contract as the batch pollers: leave stdout parseable before - // exiting 7. Members that already settled keep their real status; only - // the still-unfinished ids are marked running and named in the hint - // (re-attaching to an already-terminal run would be a wasted command). - ticker.finalize('Multi-run wait — request timed out'); - const partial = { - results: opts.runIds.map((runId): CliMultiWaitResult => { - const outcome = outcomes.get(runId); - if (outcome === undefined) return { runId, status: 'running' }; - if (outcome.kind === 'timeout') return { runId, status: 'timeout' }; - if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; - return { runId, status: outcome.run.status, testId: outcome.run.testId }; - }), - summary: { total: opts.runIds.length }, - }; - out.print(partial, () => partial.results.map(r => `${r.runId} ${r.status}`).join('\n')); - const unfinished = partial.results - .filter(r => r.status === 'running' || r.status === 'timeout') - .map(r => r.runId); - if (unfinished.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); - } - } - throw fanOutErr; - } - ticker.finalize(); - - const results: CliMultiWaitResult[] = opts.runIds.map(runId => { - const outcome = outcomes.get(runId); - if (outcome === undefined || outcome.kind === 'timeout') return { runId, status: 'timeout' }; - if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; - return { runId, status: outcome.run.status, testId: outcome.run.testId }; - }); - const passed = results.filter(r => r.status === 'passed').length; - const timedOut = results.filter(r => r.status === 'timeout').length; - const errors = results.filter(r => r.status.startsWith('error:')).length; - const failed = results.length - passed - timedOut - errors; - const payload = { - results, - summary: { total: results.length, passed, failed, timedOut, errors }, - }; - out.print(payload, () => - [ - ...results.map(r => `${r.runId} ${r.status}`), - '', - `${passed}/${results.length} passed, ${failed} failed/blocked, ${timedOut} timed out, ${errors} poll errors`, - ].join('\n'), - ); - - // Every member that did not reach a terminal verdict is re-attachable: - // timeouts (still running server-side) and poll errors (e.g. a transient - // transport failure) both belong in the hint; terminal runs do not. - const unfinishedIds = results - .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) - .map(r => r.runId); - if (unfinishedIds.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); - } - - // Worst-status exit: auth escalates (a rejected key fails every member the - // same way), then timeout/poll-error (7, resumable), then plain failure (1). - const authError = [...outcomes.values()].find( - o => - o.kind === 'error' && - (o.code === 'AUTH_REQUIRED' || o.code === 'AUTH_INVALID' || o.code === 'AUTH_FORBIDDEN'), - ); - if (authError !== undefined && authError.kind === 'error') { - throw new CLIError( - `Multi-run wait: authentication failed (${authError.code})`, - authError.exitCode, - ); - } - if (timedOut > 0 || errors > 0) { - throw new CLIError( - `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, - 7, - ); - } - if (failed > 0) { - throw new CLIError(`Multi-run wait: ${failed} run(s) finished non-passed`, 1); - } - return payload; -} - /** * `test wait ` — M3.3 piece-3. * @@ -5836,7 +5636,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all tests in. */ + /** projectId to run all BE tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5973,7 +5773,7 @@ export async function runTestRunAll( stderrFn(`idempotency-key: ${idempotencyKey}`); } - // Resolve testIds: fetch all tests in the project, apply --filter. + // Resolve testIds: fetch all BE tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -6011,8 +5811,7 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL tests in the project - // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). + // When no --filter, omit testIds → server runs ALL BE tests in the project. const batchResp = await client.triggerBatchRunFresh( { @@ -6305,22 +6104,6 @@ export async function runTestRunAll( }, }; } - if (err instanceof RequestTimeoutError) { - // Client-side per-request timeout during polling — classify as timeout - // (exit 7) so the fan-out completes and stdout carries every runId. - // Without this, RequestTimeoutError rejects the fan-out before out.print(), - // leaving JSON consumers with empty stdout (mirrors create-batch --run). - return { - testId: entry.testId, - runId, - status: 'timeout', - error: { - code: 'UNSUPPORTED', - message: err.message, - exitCode: err.exitCode, - }, - }; - } if (err instanceof ApiError) { // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, // RATE_LIMITED=11, …) instead of flattening every member failure to 1 @@ -7530,22 +7313,6 @@ export async function runTestRerun( }, }; } - if (err instanceof RequestTimeoutError) { - // Client-side per-request timeout during polling — classify as timeout - // (exit 7) so the fan-out completes and stdout carries every runId. - // Without this, RequestTimeoutError rejects the fan-out before out.print(), - // leaving JSON consumers with empty stdout (mirrors create-batch --run). - return { - testId: entry.testId, - runId: entry.runId, - status: 'timeout', - error: { - code: 'UNSUPPORTED', - message: err.message, - exitCode: err.exitCode, - }, - }; - } if (err instanceof ApiError) { // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the // batch exit-code aggregator can escalate auth failures correctly. Mirroring @@ -8156,34 +7923,6 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); - test - .command('scaffold') - .description( - 'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.', - ) - .option('--type ', 'frontend|backend (default: frontend)') - .option('--out ', 'write the scaffold to a file instead of stdout') - .option('--force', 'overwrite an existing --out file', false) - .addHelpText( - 'after', - '\nExamples:\n' + - ' testsprite test scaffold > first-test.plan.json\n' + - ' testsprite test scaffold --type backend --out tests/health.py\n' + - ' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => { - await runScaffold( - { - ...resolveCommonOptions(command), - scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend', - out: cmdOpts.out, - force: cmdOpts.force === true, - }, - deps, - ); - }); - test .command('steps ') .description( @@ -8403,7 +8142,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -8431,7 +8170,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -8458,14 +8197,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all project tests in wave order\n' + + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).\n' + - '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + - 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + - "reported under skippedFrontend with an advisory; run those with 'test run '.", + '(see `testsprite test create --help` for details).', ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -8481,7 +8217,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all tests in a project', + 'provide a , or use --all --project to run all BE tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -8510,16 +8246,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: a BE test's base - // URL is baked into its code, and the unified engine resolves each - // project's configured environment server-side (per-run URL overrides - // are not applied to batch FE runs either). Silently dropping it could - // run the suite against an unintended environment in the caller's mind - // — reject loudly instead. + // --target-url has no effect on the --all batch path: it is BE-only + // (FE tests are skipped server-side) and a BE test's base URL is baked + // into its code. Silently dropping it could run the suite against an + // unintended environment in the caller's mind — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', + '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', ); } await runTestRunAll( @@ -8559,53 +8293,26 @@ export function createTestCommand(deps: TestDeps = {}): Command { }); test - .command('wait ') + .command('wait ') .description( - 'Wait for one or more runs to reach a terminal status.\n' + - '\nWith several run-ids the runs are polled concurrently under one shared\n' + - '--timeout and a {results, summary} envelope is printed (worst status wins\n' + - 'the exit code), so every re-attach hint the CLI prints can be pasted as\n' + - 'ONE command.\n' + + 'Wait for a run to reach a terminal status.\n' + '\nExit codes:\n' + ' 0 passed\n' + ' 1 failed / blocked / cancelled\n' + ' 3 auth error\n' + - ' 4 run not found (single run-id; with several ids a per-member poll error\n' + - ' is recorded as error: in its row and folded into exit 7)\n' + - ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + + ' 4 run not found\n' + + ' 7 timeout — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) - .option( - '--max-concurrency ', - 'with several run-ids, max concurrent polls (1-100, default: 10)', - ) .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (runIds: string[], cmdOpts: WaitFlagOpts, command: Command) => { - // One id keeps the historical single-run path byte-identical (same - // output shape, same exit codes); two or more fan out. - if (runIds.length === 1) { - await runTestWait( - { - ...resolveCommonOptions(command), - runId: runIds[0]!, - timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), - }, - deps, - ); - return; - } - const maxConcurrency = parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? 10; - if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { - throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); - } - await runTestWaitMany( + .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { + await runTestWait( { ...resolveCommonOptions(command), - runIds, + runId, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), - maxConcurrency, }, deps, ); @@ -8991,7 +8698,6 @@ interface RunFlagOpts { interface WaitFlagOpts { timeout?: string; - maxConcurrency?: string; } interface RerunFlagOpts { @@ -10012,11 +9718,7 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option( - '--failed-only', - 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + - 'so this is usually a no-op; use `test steps ` for the full run trail.', - ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10046,11 +9748,7 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option( - '--failed-only', - 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + - 'so this is usually a no-op; use `test steps ` for the full run trail.', - ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10074,5 +9772,55 @@ function createTestFailureCommand(deps: TestDeps): Command { .action(async (testId: string, _cmdOpts, command: Command) => { await runFailureSummary({ ...resolveCommonOptions(command), testId }, deps); }); + failure + .command('triage') + .description( + 'Group all failed tests in a project into root-cause clusters (lightweight summary fan-out — no bundle downloads)', + ) + .option('--project ', 'project id (returned by `testsprite project list`)') + .option('--type ', 'filter by test type (frontend|backend)') + .option( + '--filter ', + 'only include tests whose name contains this substring (case-insensitive)', + ) + .option( + '--max-concurrency ', + `max parallel failure-summary fetches (1–${MAX_BATCH_CONCURRENCY}, default ${DEFAULT_TRIAGE_CONCURRENCY})`, + String(DEFAULT_TRIAGE_CONCURRENCY), + ) + .addHelpText( + 'after', + [ + 'Clusters are built client-side from existing M2.1 analysis fields:', + ' 1. shared recommendedFixTarget.reference', + ' 2. env-wide failureKind (infra, network, network_timeout, routing_404)', + ' 3. normalized rootCauseHypothesis prefix', + ' 4. singleton (one test per cluster)', + '', + 'After a batch run with many failures, triage first — then pull one bundle:', + ' testsprite test failure get --out ./.testsprite/failure', + '', + GLOBAL_OPTS_HINT, + ].join('\n'), + ) + .action( + async ( + cmdOpts: { project?: string; type?: string; filter?: string; maxConcurrency?: string }, + command: Command, + ) => { + await runFailureTriage( + { + ...resolveCommonOptions(command), + projectId: cmdOpts.project ?? '', + type: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES), + nameFilter: cmdOpts.filter, + maxConcurrency: + parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? + DEFAULT_TRIAGE_CONCURRENCY, + }, + deps, + ); + }, + ); return failure; } diff --git a/src/commands/usage.ts b/src/commands/usage.ts index 1980d26..a97dbba 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -154,12 +154,8 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); - // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). - // The API exposes no backend-specific per-run cost field, and it differs from - // the frontend rate, so state that it bills without asserting a possibly-wrong - // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: also consumes credits (exact amount not reported by the API)`, + `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, ); } diff --git a/src/index.ts b/src/index.ts index bf9a9aa..fb935c8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,6 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; -import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, createSetupCommand, @@ -13,7 +12,6 @@ import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; -import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; @@ -52,7 +50,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', + 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', ) .option( '--request-timeout ', @@ -91,7 +89,6 @@ program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); -program.addCommand(createDoctorCommand()); // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested @@ -164,14 +161,6 @@ program.hook('preAction', (_thisCommand, actionCommand) => { } }); -// Clean process lifecycle: a clear message + conventional exit code on SIGINT / -// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted -// `test run --wait` explains the run continues server-side; plus an EPIPE guard -// so piping to a reader that closes early (`| head`) exits cleanly instead of -// dumping a raw `write EPIPE` stack. -installSignalHandlers(); -installBrokenPipeGuard(); - // Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch // ignores them by default). No-op when no proxy variable is set. maybeInstallProxyAgent(); diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 2ac9158..0417166 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -34,9 +34,7 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing - // \r on the description and break the byte-identical comparisons. - const lines = content.split(/\r?\n/); + const lines = content.split('\n'); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { @@ -82,29 +80,19 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all eight required keys', () => { + it('has all seven required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual([ - 'antigravity', - 'claude', - 'cline', - 'codex', - 'copilot', - 'cursor', - 'kiro', - 'windsurf', - ]); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro', 'windsurf']); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, windsurf, copilot, antigravity, kiro, and codex are experimental', () => { + it('cursor, cline, windsurf, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); expect(TARGETS.windsurf.status).toBe('experimental'); - expect(TARGETS.copilot.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); @@ -124,7 +112,6 @@ describe('TARGETS', () => { expect(TARGETS.cline.mode).toBe('own-file'); expect(TARGETS.kiro.mode).toBe('own-file'); expect(TARGETS.windsurf.mode).toBe('own-file'); - expect(TARGETS.copilot.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -354,45 +341,11 @@ describe('windsurf renders within the rules-file budget', () => { }); }); -describe('renderForTarget("copilot")', () => { - const result = renderForTarget('copilot', 'testsprite-verify', STUB_BODY); - - it('returns the .github/instructions path', () => { - expect(result.path).toBe('.github/instructions/testsprite-verify.instructions.md'); - }); - - it('uses the Copilot frontmatter (applyTo + description)', () => { - expect(result.content.startsWith('---\n')).toBe(true); - expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); - expect(result.content).toContain("applyTo: '**'"); - }); - - it('does NOT carry the Claude/Cursor/Windsurf frontmatter keys', () => { - const match = /^---\n([\s\S]*?)\n---/.exec(result.content); - const fm = match?.[1] ?? ''; - expect(fm).not.toContain('name:'); // claude key - expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key - expect(fm).not.toContain('trigger:'); // windsurf Cascade key - }); - - it('renders the compact verify body (applyTo:** is always-on, so keep it small)', () => { - // Uses the REAL bodies (no stub): copilot always-injects, so like windsurf it - // ships the trimmed verify body while keeping the load-bearing command. - const copilot = renderForTarget('copilot', 'testsprite-verify'); - const claude = renderForTarget('claude', 'testsprite-verify'); - expect(copilot.content.length).toBeLessThan(claude.content.length); - expect(copilot.content).not.toContain('The verification loop that flies'); - expect(copilot.content).toContain('testsprite test run'); - }); -}); - // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { - // Full-body own-file targets. Compact-body targets (windsurf, copilot) are - // excluded — they render the trimmed verify body; see their dedicated tests. const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 7e6f94d..d9a85d5 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -9,8 +9,7 @@ export type AgentTarget = | 'antigravity' | 'codex' | 'kiro' - | 'windsurf' - | 'copilot'; + | 'windsurf'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -150,19 +149,6 @@ function wrapWindsurf(_name: string, description: string, body: string): string return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; } -/** - * GitHub Copilot reads path-specific custom instructions from - * `.github/instructions/*.instructions.md` (VS Code / Visual Studio / GitHub - * Copilot Chat). Each file carries YAML frontmatter with `applyTo` — a glob that - * scopes when the instructions attach. `applyTo: '**'` attaches the guidance to - * every request in the repo, which is what a persistent verification skill wants - * (there is no on-demand "model decides" mode for Copilot instruction files, so - * always-apply is the correct idiom). `description` is surfaced in Copilot's UI. - */ -function wrapCopilot(_name: string, description: string, body: string): string { - return `---\ndescription: ${description}\napplyTo: '**'\n---\n\n${body}\n`; -} - // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -187,8 +173,6 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.kiro/skills/${skill}/SKILL.md`; case 'windsurf': return `.windsurf/rules/${skill}.md`; - case 'copilot': - return `.github/instructions/${skill}.instructions.md`; case 'codex': return 'AGENTS.md'; } @@ -236,18 +220,6 @@ export const TARGETS: Record = { compactBody: true, wrap: wrapWindsurf, }, - copilot: { - status: 'experimental', - path: pathFor('copilot', SKILL_NAME), - mode: 'own-file', - // GitHub Copilot path-specific instructions: frontmatter carries `applyTo`. - // `applyTo: '**'` means the file is ALWAYS injected into Copilot requests - // (there is no on-demand "model decides" mode like Cursor/Windsurf), so - // render the compact body to keep the always-on context cost small — the - // same reasoning that drives windsurf's compact render. - compactBody: true, - wrap: wrapCopilot, - }, /** * codex target — managed-section mode. * diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index ff20e8c..aec451a 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -10,7 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { isAbsolute, join, resolve } from 'node:path'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -593,8 +593,8 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); - expect(isAbsolute(out)).toBe(true); + expect(out.endsWith('/tmp/x')).toBe(true); + expect(out.startsWith('/')).toBe(true); }); it('strips a trailing slash', () => { diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index d5135cc..ab5fd99 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -339,6 +339,7 @@ describe('makeHttpClient - API key validation', () => { ['smart dash', 'sk-user-abc\u2013def'], ['smart quote', 'sk-user-\u201cabc\u201d'], ['emoji', 'sk-user-abc\u{1f600}'], + ['whitespace-only', ' '], ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { const fetchImpl = vi.fn(); let caught: unknown; @@ -361,26 +362,6 @@ describe('makeHttpClient - API key validation', () => { expect(apiErr.exitCode).toBe(5); expect(apiErr.nextAction).toContain('api-key'); }); - - it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { - const fetchImpl = vi.fn(); - let caught: unknown; - try { - makeHttpClient( - { profile: 'default', output: 'json', debug: false, dryRun: false }, - { - env: { TESTSPRITE_API_KEY: ' ' } as NodeJS.ProcessEnv, - credentialsPath: NO_CREDS_PATH, - fetchImpl, - }, - ); - } catch (err) { - caught = err; - } - expect(fetchImpl).not.toHaveBeenCalled(); - expect(caught).toBeInstanceOf(ApiError); - expect((caught as ApiError).code).toBe('AUTH_REQUIRED'); - }); }); describe('assertValidApiKeyHeaderValue', () => { diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index cf56eaf..463ddcc 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -81,28 +81,6 @@ describe('loadConfig', () => { const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); expect(config.apiKey).toBe('sk-dev'); }); - - it('treats empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile)', () => { - writeProfile( - 'default', - { apiKey: 'sk-file', apiUrl: 'https://api.example.com:8443' }, - { path: credentialsPath }, - ); - const config = loadConfig({ - env: { TESTSPRITE_API_URL: ' ' }, - credentialsPath, - }); - expect(config.apiUrl).toBe('https://api.example.com:8443'); - }); - - it('treats empty / whitespace TESTSPRITE_API_KEY as unset (falls through to profile)', () => { - writeProfile('default', { apiKey: 'sk-file' }, { path: credentialsPath }); - const config = loadConfig({ - env: { TESTSPRITE_API_KEY: '' }, - credentialsPath, - }); - expect(config.apiKey).toBe('sk-file'); - }); }); describe('defaultConfigPath', () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 7f8065f..363c394 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -17,11 +17,6 @@ export interface LoadConfigOptions { const DEFAULT_API_URL = 'https://api.testsprite.com'; -/** Treat empty / whitespace-only env values as unset for `??` resolution chains. */ -export function normalizeEnvVar(value: string | undefined): string | undefined { - return value?.trim() || undefined; -} - export function defaultConfigPath(): string { return join(homedir(), '.testsprite', 'config'); } @@ -43,15 +38,9 @@ export function loadConfig(options: LoadConfigOptions = {}): Config { const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); - // Empty / whitespace-only env vars are treated as unset so they do not - // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell - // profile). Matches the normalization in auth configure and init/setup. - const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); - const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); - return { - apiUrl: options.endpointUrl ?? envApiUrl ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, - apiKey: envApiKey ?? fileEntry?.apiKey, + apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, profile, }; } diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index d50ad52..896d057 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { homedir, tmpdir } from 'node:os'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { @@ -139,11 +139,8 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - // POSIX file modes don't exist on Windows (stat reports 0666). - if (process.platform !== 'win32') { - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); - } + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -191,8 +188,7 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. - it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { + it('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -203,7 +199,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); + expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); }); }); diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index a1838d6..14987c7 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,30 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; +import { + DRY_RUN_SAMPLE_ENTRIES, + findSample, + sampleFailureTriageResult, + sampleJUnitReportXml, +} from './samples.js'; + +describe('sampleFailureTriageResult', () => { + it('returns a FailureTriageResult envelope wired for --dry-run', () => { + const result = sampleFailureTriageResult('proj_dry'); + expect(result.projectId).toBe('proj_dry'); + expect(result.summary).toEqual({ totalFailed: 3, clusterCount: 2, skipped: 0 }); + expect(result.clusters).toHaveLength(2); + expect(result.clusters[0]).toMatchObject({ + groupReason: 'failure_kind', + representativeTestId: expect.any(String), + memberTestIds: expect.any(Array), + confidence: expect.any(Number), + fixPriority: expect.any(Number), + }); + expect(result.clusters[1]).toMatchObject({ + groupReason: 'fix_target', + label: expect.stringContaining('Shared fix target:'), + }); + }); +}); describe('sampleJUnitReportXml', () => { it('returns well-formed JUnit XML with canned batch ids', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 4c7cbb0..42867e9 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -27,6 +27,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import type { FailureTriageResult } from '../failure-triage.js'; import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { @@ -341,6 +342,86 @@ const failureSummary: CliFailureSummary = { recommendedFixTarget: failureContext.failure.recommendedFixTarget, }; +/** + * Canned response for `test failure triage --dry-run`. Client-side + * orchestration command — not a single backend endpoint — so the full + * clustered envelope lives here alongside the per-route samples above. + */ +export function sampleFailureTriageResult(projectId: string): FailureTriageResult { + return { + projectId, + clusters: [ + { + clusterId: 'cluster_kind_network_timeout', + label: 'Environment issue (network_timeout)', + groupKey: 'kind:network_timeout', + groupReason: 'failure_kind', + failureKind: 'network_timeout', + representativeTestId: 'test_dryrun_a', + memberTestIds: ['test_dryrun_a', 'test_dryrun_b'], + members: [ + { + testId: 'test_dryrun_a', + testName: 'Dry-run checkout flow', + testType: 'frontend', + updatedAt: '2026-06-26T12:00:00.000Z', + status: 'failed', + failureKind: 'network_timeout', + snapshotId: 'snap_dryrun_a', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + { + testId: 'test_dryrun_b', + testName: 'Dry-run profile update', + testType: 'frontend', + updatedAt: '2026-06-26T12:01:00.000Z', + status: 'failed', + failureKind: 'network_timeout', + snapshotId: 'snap_dryrun_b', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + ], + canonicalRootCause: null, + confidence: 0.88, + fixPriority: 1, + }, + { + clusterId: 'cluster_ref_src_components_checkoutform_tsx_412', + label: 'Shared fix target: src/components/CheckoutForm.tsx:412', + groupKey: 'ref:src/components/CheckoutForm.tsx:412', + groupReason: 'fix_target', + failureKind: 'assertion', + representativeTestId: 'test_dryrun_c', + memberTestIds: ['test_dryrun_c'], + members: [ + { + testId: 'test_dryrun_c', + testName: 'Dry-run submit checkout', + testType: 'frontend', + updatedAt: '2026-06-26T12:02:00.000Z', + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_dryrun_c', + rootCauseHypothesis: + 'Submit button is disabled because the credit-card field is empty.', + recommendedFixTarget: { + kind: 'code', + reference: 'src/components/CheckoutForm.tsx:412', + rationale: 'Disabled state originates from `isFormValid()`.', + }, + }, + ], + canonicalRootCause: 'Submit button is disabled because the credit-card field is empty.', + confidence: 0.7, + fixPriority: 3, + }, + ], + summary: { totalFailed: 3, clusterCount: 2, skipped: 0 }, + }; +} + /** * Dry-run sample lookup keyed by OpenAPI operationId. Order matters in * {@link findSample}: more specific patterns must precede their generic diff --git a/src/lib/failure-triage.test.ts b/src/lib/failure-triage.test.ts new file mode 100644 index 0000000..0636c27 --- /dev/null +++ b/src/lib/failure-triage.test.ts @@ -0,0 +1,284 @@ +import { describe, expect, it } from 'vitest'; +import { + buildFailureClusters, + computeClusterConfidence, + computeFixPriority, + computeGroupKey, + normalizeHypothesis, + pickRepresentativeTestId, + type FailureTriageInput, + type FailureTriageMember, +} from './failure-triage.js'; + +function makeInput( + overrides: Partial & { testId: string }, +): FailureTriageInput { + return { + testName: overrides.testName ?? `Test ${overrides.testId}`, + testType: overrides.testType ?? 'frontend', + updatedAt: overrides.updatedAt ?? '2026-06-26T12:00:00.000Z', + summary: overrides.summary ?? { + status: 'failed', + failureKind: 'assertion', + snapshotId: `snap_${overrides.testId}`, + rootCauseHypothesis: 'Submit button is disabled.', + recommendedFixTarget: null, + }, + ...overrides, + }; +} + +describe('normalizeHypothesis', () => { + it('collapses whitespace and lowercases', () => { + expect(normalizeHypothesis(' Auth Token Expired ')).toBe('auth token expired'); + }); + + it('returns null for empty input', () => { + expect(normalizeHypothesis(null)).toBeNull(); + expect(normalizeHypothesis(' ')).toBeNull(); + }); +}); + +describe('computeGroupKey', () => { + it('groups by fix target reference first', () => { + const key = computeGroupKey( + makeInput({ + testId: 't1', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap', + rootCauseHypothesis: 'Different text', + recommendedFixTarget: { + kind: 'code', + reference: 'src/auth.ts:42', + rationale: 'Fix auth', + }, + }, + }), + ); + expect(key).toEqual({ groupKey: 'ref:src/auth.ts:42', groupReason: 'fix_target' }); + }); + + it('groups env-wide failure kinds', () => { + const key = computeGroupKey( + makeInput({ + testId: 't2', + summary: { + status: 'failed', + failureKind: 'network_timeout', + snapshotId: 'snap', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + }), + ); + expect(key).toEqual({ groupKey: 'kind:network_timeout', groupReason: 'failure_kind' }); + }); + + it('groups by normalized hypothesis when no ref or env kind', () => { + const key = computeGroupKey( + makeInput({ + testId: 't3', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap', + rootCauseHypothesis: 'Login form validation failed.', + recommendedFixTarget: null, + }, + }), + ); + expect(key.groupReason).toBe('hypothesis'); + expect(key.groupKey).toBe('hyp:login form validation failed.'); + }); + + it('falls back to singleton when no grouping signal', () => { + const key = computeGroupKey( + makeInput({ + testId: 't4', + summary: { + status: 'failed', + failureKind: 'unknown', + snapshotId: 'snap', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + }), + ); + expect(key).toEqual({ groupKey: 'singleton:t4', groupReason: 'singleton' }); + }); +}); + +describe('pickRepresentativeTestId', () => { + const members: FailureTriageMember[] = [ + { + testId: 't_old', + testName: 'Old', + testType: 'backend', + updatedAt: '2026-06-25T00:00:00.000Z', + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap1', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + { + testId: 't_rich', + testName: 'Rich', + testType: 'backend', + updatedAt: '2026-06-24T00:00:00.000Z', + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap2', + rootCauseHypothesis: 'Detailed root cause hypothesis.', + recommendedFixTarget: null, + }, + ]; + + it('prefers member with root-cause hypothesis', () => { + expect(pickRepresentativeTestId(members)).toBe('t_rich'); + }); +}); + +describe('computeClusterConfidence', () => { + it('scores multi-member fix_target clusters highest', () => { + expect(computeClusterConfidence('fix_target', 3)).toBe(0.92); + expect(computeClusterConfidence('singleton', 1)).toBe(0.4); + }); +}); + +describe('computeFixPriority', () => { + it('prioritizes infra failures first', () => { + expect(computeFixPriority('failure_kind', 'infra', 5)).toBe(1); + expect(computeFixPriority('singleton', 'assertion', 1)).toBe(10); + }); +}); + +describe('buildFailureClusters', () => { + it('merges tests sharing the same fix target into one cluster', () => { + const sharedRef = 'src/components/CheckoutForm.tsx:412'; + const result = buildFailureClusters('proj_abc', [ + makeInput({ + testId: 'test_a', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_a', + rootCauseHypothesis: 'Button disabled.', + recommendedFixTarget: { kind: 'code', reference: sharedRef, rationale: 'Fix form' }, + }, + }), + makeInput({ + testId: 'test_b', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_b', + rootCauseHypothesis: 'Cannot submit checkout.', + recommendedFixTarget: { kind: 'code', reference: sharedRef, rationale: 'Same file' }, + }, + }), + makeInput({ + testId: 'test_c', + summary: { + status: 'failed', + failureKind: 'network_timeout', + snapshotId: 'snap_c', + rootCauseHypothesis: null, + recommendedFixTarget: null, + }, + }), + ]); + + expect(result.summary).toEqual({ totalFailed: 3, clusterCount: 2, skipped: 0 }); + expect(result.clusters).toHaveLength(2); + + const envCluster = result.clusters.find(c => c.groupReason === 'failure_kind'); + expect(envCluster?.memberTestIds).toEqual(['test_c']); + expect(envCluster?.fixPriority).toBe(1); + + const codeCluster = result.clusters.find(c => c.groupReason === 'fix_target'); + expect(codeCluster?.memberTestIds).toEqual(['test_a', 'test_b']); + expect(codeCluster?.representativeTestId).toMatch(/^test_/); + expect(codeCluster?.confidence).toBe(0.92); + }); + + it('returns empty clusters when no inputs', () => { + const result = buildFailureClusters('proj_empty', []); + expect(result.clusters).toEqual([]); + expect(result.summary.totalFailed).toBe(0); + }); + + it('labels hypothesis clusters from the representative test, not map-insertion order', () => { + const sharedHyp = 'Shared login validation failure across checkout flows.'; + const result = buildFailureClusters('proj_hyp', [ + makeInput({ + testId: 't_first', + updatedAt: '2026-06-20T00:00:00.000Z', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_first', + rootCauseHypothesis: sharedHyp, + recommendedFixTarget: null, + }, + }), + makeInput({ + testId: 't_recent', + updatedAt: '2026-06-26T00:00:00.000Z', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_recent', + rootCauseHypothesis: sharedHyp, + recommendedFixTarget: null, + }, + }), + ]); + + const cluster = result.clusters[0]!; + expect(cluster.representativeTestId).toBe('t_recent'); + expect(cluster.label).toBe(sharedHyp); + expect(cluster.canonicalRootCause).toBe(sharedHyp); + }); + + it('assigns distinct clusterIds when slug prefixes would otherwise collide', () => { + const longRef = `${'a'.repeat(50)}`; + const result = buildFailureClusters('proj_collide', [ + makeInput({ + testId: 't_one', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_one', + rootCauseHypothesis: null, + recommendedFixTarget: { + kind: 'code', + reference: `${longRef}_one`, + rationale: 'First target', + }, + }, + }), + makeInput({ + testId: 't_two', + summary: { + status: 'failed', + failureKind: 'assertion', + snapshotId: 'snap_two', + rootCauseHypothesis: null, + recommendedFixTarget: { + kind: 'code', + reference: `${longRef}_two`, + rationale: 'Second target', + }, + }, + }), + ]); + + expect(result.clusters).toHaveLength(2); + expect(result.clusters[0]!.clusterId).not.toBe(result.clusters[1]!.clusterId); + expect(result.clusters[0]!.clusterId).toMatch(/_[0-9a-f]{8}$/); + expect(result.clusters[1]!.clusterId).toMatch(/_[0-9a-f]{8}$/); + }); +}); diff --git a/src/lib/failure-triage.ts b/src/lib/failure-triage.ts new file mode 100644 index 0000000..d7ea9c1 --- /dev/null +++ b/src/lib/failure-triage.ts @@ -0,0 +1,345 @@ +import { createHash } from 'node:crypto'; + +/** + * Client-side failure triage — groups per-test failure summaries into + * root-cause clusters using deterministic heuristics over existing + * M2.1 analysis fields. No LLM calls; the backend remains the source + * of truth for per-test hypotheses. + * + * @see `runFailureTriage` in `src/commands/test.ts` + */ + +/** Failure kinds that usually indicate a shared environment outage. */ +export const ENV_WIDE_FAILURE_KINDS: ReadonlySet = new Set([ + 'infra', + 'network', + 'network_timeout', + 'routing_404', +]); + +export type FailureTriageGroupReason = 'fix_target' | 'failure_kind' | 'hypothesis' | 'singleton'; + +export interface FailureTriageMember { + testId: string; + testName: string; + testType: 'frontend' | 'backend'; + updatedAt: string; + status: string; + failureKind: string | null; + snapshotId: string; + rootCauseHypothesis: string | null; + recommendedFixTarget: { + kind: string; + reference: string | null; + rationale: string | null; + } | null; +} + +export interface FailureTriageCluster { + clusterId: string; + label: string; + groupKey: string; + groupReason: FailureTriageGroupReason; + failureKind: string | null; + representativeTestId: string; + memberTestIds: string[]; + members: FailureTriageMember[]; + canonicalRootCause: string | null; + confidence: number; + fixPriority: number; +} + +export interface FailureTriageResult { + projectId: string; + clusters: FailureTriageCluster[]; + summary: { + totalFailed: number; + clusterCount: number; + skipped: number; + }; + skipped?: Array<{ testId: string; reason: string }>; +} + +export interface FailureTriageInput { + testId: string; + testName: string; + testType: 'frontend' | 'backend'; + updatedAt: string; + summary: { + status: string; + failureKind: string | null; + snapshotId: string; + rootCauseHypothesis: string | null; + recommendedFixTarget: { + kind: string; + reference: string | null; + rationale: string | null; + } | null; + }; +} + +export interface GroupKeyResult { + groupKey: string; + groupReason: FailureTriageGroupReason; +} + +/** + * Normalize a root-cause hypothesis for coarse grouping. Collapses + * whitespace, lowercases, and caps length so minor punctuation + * differences don't split clusters. + */ +export function normalizeHypothesis(hypothesis: string | null): string | null { + if (hypothesis === null || hypothesis.trim() === '') return null; + const collapsed = hypothesis.trim().replace(/\s+/g, ' ').toLowerCase(); + return collapsed.length > 100 ? collapsed.slice(0, 100) : collapsed; +} + +/** + * Derive a deterministic group key for one failed test's summary. + * Priority: shared fix target → env-wide failure kind → hypothesis prefix → singleton. + */ +export function computeGroupKey(input: FailureTriageInput): GroupKeyResult { + const ref = input.summary.recommendedFixTarget?.reference?.trim(); + if (ref) { + return { groupKey: `ref:${ref}`, groupReason: 'fix_target' }; + } + + const kind = input.summary.failureKind; + if (kind !== null && ENV_WIDE_FAILURE_KINDS.has(kind)) { + return { groupKey: `kind:${kind}`, groupReason: 'failure_kind' }; + } + + const hyp = normalizeHypothesis(input.summary.rootCauseHypothesis); + if (hyp) { + return { groupKey: `hyp:${hyp}`, groupReason: 'hypothesis' }; + } + + return { groupKey: `singleton:${input.testId}`, groupReason: 'singleton' }; +} + +/** + * Pick the representative test for a cluster. Prefers the member with the + * richest analysis (non-null hypothesis), then the most recently updated, + * then lexicographic testId for determinism. + */ +export function pickRepresentativeTestId(members: FailureTriageMember[]): string { + const sorted = [...members].sort((a, b) => { + const aHyp = a.rootCauseHypothesis !== null ? 1 : 0; + const bHyp = b.rootCauseHypothesis !== null ? 1 : 0; + if (bHyp !== aHyp) return bHyp - aHyp; + + const aTime = new Date(a.updatedAt).getTime(); + const bTime = new Date(b.updatedAt).getTime(); + if (bTime !== aTime) return bTime - aTime; + + return a.testId.localeCompare(b.testId); + }); + return sorted[0]!.testId; +} + +/** + * Confidence score for a cluster based on grouping signal strength and size. + */ +export function computeClusterConfidence( + groupReason: FailureTriageGroupReason, + memberCount: number, +): number { + if (memberCount < 1) return 0; + const multi = memberCount >= 2; + + switch (groupReason) { + case 'fix_target': + return multi ? 0.92 : 0.7; + case 'failure_kind': + return multi ? 0.88 : 0.65; + case 'hypothesis': + return multi ? 0.78 : 0.55; + case 'singleton': + return 0.4; + default: + return 0.4; + } +} + +/** + * Lower fixPriority means "fix this cluster first". + */ +export function computeFixPriority( + groupReason: FailureTriageGroupReason, + failureKind: string | null, + memberCount: number, +): number { + if (failureKind === 'infra' || failureKind === 'network' || failureKind === 'network_timeout') { + return 1; + } + if (failureKind === 'routing_404') { + return 2; + } + if (groupReason === 'fix_target' && memberCount >= 2) { + return 3; + } + if (groupReason === 'failure_kind' && memberCount >= 2) { + return 4; + } + if (groupReason === 'hypothesis' && memberCount >= 2) { + return 5; + } + if (groupReason === 'singleton') { + return 10; + } + return 6; +} + +function buildClusterLabel( + groupReason: FailureTriageGroupReason, + representative: FailureTriageMember, + failureKind: string | null, +): string { + const ref = representative.recommendedFixTarget?.reference; + + if (groupReason === 'fix_target' && ref) { + return `Shared fix target: ${ref}`; + } + if (groupReason === 'failure_kind' && failureKind !== null) { + return `Environment issue (${failureKind})`; + } + if (groupReason === 'hypothesis') { + const hyp = representative.rootCauseHypothesis; + if (hyp) { + return hyp.length > 80 ? `${hyp.slice(0, 77)}…` : hyp; + } + } + return `Independent failure: ${representative.testName}`; +} + +function slugifyClusterId(groupKey: string): string { + const slug = groupKey + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_|_$/g, ''); + const hash = createHash('sha256').update(groupKey).digest('hex').slice(0, 8); + const prefix = slug.length > 0 ? slug.slice(0, 48) : 'unknown'; + return `${prefix}_${hash}`; +} + +function toMember(input: FailureTriageInput): FailureTriageMember { + return { + testId: input.testId, + testName: input.testName, + testType: input.testType, + updatedAt: input.updatedAt, + status: input.summary.status, + failureKind: input.summary.failureKind, + snapshotId: input.summary.snapshotId, + rootCauseHypothesis: input.summary.rootCauseHypothesis, + recommendedFixTarget: input.summary.recommendedFixTarget, + }; +} + +/** + * Group triage inputs into clusters. Deterministic: same inputs always + * produce the same cluster ids and representative tests. + */ +export function buildFailureClusters( + projectId: string, + inputs: FailureTriageInput[], +): FailureTriageResult { + const groups = new Map< + string, + { reason: FailureTriageGroupReason; members: FailureTriageMember[] } + >(); + + for (const input of inputs) { + const { groupKey, groupReason } = computeGroupKey(input); + const member = toMember(input); + const existing = groups.get(groupKey); + if (existing) { + existing.members.push(member); + } else { + groups.set(groupKey, { reason: groupReason, members: [member] }); + } + } + + const clusters: FailureTriageCluster[] = []; + for (const [groupKey, { reason, members }] of groups) { + const representativeTestId = pickRepresentativeTestId(members); + const rep = members.find(m => m.testId === representativeTestId) ?? members[0]!; + const memberCount = members.length; + const failureKind = rep.failureKind; + + clusters.push({ + clusterId: `cluster_${slugifyClusterId(groupKey)}`, + label: buildClusterLabel(reason, rep, failureKind), + groupKey, + groupReason: reason, + failureKind, + representativeTestId, + memberTestIds: members.map(m => m.testId).sort(), + members: [...members].sort((a, b) => a.testId.localeCompare(b.testId)), + canonicalRootCause: rep.rootCauseHypothesis, + confidence: computeClusterConfidence(reason, memberCount), + fixPriority: computeFixPriority(reason, failureKind, memberCount), + }); + } + + clusters.sort((a, b) => { + if (a.fixPriority !== b.fixPriority) return a.fixPriority - b.fixPriority; + if (b.memberTestIds.length !== a.memberTestIds.length) { + return b.memberTestIds.length - a.memberTestIds.length; + } + return a.clusterId.localeCompare(b.clusterId); + }); + + return { + projectId, + clusters, + summary: { + totalFailed: inputs.length, + clusterCount: clusters.length, + skipped: 0, + }, + }; +} + +/** + * Text renderer for `test failure triage` output. + */ +export function renderFailureTriageText(result: FailureTriageResult): string { + const lines: string[] = []; + lines.push(`projectId: ${result.projectId}`); + lines.push( + `summary: ${result.summary.totalFailed} failed test(s) → ${result.summary.clusterCount} cluster(s)`, + ); + if (result.summary.skipped > 0) { + lines.push(`skipped: ${result.summary.skipped} test(s) could not be summarized`); + } + lines.push(''); + + if (result.clusters.length === 0) { + lines.push('No failed tests found — nothing to triage.'); + return lines.join('\n'); + } + + for (const [idx, cluster] of result.clusters.entries()) { + lines.push( + `[${idx + 1}] ${cluster.label} (confidence ${(cluster.confidence * 100).toFixed(0)}%, fix priority ${cluster.fixPriority})`, + ); + lines.push(` clusterId: ${cluster.clusterId}`); + lines.push(` groupReason: ${cluster.groupReason}`); + if (cluster.failureKind !== null) lines.push(` failureKind: ${cluster.failureKind}`); + lines.push(` representative: ${cluster.representativeTestId}`); + lines.push( + ` affected (${cluster.memberTestIds.length}): ${cluster.memberTestIds.join(', ')}`, + ); + if (cluster.canonicalRootCause !== null) { + const hyp = + cluster.canonicalRootCause.length > 120 + ? `${cluster.canonicalRootCause.slice(0, 117)}…` + : cluster.canonicalRootCause; + lines.push(` rootCause: ${hyp}`); + } + lines.push(''); + } + + return lines.join('\n').trimEnd(); +} diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index a9c1193..0320367 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -449,52 +449,6 @@ describe('HttpClient per-request timeout', () => { expect(callCount).toBe(1); }); - it('clears the per-attempt timeout after a successful response body is read', async () => { - let capturedSignal: AbortSignal | undefined; - const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { - capturedSignal = init?.signal ?? undefined; - return jsonResponse({ ok: true }); - }); - const client = new HttpClient({ - baseUrl: 'https://api.example.com/api/cli/v1', - apiKey: 'sk-test', - fetchImpl: fetchImpl as unknown as typeof fetch, - sleep: () => Promise.resolve(), - random: () => 0, - requestTimeoutMs: 25, - }); - - await expect(client.get('/me')).resolves.toEqual({ ok: true }); - expect(capturedSignal?.aborted).toBe(false); - await new Promise(resolve => setTimeout(resolve, 50)); - expect(capturedSignal?.aborted).toBe(false); - }); - - it('clears a failed attempt timeout before retry backoff sleeps', async () => { - const attemptSignals: AbortSignal[] = []; - let calls = 0; - const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { - if (init?.signal) attemptSignals.push(init.signal); - calls += 1; - if (calls === 1) return errorEnvelopeResponse(500, 'INTERNAL'); - return jsonResponse({ ok: true }); - }); - const client = new HttpClient({ - baseUrl: 'https://api.example.com/api/cli/v1', - apiKey: 'sk-test', - fetchImpl: fetchImpl as unknown as typeof fetch, - sleep: () => new Promise(resolve => setTimeout(resolve, 50)), - random: () => 0, - requestTimeoutMs: 25, - }); - - await expect(client.get('/me')).resolves.toEqual({ ok: true }); - expect(attemptSignals).toHaveLength(2); - expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); - await new Promise(resolve => setTimeout(resolve, 50)); - expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); - }); - it('caller-supplied AbortSignal still propagates as AbortError (not RequestTimeoutError)', async () => { const controller = new AbortController(); // Abort immediately @@ -521,33 +475,6 @@ describe('HttpClient per-request timeout', () => { expect((err as Error).name).toBe('AbortError'); }); - it('preserves RequestTimeoutError when the caller aborts after the request timeout wins', async () => { - const controller = new AbortController(); - const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { - return new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => { - const reason = init.signal?.reason; - controller.abort(new Error('caller aborted after timeout')); - const err = new Error(reason?.message ?? 'timed out'); - err.name = reason?.name ?? 'TimeoutError'; - reject(err); - }); - }); - }); - const client = new HttpClient({ - baseUrl: 'https://api.example.com/api/cli/v1', - apiKey: 'sk-test', - fetchImpl: fetchImpl as unknown as typeof fetch, - sleep: () => Promise.resolve(), - random: () => 0, - requestTimeoutMs: 1, - }); - const err = await client.get('/me', { signal: controller.signal }).catch(e => e); - expect(err).toBeInstanceOf(RequestTimeoutError); - expect((err as RequestTimeoutError).exitCode).toBe(7); - expect(fetchImpl).toHaveBeenCalledTimes(1); - }); - it('defaults to REQUEST_TIMEOUT_DEFAULT_MS when no requestTimeoutMs is supplied', () => { // Verify the default is wired without actually waiting 120s. // We test via the exported constant rather than exercising the stall. diff --git a/src/lib/http.ts b/src/lib/http.ts index e8d30a2..b5dabb1 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -339,7 +339,6 @@ export class HttpClient { * GET /api/cli/v1/tests/{testId}/runs * List a test's prior run history, newest-first. * - * Per the run-history read-endpoint spec §Route 1. * Limit-before-filter caveat: a `source`-filtered page may return fewer * than `pageSize` rows while still yielding a non-null `nextCursor`. * That means "none in THIS window", not end-of-history. @@ -396,15 +395,9 @@ export class HttpClient { timeoutSignal: AbortSignal, callerSignal: AbortSignal | undefined, requestId: string, - effectiveSignal: AbortSignal = timeoutSignal, ): void { if (isAbortError(err) || isTimeoutError(err)) { - const timeoutWon = - timeoutSignal.aborted && - (callerSignal == null || - !callerSignal.aborted || - effectiveSignal.reason === timeoutSignal.reason); - if (timeoutWon) { + if (timeoutSignal.aborted && (callerSignal == null || !callerSignal.aborted)) { throw new RequestTimeoutError(this.requestTimeoutMs, requestId); } throw err; @@ -435,200 +428,190 @@ export class HttpClient { // signal. The polling path supplies its own deadline-aware signal per // iteration — this timeout (120s default) is safely larger than any single // long-poll window (<=25s via ?waitSeconds), so it never bites polling. - const requestTimeout = createRequestTimeout(this.requestTimeoutMs); - const timeoutSignal = requestTimeout.signal; + const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs); const effectiveSignal = options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; try { - try { - response = await this.fetchImpl(url, { - method, - headers: this.buildHeaders(requestId, options), - body: options.body !== undefined ? JSON.stringify(options.body) : undefined, - signal: effectiveSignal, - }); - } catch (err) { - // Distinguish a client-side request timeout from a caller-supplied abort. - // - // The request-timeout controller aborts with an Error/DOMException whose - // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. - // A caller-supplied abort sets `name === 'AbortError'`. - // We treat both abort variants together: if the timeout signal fired and - // the caller hadn't already aborted, surface a clear RequestTimeoutError. - // A timeout/abort during the fetch itself: classify it (RequestTimeoutError - // when our deadline fired; otherwise rethrow the caller's abort unmodified). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); - // If a RequestTimeoutError already propagated from somewhere (e.g. from a - // nested call or from a test-injected fetchImpl), pass it through unchanged - // rather than re-wrapping it as a TransportError. - if (err instanceof RequestTimeoutError) throw err; - const message = err instanceof Error ? err.message : String(err); - this.debug({ - kind: 'error', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - durationMs: Date.now() - startedAt, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) throw new TransportError(message, requestId); - this.transition( - `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - requestTimeout.clear(); - await this.sleep(decision.delayMs); - continue; - } - - const durationMs = Date.now() - startedAt; - if (response.ok) { - this.debug({ - kind: 'response', - method, - url, - attempt, - status: response.status, - requestId, - durationMs, - }); - try { - return { body: (await response.json()) as T, requestId, status: response.status }; - } catch (err) { - // A timeout/abort can fire mid-body-read (headers received, stream stalls). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); - // Otherwise the successful response body was not valid JSON — a - // misconfigured endpoint, a proxy / captive-portal / login page that - // returns HTML with a 200 status, or an empty body. Surface a typed - // error carrying the requestId instead of letting the raw SyntaxError - // escape to index.ts, where it would print a bare `{"error":"..."}` - // and break the --output json envelope contract. - throw malformedResponseError(response, requestId, err); - } - } + response = await this.fetchImpl(url, { + method, + headers: this.buildHeaders(requestId, options), + body: options.body !== undefined ? JSON.stringify(options.body) : undefined, + signal: effectiveSignal, + }); + } catch (err) { + // Distinguish a client-side request timeout from a caller-supplied abort. + // + // Node 22 `AbortSignal.timeout()` throws a `DOMException` with + // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. + // A caller-supplied abort sets `name === 'AbortError'`. + // We treat both abort variants together: if the timeout signal fired and + // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A timeout/abort during the fetch itself: classify it (RequestTimeoutError + // when our deadline fired; otherwise rethrow the caller's abort unmodified). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); + // If a RequestTimeoutError already propagated from somewhere (e.g. from a + // nested call or from a test-injected fetchImpl), pass it through unchanged + // rather than re-wrapping it as a TransportError. + if (err instanceof RequestTimeoutError) throw err; + const message = err instanceof Error ? err.message : String(err); + this.debug({ + kind: 'error', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + durationMs: Date.now() - startedAt, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) throw new TransportError(message, requestId); + this.transition( + `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + await this.sleep(decision.delayMs); + continue; + } - let rawBody: unknown; + const durationMs = Date.now() - startedAt; + if (response.ok) { + this.debug({ + kind: 'response', + method, + url, + attempt, + status: response.status, + requestId, + durationMs, + }); try { - rawBody = await safeReadJson(response); + return { body: (await response.json()) as T, requestId, status: response.status }; } catch (err) { - // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse - // errors), so a timeout fired mid-body-read on a non-OK response lands here. - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); - throw err; + // A timeout/abort can fire mid-body-read (headers received, stream stalls). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); } + } - // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. - if (rawBody === null && isTransportEdgeStatus(response.status)) { - this.debug({ - kind: 'error', - method, - url, - attempt, - status: response.status, - requestId, - errorCode: 'TRANSPORT', - durationMs, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) { - throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); - } - this.transition( - `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - requestTimeout.clear(); - await this.sleep(decision.delayMs); - continue; - } + let rawBody: unknown; + try { + rawBody = await safeReadJson(response); + } catch (err) { + // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse + // errors), so a timeout fired mid-body-read on a non-OK response lands here. + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); + throw err; + } - const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); - // Clamp server-directed Retry-After to [1s, 300s] and surface on the - // thrown error so outer callers (e.g. runBatchRun outer retry loop) - // can honor it without re-reading the now-consumed HTTP response. - const retryAfterMsForError = - retryAfterSec !== undefined - ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 - : undefined; - const apiError = ApiError.fromEnvelope( - rawBody, - response.status, - retryAfterMsForError, - // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing - // links) resolve the environment-correct portal domain. - this.baseUrl, - ); + // Edge proxies / load balancers return 408/502/504 without our error + // envelope on transient outages. Per the CLI error spec §7 these are + // transport-level retries, not facade errors — fold them in here so + // we get the bounded backoff budget instead of a single INTERNAL bail. + if (rawBody === null && isTransportEdgeStatus(response.status)) { this.debug({ kind: 'error', method, url, attempt, status: response.status, - errorCode: apiError.code, requestId, + errorCode: 'TRANSPORT', durationMs, }); - const retryOnConflict = options.retryOnConflict !== false; - const retryOnRateLimit = options.retryOnRateLimit !== false; - const decision = apiRetryDecision( - apiError.code, - attempt, - retryAfterSec, - this.random, - retryOnConflict, - retryOnRateLimit, - ); - if (!decision.retry) throw apiError; - const delaySec = Math.round(decision.delayMs / 1000); - if (apiError.code === 'RATE_LIMITED') { - this.transition( - `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, - ); - } else if (apiError.code === 'INTERNAL') { - this.transition( - `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } else if (apiError.code === 'UNAVAILABLE') { - this.transition( - `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, - ); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) { + throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); } + this.transition( + `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); this.debug({ kind: 'retry', method, url, attempt, requestId, - errorCode: apiError.code, + errorCode: 'TRANSPORT', delayMs: decision.delayMs, }); - requestTimeout.clear(); await this.sleep(decision.delayMs); - } finally { - requestTimeout.clear(); + continue; } + + const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); + // Clamp server-directed Retry-After to [1s, 300s] and surface on the + // thrown error so outer callers (e.g. runBatchRun outer retry loop) + // can honor it without re-reading the now-consumed HTTP response. + const retryAfterMsForError = + retryAfterSec !== undefined ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 : undefined; + const apiError = ApiError.fromEnvelope( + rawBody, + response.status, + retryAfterMsForError, + // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing + // links) resolve the environment-correct portal domain. + this.baseUrl, + ); + this.debug({ + kind: 'error', + method, + url, + attempt, + status: response.status, + errorCode: apiError.code, + requestId, + durationMs, + }); + const retryOnConflict = options.retryOnConflict !== false; + const retryOnRateLimit = options.retryOnRateLimit !== false; + const decision = apiRetryDecision( + apiError.code, + attempt, + retryAfterSec, + this.random, + retryOnConflict, + retryOnRateLimit, + ); + if (!decision.retry) throw apiError; + const delaySec = Math.round(decision.delayMs / 1000); + if (apiError.code === 'RATE_LIMITED') { + this.transition( + `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, + ); + } else if (apiError.code === 'INTERNAL') { + this.transition( + `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } else if (apiError.code === 'UNAVAILABLE') { + this.transition( + `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: apiError.code, + delayMs: decision.delayMs, + }); + await this.sleep(decision.delayMs); } } @@ -638,7 +621,7 @@ export class HttpClient { accept: 'application/json', 'user-agent': `testsprite-cli/${VERSION}`, }; - // The CLI v1 facade authenticates via `x-api-key` per the CLI OpenAPI contract + // The CLI v1 facade authenticates via `x-api-key`. // (securitySchemes.ApiKeyAuth). Sending only Authorization Bearer would be // treated as a missing key by the backend. if (this.apiKey) headers['x-api-key'] = this.apiKey; @@ -704,39 +687,6 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } -interface RequestTimeoutHandle { - signal: AbortSignal; - clear: () => void; -} - -function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { - const controller = new AbortController(); - const timer = setTimeout(() => { - controller.abort(makeTimeoutReason()); - }, timeoutMs); - unrefTimer(timer); - - return { - signal: controller.signal, - clear: () => clearTimeout(timer), - }; -} - -function makeTimeoutReason(): Error { - if (typeof DOMException !== 'undefined') { - return new DOMException('The operation timed out.', 'TimeoutError'); - } - const err = new Error('The operation timed out.'); - err.name = 'TimeoutError'; - return err; -} - -function unrefTimer(timer: ReturnType): void { - if (typeof timer !== 'object' || timer === null || !('unref' in timer)) return; - const unref = (timer as { unref?: () => void }).unref; - if (typeof unref === 'function') unref.call(timer); -} - async function safeReadJson(response: Response): Promise { try { return await response.json(); @@ -810,7 +760,7 @@ export function parseRetryAfter(headerValue: string | null): number | undefined /** * HTTP statuses that arrive without our error envelope on transient * upstream outages (edge LB returning HTML, etc.). Treated as transport - * failures per the CLI error spec §7. + * failures. */ function isTransportEdgeStatus(status: number): boolean { return status === 408 || status === 502 || status === 504; diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts deleted file mode 100644 index 1fcf8eb..0000000 --- a/src/lib/interrupt.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { EventEmitter } from 'node:events'; -import { writeSync } from 'node:fs'; -import { describe, expect, it, vi } from 'vitest'; -import { - SIGINT_EXIT_CODE, - TERMINATION_EXIT_CODES, - formatInterruptMessage, - installBrokenPipeGuard, - installSignalHandlers, -} from './interrupt.js'; - -// installSignalHandlers' default stderr writes via fs.writeSync (synchronous, so -// the hint survives a piped stderr before exit); mock it to assert on that path. -vi.mock('node:fs', async importOriginal => { - const actual = (await importOriginal()) as Record; - return { ...actual, writeSync: vi.fn() }; -}); - -describe('formatInterruptMessage', () => { - it('defaults to SIGINT and explains the run continues server-side', () => { - const message = formatInterruptMessage(); - expect(message).toContain('Interrupted (SIGINT)'); - expect(message).toContain('test wait'); - expect(message).toContain('test list'); - }); - - it('names the specific signal when given one', () => { - expect(formatInterruptMessage('SIGTERM')).toContain('Interrupted (SIGTERM)'); - expect(formatInterruptMessage('SIGHUP')).toContain('Interrupted (SIGHUP)'); - }); -}); - -describe('installSignalHandlers', () => { - it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { - const handlers = new Map void>(); - const stderr: string[] = []; - const exit = vi.fn(); - - installSignalHandlers({ - on: (signal, handler) => handlers.set(signal, handler), - stderr: line => stderr.push(line), - exit, - }); - - expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); - - handlers.get('SIGINT')!(); - expect(exit).toHaveBeenLastCalledWith(130); - handlers.get('SIGTERM')!(); - expect(exit).toHaveBeenLastCalledWith(143); - handlers.get('SIGHUP')!(); - expect(exit).toHaveBeenLastCalledWith(129); - - // Each handler emits a leading blank line then the explanation. - expect(stderr[0]).toBe(''); - expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); - expect(SIGINT_EXIT_CODE).toBe(130); - expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); - expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); - }); - - it('writes the hint synchronously via writeSync before exit (survives a piped stderr)', () => { - vi.mocked(writeSync).mockClear(); - const handlers = new Map void>(); - const exit = vi.fn(); - // No stderr dep: exercise the synchronous default path. - installSignalHandlers({ - on: (signal, handler) => handlers.set(signal, handler), - exit, - }); - handlers.get('SIGINT')!(); - expect(exit).toHaveBeenCalledWith(130); - const written = vi - .mocked(writeSync) - .mock.calls.map(call => String(call[1])) - .join(''); - expect(written).toContain('Interrupted (SIGINT)'); - }); -}); - -describe('installBrokenPipeGuard', () => { - function makeEpipe(): NodeJS.ErrnoException { - return Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); - } - - it('exits 0 on stdout EPIPE (clean SIGPIPE-equivalent for `| head`)', () => { - const stdout = new EventEmitter(); - const stderr = new EventEmitter(); - const exit = vi.fn(); - installBrokenPipeGuard({ stdout, stderr, exit }); - - stdout.emit('error', makeEpipe()); - expect(exit).toHaveBeenCalledWith(0); - }); - - it('re-throws a non-EPIPE stdout error instead of silently swallowing it', () => { - const stdout = new EventEmitter(); - const stderr = new EventEmitter(); - const exit = vi.fn(); - installBrokenPipeGuard({ stdout, stderr, exit }); - - expect(() => - stdout.emit('error', Object.assign(new Error('boom'), { code: 'ENOSPC' })), - ).toThrow('boom'); - expect(exit).not.toHaveBeenCalled(); - }); - - it('swallows stderr EPIPE without exiting or throwing', () => { - const stdout = new EventEmitter(); - const stderr = new EventEmitter(); - const exit = vi.fn(); - installBrokenPipeGuard({ stdout, stderr, exit }); - - expect(() => stderr.emit('error', makeEpipe())).not.toThrow(); - expect(exit).not.toHaveBeenCalled(); - }); -}); diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts deleted file mode 100644 index cc5b4d5..0000000 --- a/src/lib/interrupt.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Process lifecycle hardening: graceful termination signals and broken-pipe. - * - * Termination signals: without a handler, Node terminates the process abruptly - * with no output, so a user (Ctrl+C), a CI runner or `docker stop` (SIGTERM), or - * a closed terminal/SSH session (SIGHUP) that interrupts a long - * `test run --wait` is left unsure whether the run was cancelled or is still - * executing server-side (it is: the CLI only polls; the run lives on the - * backend). The handler prints a one-line explanation plus how to resume, then - * exits with the conventional `128 + signal` code. - * - * Broken pipe: when output is piped to a reader that closes early - * (`testsprite ... | head`), the kernel raises `EPIPE` on the next stdout write. - * Node turns an `'error'` with no listener into an uncaughtException and dumps a - * raw `write EPIPE` stack (exit 1). The guard swallows it and exits 0, the - * conventional SIGPIPE-equivalent result for "the reader went away". - * - * `process` and the streams are injectable so the wiring is unit-testable - * without spawning a subprocess or sending a real signal. - */ - -import { writeSync } from 'node:fs'; - -/** - * Termination signals handled, mapped to their conventional `128 + signum` - * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). - */ -export const TERMINATION_EXIT_CODES = { - SIGINT: 130, // 128 + 2 - SIGTERM: 143, // 128 + 15 - SIGHUP: 129, // 128 + 1 -} as const; - -export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; - -/** Back-compat alias: SIGINT's conventional exit code. */ -export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; - -export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { - return ( - `Interrupted (${signal}). Any run already started keeps executing on the server; ` + - 'check it with `testsprite test list` or `testsprite test wait `.' - ); -} - -export interface InterruptDeps { - /** Signal registrar. Defaults to `process.on`. */ - on?: (signal: TerminationSignal, handler: () => void) => void; - /** Line-oriented stderr writer (appends a newline). */ - stderr?: (line: string) => void; - /** Process exit. Defaults to `process.exit`. */ - exit?: (code: number) => void; -} - -/** - * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a - * single top-level call in `index.ts`; not designed to be installed twice. - */ -export function installSignalHandlers(deps: InterruptDeps = {}): void { - const on = - deps.on ?? - ((signal: TerminationSignal, handler: () => void) => { - process.on(signal, handler); - }); - const stderr = - deps.stderr ?? - ((line: string) => { - // A signal handler calls process.exit() right after writing, which can - // truncate an async process.stderr.write() when stderr is a pipe. Write - // synchronously so the interrupt hint is flushed before the process exits. - try { - writeSync(process.stderr.fd, `${line}\n`); - } catch { - // Best-effort: if stderr is already gone (EPIPE), still exit cleanly. - } - }); - const exit = deps.exit ?? ((code: number) => process.exit(code)); - - for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { - on(signal, () => { - // Blank line first so the message starts on its own row rather than - // trailing the progress ticker's in-place line. - stderr(''); - stderr(formatInterruptMessage(signal)); - exit(TERMINATION_EXIT_CODES[signal]); - }); - } -} - -export interface BrokenPipeDeps { - /** stdout stream. Defaults to `process.stdout`. */ - stdout?: NodeJS.EventEmitter; - /** stderr stream. Defaults to `process.stderr`. */ - stderr?: NodeJS.EventEmitter; - /** Process exit. Defaults to `process.exit`. */ - exit?: (code: number) => void; -} - -/** - * Guard against `EPIPE` on stdout/stderr so piping to a reader that closes - * early (`testsprite ... | head`) exits cleanly instead of crashing with an - * unhandled `write EPIPE` stack. Only `EPIPE` is swallowed; any other stream - * error is left to surface normally. - */ -export function installBrokenPipeGuard(deps: BrokenPipeDeps = {}): void { - const stdout = deps.stdout ?? process.stdout; - const stderr = deps.stderr ?? process.stderr; - const exit = deps.exit ?? ((code: number) => process.exit(code)); - - stdout.on('error', (error: NodeJS.ErrnoException) => { - // Reader went away (`| head`, `| less` then q): exit cleanly like SIGPIPE - // rather than dumping an unhandled `write EPIPE` stack. Any other stdout - // error is a genuine, actionable failure, so re-throw it (Node's default). - if (error.code === 'EPIPE') { - exit(0); - return; - } - throw error; - }); - stderr.on('error', (error: NodeJS.ErrnoException) => { - // stderr closed: nothing can be reported over it, so swallow EPIPE. Any - // other error re-throws so a genuine failure is not silently hidden. - if (error.code === 'EPIPE') return; - throw error; - }); -} diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 2b26c0e..c15b09d 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -13,30 +13,24 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- -// The implementation joins paths with the native separator; normalize so the -// fakes below match on Windows (backslashes) as well as POSIX. -const toPosix = (p: string) => p.replaceAll('\\', '/'); - describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => - toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => - toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); @@ -79,7 +73,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -204,6 +198,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); + expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/version.ts b/src/version.ts index f60fdfd..efa916d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.3.0'; +export const VERSION = '0.2.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 62efe5d..984fe68 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) +Cline, Windsurf, Antigravity, Codex) Options: -h, --help display help for command @@ -29,8 +29,7 @@ into a project for a coding agent Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - windsurf, copilot, codex (comma-separated or repeated) - (default: []) + windsurf, codex (comma-separated or repeated) (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -116,8 +115,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, windsurf, copilot, codex (default: - claude) (default: "claude") + cursor, cline, kiro, windsurf, codex (default: claude) + (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -133,29 +132,20 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - credential [options] Set the static backend credential injected - into every backend test - (Bearer token / API key / Basic token / - public). Free tier. - auto-auth [options] Configure the recurring-token - (auto-refresh login) for backend tests - (Pro). - A fresh token is fetched on each run and - injected into every backend test. - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + help [command] display help for command " `; @@ -210,10 +200,6 @@ Commands: (--plan-from, FE-only, M3.2 piece-5) create-batch [options] Create multiple FE tests from a JSONL of plan specs (FE-only) - scaffold [options] Emit a schema-correct starter test - definition (frontend plan JSON by - default, or a backend Python skeleton). - Pure-local: no network, no credentials. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) @@ -251,7 +237,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all tests in a project (M4). + Use --all --project for a wave-ordered batch run of all BE tests (M4). Exit codes: 0 passed (or queued without --wait) @@ -265,20 +251,14 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get - wait [options] Wait for one or more runs to reach a terminal status. - - With several run-ids the runs are polled concurrently under one shared - --timeout and a {results, summary} envelope is printed (worst status wins - the exit code), so every re-attach hint the CLI prints can be pasted as - ONE command. + wait [options] Wait for a run to reach a terminal status. Exit codes: 0 passed 1 failed / blocked / cancelled 3 auth error - 4 run not found (single run-id; with several ids a per-member poll error - is recorded as error: in its row and folded into exit 7) - 7 timeout or per-member poll error — resume with: testsprite test wait + 4 run not found + 7 timeout — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get @@ -371,9 +351,7 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Trim to the failed step ±1. The bundle is already - failure-focused server-side, so this is usually a no-op; use - \`test steps \` for the full run trail. + --failed-only Keep only the failed step plus its immediate neighbors (±1) -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -413,6 +391,35 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test failure triage 1`] = ` +"Usage: testsprite test failure triage [options] + +Group all failed tests in a project into root-cause clusters (lightweight +summary fan-out — no bundle downloads) + +Options: + --project project id (returned by \`testsprite project list\`) + --type filter by test type (frontend|backend) + --filter only include tests whose name contains this substring + (case-insensitive) + --max-concurrency max parallel failure-summary fetches (1–100, default + 5) (default: "5") + -h, --help display help for command +Clusters are built client-side from existing M2.1 analysis fields: + 1. shared recommendedFixTarget.reference + 2. env-wide failureKind (infra, network, network_timeout, routing_404) + 3. normalized rootCauseHypothesis prefix + 4. singleton (one test per cluster) + +After a batch run with many failures, triage first — then pull one bundle: + testsprite test failure get --out ./.testsprite/failure + + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test get 1`] = ` "Usage: testsprite test get [options] @@ -559,7 +566,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all tests in a project (M4). +Use --all --project for a wave-ordered batch run of all BE tests (M4). Exit codes: 0 passed (or queued without --wait) @@ -583,9 +590,9 @@ Options: 600) --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. - --all run all tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) --project project id (required with --all; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains @@ -600,17 +607,13 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all project tests in wave order + testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Frontend tests: the current unified engine runs FE tests too (they are billed -like any run). On the legacy backend-only engine FE tests cannot run — they are -reported under skippedFrontend with an advisory; run those with 'test run '. - Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " @@ -655,13 +658,10 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network and credentials; emit a canned - sample matching the OpenAPI contract. Useful for - learning the CLI surface without an API key. - Note: file inputs you pass - (--plan-from/--plans/--steps) are still read and - validated locally; only --code-file uses a - placeholder. + --dry-run Skip the network, credentials, and filesystem; + emit a canned sample matching the OpenAPI + contract. Useful for learning the CLI surface + without an API key. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override @@ -679,12 +679,10 @@ Commands: project Manage TestSprite projects test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent - config (Claude Code, Cursor, Cline, Antigravity, - Kiro, Windsurf, Copilot, Codex) + config (Claude Code, Cursor, Cline, Windsurf, + Antigravity, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) - doctor Diagnose CLI setup: version, Node, profile, - endpoint, credentials, connectivity, skill help [command] display help for command " `; diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 959f212..b0537b1 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; +import { existsSync, mkdtempSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; @@ -364,10 +364,7 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, - // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — - // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, - USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -900,10 +897,7 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - // POSIX file modes don't exist on Windows (stat reports 0666). - if (process.platform !== 'win32') { - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); - } + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -1050,7 +1044,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - rmSync(credPath, { force: true }); + if (existsSync(credPath)) execFileSync('rm', [credPath]); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 4fb4584..094aec2 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -174,11 +174,6 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); - } else if (target === 'copilot') { - // GitHub Copilot instructions frontmatter: applyTo glob + description - expect(content.startsWith('---'), `copilot: should start with ---`).toBe(true); - expect(content).toContain("applyTo: '**'"); - expect(content).toContain('description:'); } // (b) branding — the renamed H1 must be present in every body variant @@ -216,9 +211,6 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); - } else if (target === 'copilot') { - expect(content.startsWith('---'), `copilot/onboard: should start with ---`).toBe(true); - expect(content).toContain("applyTo: '**'"); } // Load-bearing onboard string: the skill body must reference setup @@ -811,7 +803,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 8 targets × 2 skills = 16 rows + // Expected: 7 targets × 2 skills = 14 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -847,7 +839,6 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', - 'copilot', 'codex', ]); }); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index 5a07434..3b35751 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -229,7 +229,6 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', - 'copilot', 'codex', ]); }); diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index ee89b3b..d6850e2 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -38,6 +38,7 @@ const cases: Array<[string, string[]]> = [ ['test steps', ['test', 'steps', '--help']], ['test result', ['test', 'result', '--help']], ['test failure get', ['test', 'failure', 'get', '--help']], + ['test failure triage', ['test', 'failure', 'triage', '--help']], ['test rerun', ['test', 'rerun', '--help']], ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts deleted file mode 100644 index c798bb3..0000000 --- a/test/helpers/hermetic-env.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). - * - * Two leaks this closes, both of which made results depend on the - * developer's machine: - * - * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` - * precedence over the credentials file, so a key exported in the - * developer's shell silently overrode test fixtures. - * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but - * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` - * recipe never isolated Windows runs. Both vars are redirected to a - * throwaway dir so no test can read or write `~/.testsprite`. - * - * Tests that need these vars set them explicitly (on `process.env` or via - * injected `env` deps) after this runs. - */ -import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; -import { homedir, tmpdir } from 'node:os'; -import { join } from 'node:path'; - -const realHome = homedir(); -const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); -if (process.platform === 'win32') { - // Node version shims (Volta) resolve LocalAppData under USERPROFILE and - // abort if it's missing, which would break the `npm run build` beforeAll - // in the subprocess/snapshot suites. - mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); -} -// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless -// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. -const realVoltaHome = join(realHome, '.volta'); -if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { - process.env.VOLTA_HOME = realVoltaHome; -} -process.env.HOME = hermeticHome; -process.env.USERPROFILE = hermeticHome; - -for (const key of Object.keys(process.env)) { - if (key.startsWith('TESTSPRITE_')) delete process.env[key]; -} diff --git a/vitest.config.ts b/vitest.config.ts index bd9b2ad..add8f0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,9 +4,6 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], - // Strip real TESTSPRITE_* env vars and redirect the home dir so results - // never depend on the developer's shell or ~/.testsprite (see the file). - setupFiles: ['./test/helpers/hermetic-env.ts'], // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). fileParallelism: false,