diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 468c8ed..0000000 --- a/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -# Check out all text files with LF on every platform. Tests compare bytes -# from checked-out files (skill templates, snapshots); a CRLF working tree -# (core.autocrlf on Windows) broke those comparisons. -* text=auto eol=lf - -*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1c165f6..c490f9b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,9 +12,7 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml deleted file mode 100644 index d34744f..0000000 --- a/.github/workflows/ci-nudge.yml +++ /dev/null @@ -1,126 +0,0 @@ -# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). -# When a PR's CI finishes red, posts ONE sticky comment listing exactly which -# jobs failed and the local one-liner that reproduces/fixes each (the automated -# version of the hand-written "run `npm run format` and you're green" review -# comments). The comment flips to a green confirmation once all checks pass. -# -# Runs in base-repo context via workflow_run — no PR code is checked out, so -# fork PRs are safe. State is recomputed from check-runs each time, so the two -# CI workflows (CI + Test Coverage) can complete in any order. -# -# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as -# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to -# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). -name: CI failure nudge - -on: - workflow_run: - workflows: ['CI', 'Test Coverage'] - types: [completed] - -permissions: - checks: read # read the head SHA's check-run state - pull-requests: write - issues: write # PR comments ride the issues API - -env: - MARKER: '' - -jobs: - nudge: - # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). - if: >- - github.repository == 'TestSprite/testsprite-cli' && - github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest - steps: - - id: app-token - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 - env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - with: - script: | - const { owner, repo } = context.repo; - const run = context.payload.workflow_run; - const marker = process.env.MARKER; - - const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) - : null; - async function write(fn) { - if (appClient) { - try { return await fn(appClient.rest); } - catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } - } - return await fn(github.rest); - } - - // Resolve the PR for this run. `workflow_run.pull_requests` is empty - // for fork PRs, so fall back to the commit→PRs lookup. - let pr = (run.pull_requests || [])[0]; - if (!pr) { - const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( - { owner, repo, commit_sha: run.head_sha }); - pr = data.find(p => p.state === 'open'); - } else { - pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; - } - if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; - - // Recompute full CI state from this SHA's github-actions check runs. - const checks = await github.paginate(github.rest.checks.listForRef, - { owner, repo, ref: run.head_sha, per_page: 100 }); - const ours = checks.filter(c => c.app && c.app.slug === 'github-actions'); - const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); - const pending = ours.filter(c => c.status !== 'completed'); - - const comments = await github.paginate(github.rest.issues.listComments, - { owner, repo, issue_number: pr.number, per_page: 100 }); - const sticky = comments.find(c => (c.body || '').includes(marker)); - - // Job name → the local command that reproduces/fixes it. - const FIX = { - 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', - 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', - 'Unit Tests': 'run `npm test` and fix the failing tests', - 'Build': 'run `npm run build` and fix the compile errors', - 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', - 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', - }; - - if (failing.length === 0) { - // Only speak up on success if we previously flagged a failure, and - // only once everything has actually finished. - if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { - await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, - body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); - } - return; - } - - const lines = failing - .sort((a, b) => a.name.localeCompare(b.name)) - .map(c => { - const fix = FIX[c.name] || 'see the logs for details'; - return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; - }); - const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` - + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` - + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` - + `flips green automatically once all checks pass.`; - - if (sticky) { - if (sticky.body !== body) { - await write(rest => rest.issues.updateComment( - { owner, repo, comment_id: sticky.id, body })); - } - } else { - await write(rest => rest.issues.createComment( - { owner, repo, issue_number: pr.number, body })); - } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index e9923dd..63323c0 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,15 +1,13 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. +# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → +# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# TESTSPRITE_HOB_APP_ID = the App's numeric App ID -# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) -# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from -# before it was renamed — same App ID + key; either naming works.) +# ALFHEIM_AGENT_APP_ID = the App's numeric App ID +# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -32,23 +30,23 @@ env: jobs: triage: # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} runs-on: ubuntu-latest steps: - # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. + # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 with: - # App token when available (→ testsprite-hob[bot]); else the default + # App token when available (→ alfheim-agent[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml deleted file mode 100644 index a9cd23d..0000000 --- a/.github/workflows/pr-triage.yml +++ /dev/null @@ -1,189 +0,0 @@ -# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). -# Enforces the issue-first workflow on community PRs: every non-docs PR must -# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR -# author (claimed via `/assign`, which issue-triage.yml handles). Violations -# get a `needs-issue` label + a sticky comment, and — for PRs opened after -# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. -# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. -# -# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; -# the comment tells the contributor to edit the PR description or push a -# commit (`edited` / `synchronize`) to re-run the gate. -# -# Runs with NO checkout — metadata-only, so it is safe under pull_request_target -# (which is required for fork PRs to get a write-capable token). -# -# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The -# App currently has Issues R/W + Metadata only — commenting/labeling a PULL -# REQUEST needs the "Pull requests: Read & write" App permission (issues-API -# endpoints are permission-checked by target type). Until an org admin adds -# that permission and re-approves the installation, every call gracefully falls -# back to the default GITHUB_TOKEN and posts as github-actions[bot]. -# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the -# original names from before the App was renamed (same App ID + key). -name: PR triage (issue-link gate) - -on: - pull_request_target: - types: [opened, edited, reopened, synchronize] - -permissions: - pull-requests: write # add/remove the needs-issue label - issues: write # create/update the nudge comment (PR comments ride the issues API) - -env: - LABEL: 'needs-issue' - MARKER: '' - # PRs created before this instant are grandfathered (nudge, no failing check). - GATE_SINCE: '2026-07-04T00:00:00Z' - -jobs: - gate: - # Public repo only — this file also lives in the private mirror, where PRs - # are internal work that never links public issues. Skip bot authors - # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING - # exempts docs/small fixes from the issue-first ask). - if: >- - github.repository == 'TestSprite/testsprite-cli' && - github.event.pull_request.state == 'open' && - github.event.pull_request.user.type != 'Bot' && - !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && - !startsWith(github.event.pull_request.title, 'docs') - runs-on: ubuntu-latest - steps: - # Mint an App token so actions post as testsprite-hob[bot]. Until the App + - # secrets + Pull-requests permission exist this no-ops / gets 403 and the - # script below falls back to the default token per-call. - - id: app-token - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} - - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 - env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - with: - # `github` client = default GITHUB_TOKEN: used for all reads (the App - # can't read PRs without the Pull-requests permission) and as the - # write fallback. App client (when mintable) is preferred for writes. - script: | - const { owner, repo } = context.repo; - const pr = context.payload.pull_request; - const label = process.env.LABEL; - const marker = process.env.MARKER; - const author = pr.user.login; - - const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) - : null; - // Prefer the App identity; fall back to github-actions[bot] when the - // App is absent or lacks the Pull-requests permission (403/404). - async function write(fn) { - if (appClient) { - try { return await fn(appClient.rest); } - catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } - } - return await fn(github.rest); - } - - // Linked issues: GitHub-computed closing references carry number + - // assignees in one query. - const gql = await github.graphql( - `query($owner:String!,$repo:String!,$num:Int!){ - repository(owner:$owner,name:$repo){ - pullRequest(number:$num){ - closingIssuesReferences(first:10){ - nodes{ number assignees(first:10){ nodes{ login } } } - } - } - } - }`, { owner, repo, num: pr.number }); - const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes - .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); - - // Body-text fallback for the brief window before GitHub computes the - // link (same-repo bare "#N" references only). - const bodyRefs = [...(pr.body || '') - .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] - .map(m => Number(m[3])) - .filter(n => !linkedIssues.some(i => i.number === n)) - .slice(0, 5); - for (const num of bodyRefs) { - try { - const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); - if (data.pull_request) continue; // "#N" pointed at a PR, not an issue - linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); - } catch (e) { /* unknown number — ignore */ } - } - - const linked = linkedIssues.length > 0; - const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); - const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); - - const hasLabel = (pr.labels || []).some(l => l.name === label); - const comments = await github.paginate(github.rest.issues.listComments, - { owner, repo, issue_number: pr.number, per_page: 100 }); - const nudge = comments.find(c => (c.body || '').includes(marker)); - const stateLine = ``; - - const contributingUrl = - `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; - const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; - - let body; - if (state === 'ok') { - body = `${marker}\n${stateLine}\n` - + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` - + `The \`${label}\` label has been removed.`; - } else if (state === 'unassigned') { - const list = linkedIssues.map(i => { - const holders = i.assignees.filter(a => a !== author); - return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); - }).join(', '); - body = `${marker}\n${stateLine}\n` - + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` - + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` - + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` - + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` - + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; - } else { - body = `${marker}\n${stateLine}\n` - + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` - + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` - + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` - + `so it is not review-ready. ${rerunHint} ` - + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; - } - - // Sticky comment: create once, update when the state changes. - if (!nudge) { - if (state !== 'ok') { - await write(rest => rest.issues.createComment( - { owner, repo, issue_number: pr.number, body })); - } - } else if (!nudge.body.includes(stateLine)) { - await write(rest => rest.issues.updateComment( - { owner, repo, comment_id: nudge.id, body })); - } - - // Label tracks the gate state. - if (state === 'ok' && hasLabel) { - await write(rest => rest.issues.removeLabel( - { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); - } - if (state !== 'ok' && !hasLabel) { - await write(rest => rest.issues.addLabels( - { owner, repo, issue_number: pr.number, labels: [label] })); - } - - // Hard gate for PRs opened after the cutoff; older PRs are nudged only. - const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); - if (state !== 'ok' && gated) { - core.setFailed(state === 'unlinked' - ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' - : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); - } else if (state !== 'ok') { - core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); - } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index fbaa607..0000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,64 +0,0 @@ -# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). -# -# Generous windows for an open-source repo (first-response SLA is 5 business days, -# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single -# reply resets the clock. Newcomer- and security-relevant work is exempt from -# closing. `good first issue` / `help wanted` stay open for whoever picks them up. -# -# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike -# the event-gated triage bot it is fenced to the PUBLIC repo with a repository -# guard — on private atlas it is a no-op (no nagging internal issues/PRs). -name: Stale - -on: - schedule: - - cron: '30 1 * * *' # daily 01:30 UTC - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - stale: - if: ${{ github.repository == 'TestSprite/testsprite-cli' }} - runs-on: ubuntu-latest - permissions: - issues: write # comment + (un)label + close stale issues - pull-requests: write # comment + (un)label + close stale PRs - steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 - with: - # ── issues ────────────────────────────────────────────────────── - days-before-issue-stale: 60 - days-before-issue-close: 14 - stale-issue-label: stale - stale-issue-message: > - This issue has had no activity for 60 days, so it's been marked - `stale`. If it's still relevant, just leave a comment (or remove the - `stale` label) and we'll keep it open — otherwise it will be closed - in 14 days. Thanks for helping us keep the tracker tidy! - close-issue-message: > - Closing as `stale` after no activity. This isn't a judgement on the - idea — please reopen or open a fresh issue if it's still relevant. - - # ── pull requests (more grace — external contributors may be slow) ─ - days-before-pr-stale: 45 - days-before-pr-close: 21 - stale-pr-label: stale - stale-pr-message: > - This PR has had no activity for 45 days, so it's been marked `stale`. - Push a commit or leave a comment to keep it open — otherwise it will - be closed in 21 days. We'd still love to merge it; ping a maintainer - if you're blocked on a review. - close-pr-message: > - Closing as `stale` after no activity. Reopen any time you can pick it - back up — your work isn't lost. - - # ── shared behaviour ──────────────────────────────────────────── - exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' - exempt-pr-labels: 'pinned,security,in-progress,hackathon' - exempt-draft-pr: true - remove-stale-when-updated: true - ascending: true # oldest first — fairest under the per-run op cap - operations-per-run: 60 - enable-statistics: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bac6f2..4f1a0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,48 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] -## [0.3.0] - 2026-07-08 - ### Added -- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. -- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. -- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. -- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. -- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. -- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. -- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. -- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. -- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. -- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. -- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. -- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. -- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. - -### Changed - -- **Node.js 20 is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. -- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). -- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. -- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. -- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). -- The `usage` command no longer implies backend test runs are free. -- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. - -### Fixed - -- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). -- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. -- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. -- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. -- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. -- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. -- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. - -### Security - -- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. -- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. +- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. ## [0.2.0] - 2026-06-29 @@ -187,9 +149,6 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD -[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 -[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 -[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e016c2..2110f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,53 +20,14 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Docs and small fixes** (typos, doc corrections, comment-only changes): - just open a pull request. No issue required (linking one is still - appreciated). -- **Features and behavior changes** (new commands or flags, changed output, - new dependencies, refactors) follow **issue-first**: - 1. Find an existing issue, or open a new one describing the change. - 2. Claim it by commenting `/assign` on the issue — the literal slash - command on its own line. The triage bot assigns you automatically; - free-text requests ("can I take this?") are **not** detected. - 3. If the issue is new, wait for triage — we check proposals against - [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) - below before any code is written, so you don't invest in something - we'd ask you to rework or decline. - 4. Open your PR with a closing link (e.g. `Closes #123`) in the - description. -- **PR gate:** a bot checks every non-docs community PR for a closing-linked - issue that is **assigned to the PR author**. PRs that don't meet this get - the `needs-issue` label and a failing `PR triage` check, and **are not - reviewed** until it's fixed — file or claim the issue, add the closing - link, then edit the PR description (or push a commit) to re-run the check. -- Suspected **security vulnerabilities** are the exception to "file an - issue": report them privately per [SECURITY.md](./SECURITY.md) instead. +- **Small fixes and improvements:** just open a pull request. No issue required. +- **Large or breaking changes** (new commands, changed flags/output, new + dependencies, refactors): **open an issue first** to discuss the design. This + avoids wasted work on something we'd ask you to rework or that's out of scope. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. -### Standing scope policies - -Pre-decided policies, so proposals don't have to relitigate them: - -- **Runtime dependencies are budgeted.** The CLI ships with a deliberately - tiny runtime dependency set (`commander`, `valibot`, plus `undici` — - approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new - runtime dependency needs explicit maintainer sign-off **in the issue, - before the PR**. Utility modules land only together with the consumer - that uses them — standalone libraries are declined. -- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, - `cline`, `codex`. Accepted and in progress: `windsurf`, `gemini`, `kiro`. - A proposal for a new target needs (1) the editor's official rules/skill - file mechanism, documented, and (2) the proposer prepared to maintain the - target going forward. -- **Outbound network calls.** The CLI talks only to the configured - TestSprite API endpoint. The one approved exception is an opt-out-able - npm registry version check (at most once per 24h, carrying nothing but - the package name, fully silenced by its env opt-out and in CI/JSON/dry-run - modes). Any other outbound call is out of scope. - We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6686d52..6b798ef 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,7 +15,6 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) - - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -118,15 +117,10 @@ testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.m testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md testsprite agent list # list all 8 targets with status + mode + path -testsprite agent status # check installed skills against this CLI version ``` Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). -Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). - -`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. - The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. @@ -226,15 +220,6 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` -#### `testsprite test diff ` - -Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. - -```bash -testsprite test diff run_aaaa run_bbbb --output json -testsprite test diff run_aaaa run_bbbb --dry-run --output json -``` - #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -268,28 +253,7 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope (project commands require `write:projects`). - -#### `testsprite test scaffold` - -Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. - -```bash -testsprite test scaffold > first-test.plan.json -testsprite test scaffold --type backend --out tests/health.py -testsprite test scaffold --out plan.json --force # overwrite an existing file -``` - -#### `testsprite test lint` - -Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. - -```bash -testsprite test lint --plan-from ./checkout.plan.json -testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported -testsprite test lint --plans ./plans.jsonl # one plan spec per line -testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests -``` +Require the `write:tests` scope. #### `testsprite test create` @@ -328,7 +292,7 @@ testsprite test update test_xxxxxxxx --dry-run --output json #### `testsprite test delete ` / `test delete-batch` -Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -358,66 +322,20 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` -#### `testsprite project credential ` - -Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. - -```bash -testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt -testsprite project credential proj_xxxxxxxx --type public -testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json -``` - -`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. - -#### `testsprite project auto-auth ` - -Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. - -```bash -# Password login: POST the login endpoint, extract the token, inject as a Bearer header -testsprite project auto-auth proj_xxxxxxxx \ - --method password --inject bearer \ - --login-url https://api.example.com/login --login-method POST \ - --login-content-type application/json \ - --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ - --username ci@example.com --password-file ./pw.txt \ - --token-path '$.data.accessToken' - -# OAuth refresh-token flow -testsprite project auto-auth proj_xxxxxxxx \ - --method refresh_token --inject header --inject-key X-Auth-Token \ - --token-endpoint https://auth.example.com/oauth/token \ - --client-id my-client --client-secret-file ./secret.txt \ - --refresh-token-file ./refresh.txt --scope api.read - -# AWS Cognito refresh -testsprite project auto-auth proj_xxxxxxxx \ - --method aws_cognito_refresh --inject bearer \ - --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 - -# Turn it off (stored config is kept) -testsprite project auto-auth proj_xxxxxxxx --disable -``` - -Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. - ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. - -`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). ```bash # Trigger and return immediately @@ -430,7 +348,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json -# Batch run with JUnit XML for CI (sidecar; --output json unchanged) +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --output json @@ -509,17 +427,16 @@ Flags: `--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. -#### `testsprite test wait ` +#### `testsprite test wait ` -Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. +Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json -testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. #### `testsprite test artifact get ` @@ -534,30 +451,6 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). -### Account & diagnostics - -#### `testsprite usage` (alias: `testsprite credits`) - -Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. - -```bash -testsprite usage --output json -testsprite credits -testsprite usage --dry-run --output json -``` - -#### `testsprite doctor` - -One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: - -```bash -testsprite doctor -testsprite doctor --output json -testsprite doctor && testsprite test run test_xxxxxxxx --wait -``` - -Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. - ## Configuration ### Profiles & credentials @@ -580,17 +473,14 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ----------------------------------------- | ---------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | -| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | -| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | -| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | ### Update notice @@ -607,14 +497,13 @@ only outbound call the CLI makes besides your configured API endpoint. API-key scopes gate the write and run surfaces: -| Scope | Required by | -| ---------------- | -------------------------------------------------------------------- | -| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `write:projects` | `project create / update / credential / auto-auth` | -| `run:tests` | `test run / rerun / flaky / wait / artifact get` | +| Scope | Required by | +| --------------- | -------------------------------------------------------------------- | +| `read:me` | `auth whoami` | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `run:tests` | `test run / rerun / wait / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -632,25 +521,20 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| --------------------- | --------------------------------------------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | -| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | - -### Signals & pipes - -On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. +| Code | Meaning | +| ---- | --------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | ## Design principles diff --git a/README.md b/README.md index 78d293a..404f237 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,6 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. -> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). - From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -91,34 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | -| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | -| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | -| | `test wait` | Block on one or more `runId`s until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package.json b/package.json index e78ce17..341e713 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.2.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index df04c18..cc9342c 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,27 +91,6 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. -**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This -covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, -`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block -(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the -project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for -the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; -basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a -literal `Bearer …` / `sk-…` / key value into the script: - -```python -r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) -``` - -Configure the credential once on the project (ask the user for the value — never invent it or -reuse a key you happened to see): a static credential with -`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, -or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat -runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded -key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it -as a must-fix. - **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index fe7c661..650a88b 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -165,39 +165,6 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. -**Authentication — read the injected credential, NEVER hardcode any credential.** This -applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth -blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any -other credential into the test. Before your script runs, TestSprite prepends a managed -credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` -already contains the right header(s) for the configured auth type: - -```python -# Auto-injected credentials — do not modify -__AUTH_CREDENTIAL__ = "..." -__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" -__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} -``` - -Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type -the project is configured for, so the same line works for Bearer, API-key, or basic auth: - -```python -r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) -``` - -Configure the credential **once on the project** (ask the user for the value — never invent -or reuse a key you happened to see), and the block stays correct + refreshable: - -- **Bearer / API key / basic (static):** - `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` -- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` - -A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so -the test breaks on later runs; the managed block is rewritten with a fresh value each run. -`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as -a must-fix, not a nuisance. - **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 24d8ec6..4f5fdd3 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -661,39 +661,29 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target defaults to claude and installs the skill file', async () => { - const { store, fs: agentFs } = makeMemFs(); - const { capture, deps } = makeCapture(); - - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - - const claudeAbs = path.resolve(CWD, TARGETS.claude.path); - expect(store.has(claudeAbs)).toBe(true); - expect(capture.stderr.join('\n')).toContain('defaulting to claude'); - }); - - it('non-TTY default writes the canonical claude content', async () => { - const { store, fs: agentFs } = makeMemFs(); + it('non-TTY with no target throws exit 5', async () => { + const { fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - await runInstall( - { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + } catch (err) { + thrown = err; + } - const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); - const abs = path.resolve(CWD, relPath); - expect(store.get(abs)).toBe(content); + expect(thrown).toBeInstanceOf(ApiError); + expect((thrown as ApiError).exitCode).toBe(5); }); it('TTY with injected prompt returning "claude" installs claude', async () => { diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 7d7e7c8..0a40c4b 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -371,19 +371,18 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - stderrFn( - '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', + throw localValidationError( + 'target', + `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, ); - resolvedTargetStrings = ['claude']; - } else { - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 617fd6f..75fdff0 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -199,14 +199,8 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); - // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite project create --type frontend'); - expect(stdout).toContain('testsprite test run --all --project '); - expect(stdout).toContain('the testsprite-onboard skill is installed'); - // No "current project" wording, no bare test list. - expect(stdout).not.toContain('current project'); - expect(stdout).not.toContain('testsprite test list'); + expect(stdout).toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -312,15 +306,6 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); - // --no-agent points at manual test creation; must not claim the skill is installed. - expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite project create --type frontend'); - expect(stdout).toContain('testsprite test create --project '); - expect(stdout).toContain('testsprite test run --all --project '); - expect(stdout).not.toContain('skill is installed'); - // No "current project" wording, no bare test list. - expect(stdout).not.toContain('current project'); - expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { diff --git a/src/commands/init.ts b/src/commands/init.ts index c2d5678..aff0644 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -407,33 +407,12 @@ function renderInitText(data: unknown): string { } lines.push(''); lines.push('Next steps:'); - lines.push(' # 1. Create your first project (frontend example) — prints a projectId'); - lines.push( - ' testsprite project create --type frontend --name "My App" --url https://your-app.com', - ); - lines.push(''); + lines.push(' testsprite test list # list tests in the current project'); + lines.push(' testsprite agent list # check installed agent targets'); if (s.agent) { lines.push( - ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', - ); - lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); - lines.push(' testsprite test run --all --project '); - lines.push(''); - lines.push(' # Manage installed agent skills'); - lines.push(' testsprite agent list'); - lines.push( - ' testsprite agent install --target= # re-install or install additional targets', - ); - } else { - lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); - lines.push(' testsprite test create --project ...'); - lines.push(' testsprite test run --all --project '); - lines.push( - ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', + ' testsprite agent install --target= # re-install or install additional targets', ); - lines.push(''); - lines.push(' # Manage installed agent skills'); - lines.push(' testsprite agent list'); } return lines.join('\n'); diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index daea851..f63928c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -8,9 +8,7 @@ import { type CliProject, type CliUpdateProjectResponse, createProjectCommand, - runAutoAuth, runCreate, - runCredential, runGet, runList, runUpdate, @@ -74,10 +72,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create, update, credential and auto-auth subcommands', () => { + it('exposes list, get, create and update subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); + expect(names).toEqual(['create', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -329,21 +327,6 @@ describe('runList', () => { }); }); -describe('DEV-244 — project update no longer accepts the dead --description flag', () => { - it('rejects --description on `project update` as an unknown option', async () => { - const project = createProjectCommand(); - const update = project.commands.find(c => c.name() === 'update')!; - project.exitOverride(); - update.exitOverride(); - - await expect( - project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { - from: 'user', - }), - ).rejects.toThrow(/unknown option.*--description/i); - }); -}); - describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -881,6 +864,7 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', + description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -945,249 +929,3 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); - -describe('runCredential', () => { - interface Captured { - url: string; - method: string; - body: unknown; - headers: Headers; - } - function captureFetch(captured: Captured[], body: unknown) { - return makeFetch((url, init) => { - captured.push({ - url, - method: init.method ?? 'GET', - body: init.body ? JSON.parse(init.body as string) : undefined, - headers: new Headers(init.headers as Record), - }); - return { status: 200, body }; - }); - } - - it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured, { - projectId: 'p1', - authType: 'Bearer token', - rewroteCount: 2, - }); - const res = await runCredential( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - authType: 'Bearer token', - credential: 'tok-123', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(res.rewroteCount).toBe(2); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.url).toContain('/projects/p1/credential'); - expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); - expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); - }); - - it('public clears the credential (no credential in body, none required)', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured, { - projectId: 'p1', - authType: 'public', - rewroteCount: 0, - }); - await runCredential( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.body).toEqual({ authType: 'public' }); - }); - - it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runCredential( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); - - it('rejects an unknown --type locally (no fetch)', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runCredential( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - authType: 'jwt', - credential: 'x', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); -}); - -describe('runAutoAuth', () => { - interface Captured { - url: string; - method: string; - body: Record; - headers: Headers; - } - function captureFetch(captured: Captured[]) { - return makeFetch((url, init) => { - captured.push({ - url, - method: init.method ?? 'GET', - body: init.body ? JSON.parse(init.body as string) : {}, - headers: new Headers(init.headers as Record), - }); - return { - status: 200, - body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, - }; - }); - } - - it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'aws_cognito_refresh', - inject: 'bearer', - region: 'us-east-1', - clientId: 'abc', - refreshToken: 'rt-xyz', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - const put = captured.find(c => c.method === 'PUT')!; - expect(put.url).toContain('/projects/p1/auto-auth'); - expect(put.body).toEqual({ - enabled: true, - method: 'aws_cognito_refresh', - inject: 'bearer', - region: 'us-east-1', - clientId: 'abc', - refreshToken: 'rt-xyz', - }); - expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); - }); - - it('--disable sends enabled:false', async () => { - const { credentialsPath } = makeCreds(); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - disable: true, - method: 'password', - inject: 'bearer', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); - }); - - it('reads a secret from --refresh-token-file', async () => { - const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); - const rtFile = join(dir, 'rt.txt'); - writeFileSync(rtFile, ' rt-from-file\n'); - const captured: Captured[] = []; - const fetchImpl = captureFetch(captured); - await runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'refresh_token', - inject: 'bearer', - tokenEndpoint: 'https://idp.example.com/token', - refreshTokenFile: rtFile, - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); - }); - - it('rejects an unknown --method / --inject locally (no fetch)', async () => { - const { credentialsPath } = makeCreds(); - let fetched = false; - const fetchImpl = makeFetch(() => { - fetched = true; - return { body: {} }; - }); - await expect( - runAutoAuth( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'p1', - method: 'magic', - inject: 'bearer', - }, - { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - expect(fetched).toBe(false); - }); -}); - -describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { - const noNetwork = () => { - throw new Error('network should not be hit'); - }; - - it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { - const { credentialsPath } = makeCreds(); - await expect( - runCreate( - { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, - { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { - const { credentialsPath } = makeCreds(); - await expect( - runUpdate( - { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, - { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); -}); diff --git a/src/commands/project.ts b/src/commands/project.ts index 96f2fc8..ef4d5e8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -142,16 +142,20 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); - // P1-3: client-side length checks matching server limits. - // Whitespace-only / empty rejection (parity with `test create`'s requireString; - // a truthy `--name " "` otherwise creates a blank-named project on the backend). - if (opts.name === undefined || opts.name.trim().length === 0) { - throw localValidationError('--name is required and must not be empty or whitespace-only'); + // Reject empty / whitespace-only names so a junk record never reaches the + // backend — matches the `requireString` whitespace guard `test create` uses + // (dogfood P1 fix #1). Without this, `--name " "` passes the action + // handler's `if (!name)` check (a non-empty string is truthy) and is sent + // verbatim, creating a blank-named project. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); } if (opts.password !== undefined && opts.password.trim().length === 0) { throw localValidationError('--password must not be empty or whitespace-only'); } - if (opts.name.length > 200) { + + // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } if (opts.description !== undefined && opts.description.length > 2000) { @@ -244,6 +248,7 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; + description?: string; instruction?: string; idempotencyKey?: string; } @@ -259,8 +264,6 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. - // Reject a whitespace-only `--name` on update too (parity with create); name - // stays optional here, so only validate when the flag is supplied. if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } @@ -270,6 +273,10 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } + if (opts.description !== undefined && opts.description.length > 2000) { + throw localValidationError('--description must be at most 2000 characters'); + } + // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); @@ -281,6 +288,7 @@ export async function runUpdate( targetUrl: opts.targetUrl !== undefined, username: opts.username !== undefined, password: passwordSupplied, + description: opts.description !== undefined, instruction: opts.instruction !== undefined, }; const presentFieldNames = Object.entries(mutableFields) @@ -288,7 +296,7 @@ export async function runUpdate( .map(([field]) => field); if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', ); } @@ -328,6 +336,7 @@ export async function runUpdate( targetUrl: opts.targetUrl, username: opts.username, password, + description: opts.description, instruction: opts.instruction, }; const body = Object.fromEntries( @@ -346,228 +355,6 @@ export async function runUpdate( return updated; } -// --------------------------------------------------------------------------- -// project credential — set the static backend credential -// --------------------------------------------------------------------------- - -const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; - -export interface CliProjectCredentialResponse { - projectId: string; - authType: string; - rewroteCount: number; -} - -interface CredentialOptions extends CommonOptions { - projectId: string; - authType: string; - credential?: string; - credentialFile?: string; - idempotencyKey?: string; -} - -export async function runCredential( - opts: CredentialOptions, - deps: ProjectDeps = {}, -): Promise { - const out = makeOutput(opts.output, deps); - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - assertIdempotencyKey(opts.idempotencyKey); - - if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { - throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); - } - - // Resolve the credential value (flag or file). Required for every type - // except `public` (which clears it). - let credential = opts.credential; - if (credential === undefined && opts.credentialFile !== undefined) { - credential = readFileSync(opts.credentialFile, 'utf8').trim(); - } - if (opts.authType !== 'public' && (credential === undefined || credential === '')) { - throw localValidationError( - '--credential (or --credential-file) is required unless --type is "public"', - ); - } - - const body: Record = { authType: opts.authType }; - if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; - - const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; - if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { - stderr(`idempotency-key: ${idempotencyKey}`); - } - - if (opts.dryRun) { - const sample: CliProjectCredentialResponse = { - projectId: opts.projectId, - authType: opts.authType, - rewroteCount: 0, - }; - out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); - return sample; - } - - const client = makeClient(opts, deps); - const res = await client.put( - `/projects/${encodeURIComponent(opts.projectId)}/credential`, - { body, headers: { 'idempotency-key': idempotencyKey } }, - ); - out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); - return res; -} - -function renderCredentialText(r: CliProjectCredentialResponse): string { - return [ - `projectId ${r.projectId}`, - `authType ${r.authType}`, - `rewroteCount ${r.rewroteCount}`, - ].join('\n'); -} - -// --------------------------------------------------------------------------- -// project auto-auth — configure the recurring-token (auto-refresh) login -// --------------------------------------------------------------------------- - -const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; -const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; - -export interface CliProjectAutoAuthResponse { - projectId: string; - enabled: boolean; - method: string; - inject: string; - /** - * Present when the server's trial refresh failed: `enabled` is then `false` - * and this carries the reason (e.g. a bad refresh token). The config is still - * stored, but auto-auth won't run until the login succeeds. - */ - lastRefreshError?: string; -} - -interface AutoAuthOptions extends CommonOptions { - projectId: string; - disable?: boolean; - method: string; - inject: string; - injectKey?: string; - // password method - loginUrl?: string; - loginMethod?: string; - loginContentType?: string; - loginBodyTemplate?: string; - username?: string; - password?: string; - passwordFile?: string; - tokenPath?: string; - // refresh_token method - tokenEndpoint?: string; - clientId?: string; - clientSecret?: string; - clientSecretFile?: string; - refreshToken?: string; - refreshTokenFile?: string; - scope?: string; - // aws_cognito_refresh method - region?: string; - idempotencyKey?: string; -} - -export async function runAutoAuth( - opts: AutoAuthOptions, - deps: ProjectDeps = {}, -): Promise { - const out = makeOutput(opts.output, deps); - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - assertIdempotencyKey(opts.idempotencyKey); - - if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { - throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); - } - if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { - throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); - } - - // Resolve secrets from --*-file variants so they stay out of shell history. - const password = - opts.password ?? - (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); - const clientSecret = - opts.clientSecret ?? - (opts.clientSecretFile !== undefined - ? readFileSync(opts.clientSecretFile, 'utf8').trim() - : undefined); - const refreshToken = - opts.refreshToken ?? - (opts.refreshTokenFile !== undefined - ? readFileSync(opts.refreshTokenFile, 'utf8').trim() - : undefined); - - const enabled = opts.disable !== true; - const body: Record = { enabled, method: opts.method, inject: opts.inject }; - const maybe = (k: string, v: string | undefined): void => { - if (v !== undefined) body[k] = v; - }; - maybe('injectKey', opts.injectKey); - maybe('loginUrl', opts.loginUrl); - maybe('loginMethod', opts.loginMethod); - maybe('loginContentType', opts.loginContentType); - maybe('loginBodyTemplate', opts.loginBodyTemplate); - maybe('username', opts.username); - maybe('password', password); - maybe('tokenPath', opts.tokenPath); - maybe('tokenEndpoint', opts.tokenEndpoint); - maybe('clientId', opts.clientId); - maybe('clientSecret', clientSecret); - maybe('refreshToken', refreshToken); - maybe('scope', opts.scope); - maybe('region', opts.region); - - const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; - if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { - stderr(`idempotency-key: ${idempotencyKey}`); - } - - if (opts.dryRun) { - const sample: CliProjectAutoAuthResponse = { - projectId: opts.projectId, - enabled, - method: opts.method, - inject: opts.inject, - }; - out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); - return sample; - } - - const client = makeClient(opts, deps); - const res = await client.put( - `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, - { body, headers: { 'idempotency-key': idempotencyKey } }, - ); - out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); - return res; -} - -function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { - const lines = [ - `projectId ${r.projectId}`, - `enabled ${r.enabled}`, - `method ${r.method}`, - `inject ${r.inject}`, - ]; - if (r.lastRefreshError) { - lines.push(`lastRefreshError ${r.lastRefreshError}`); - } - // A disabled result after a write means the trial login failed — call it out - // so the user doesn't assume auto-auth is live. - if (!r.enabled) { - lines.push( - 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', - ); - } - return lines.join('\n'); -} - export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -661,6 +448,7 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') + .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -677,6 +465,7 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, + description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -684,99 +473,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); - project - .command('credential ') - .description( - 'Set the static backend credential injected into every backend test\n' + - '(Bearer token / API key / Basic token / public). Free tier.', - ) - .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') - .option('--credential ', 'credential value (required unless --type public)') - .option('--credential-file ', 'read the credential value from a file') - .option( - '--idempotency-key ', - 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { - await runCredential( - { - ...resolveCommonOptions(command), - projectId, - authType: cmdOpts.type, - credential: cmdOpts.credential, - credentialFile: cmdOpts.credentialFile, - idempotencyKey: cmdOpts.idempotencyKey, - }, - deps, - ); - }); - - project - .command('auto-auth ') - .description( - 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + - 'A fresh token is fetched on each run and injected into every backend test.', - ) - .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') - .requiredOption('--inject ', 'bearer | header | cookie') - .option('--disable', 'turn auto-auth off (keeps stored config)') - .option('--inject-key ', 'header/cookie name when --inject is header/cookie') - // password method - .option('--login-url ', 'login endpoint (method=password)') - .option('--login-method ', 'POST | PUT (method=password)') - .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') - .option('--login-body-template ', 'login body template with {{username}}/{{password}}') - .option('--username ', 'login username (method=password)') - .option('--password ', 'login password (method=password)') - .option('--password-file ', 'read login password from a file') - .option('--token-path ', 'JSONPath to the token in the login response') - // refresh_token method - .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') - .option('--client-id ', 'OAuth client id') - .option('--client-secret ', 'OAuth client secret') - .option('--client-secret-file ', 'read OAuth client secret from a file') - .option('--refresh-token ', 'OAuth/Cognito refresh token') - .option('--refresh-token-file ', 'read the refresh token from a file') - .option('--scope ', 'OAuth scope') - // aws_cognito_refresh method - .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") - .option( - '--idempotency-key ', - 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', - ) - .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { - await runAutoAuth( - { - ...resolveCommonOptions(command), - projectId, - disable: cmdOpts.disable, - method: cmdOpts.method, - inject: cmdOpts.inject, - injectKey: cmdOpts.injectKey, - loginUrl: cmdOpts.loginUrl, - loginMethod: cmdOpts.loginMethod, - loginContentType: cmdOpts.loginContentType, - loginBodyTemplate: cmdOpts.loginBodyTemplate, - username: cmdOpts.username, - password: cmdOpts.password, - passwordFile: cmdOpts.passwordFile, - tokenPath: cmdOpts.tokenPath, - tokenEndpoint: cmdOpts.tokenEndpoint, - clientId: cmdOpts.clientId, - clientSecret: cmdOpts.clientSecret, - clientSecretFile: cmdOpts.clientSecretFile, - refreshToken: cmdOpts.refreshToken, - refreshTokenFile: cmdOpts.refreshTokenFile, - scope: cmdOpts.scope, - region: cmdOpts.region, - idempotencyKey: cmdOpts.idempotencyKey, - }, - deps, - ); - }); - return project; } @@ -804,41 +500,11 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; + description?: string; instruction?: string; idempotencyKey?: string; } -interface CredentialFlagOpts { - type: string; - credential?: string; - credentialFile?: string; - idempotencyKey?: string; -} - -interface AutoAuthFlagOpts { - disable?: boolean; - method: string; - inject: string; - injectKey?: string; - loginUrl?: string; - loginMethod?: string; - loginContentType?: string; - loginBodyTemplate?: string; - username?: string; - password?: string; - passwordFile?: string; - tokenPath?: string; - tokenEndpoint?: string; - clientId?: string; - clientSecret?: string; - clientSecretFile?: string; - refreshToken?: string; - refreshTokenFile?: string; - scope?: string; - region?: string; - idempotencyKey?: string; -} - function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index f7c7602..6ae831c 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -4631,48 +4631,6 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); - it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { - const { credentialsPath } = makeCreds(); - const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); - const fetchImpl = makeFetch((url, init) => { - const method = init.method ?? 'GET'; - if (method === 'GET') return { status: 200, body: { items: [] } }; - return { - status: 200, - body: { - ...SAMPLE_RESPONSE, - type: 'backend', - warnings: [ - 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', - ], - }, - }; - }); - const out: string[] = []; - const err: string[] = []; - await runCreate( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_alice', - type: 'backend', - name: 'hardcoded be', - codeFile, - }, - { - credentialsPath, - fetchImpl, - stdout: line => out.push(line), - stderr: line => err.push(line), - }, - ); - // Warning lands on stderr, prefixed `[warn]`. - expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); - // stdout stays the parseable wire object — no warning noise. - expect(out.join('\n')).not.toContain('[warn]'); - }); - it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5716,36 +5674,6 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); - it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { - const { credentialsPath } = makeCreds(); - const codeFile = writeCodeFile('// fe code'); - const err = (await runCreate( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_fe', - type: 'frontend', - name: 'fe test', - codeFile, - produces: ['some_var'], - }, - { - credentialsPath, - fetchImpl: () => Promise.resolve(new Response('{}')), - stdout: () => undefined, - stderr: () => undefined, - }, - ).catch((e: unknown) => e)) as ApiError; - expect(err).toBeInstanceOf(ApiError); - // details.field is the BARE flag name (was '--produces' → double-dashed subject). - expect(err.details).toMatchObject({ field: 'produces' }); - expect(err.nextAction).toContain('--produces'); - expect(err.nextAction).not.toContain('----'); // regression: '----produces' - expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag - expect(err.nextAction).not.toContain('backend..'); // regression: double period - }); - it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 0602afc..1c6edd8 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -543,12 +543,6 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; - /** - * Non-fatal advisories from the backend (e.g. the BE auth guardrail - * flagging a hardcoded credential). Rendered on stderr; the create - * still succeeded. - */ - warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -758,19 +752,14 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); - if (opts.category !== undefined) depFlags.push('category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); + if (opts.category !== undefined) depFlags.push('--category'); if (depFlags.length > 0) { - // Pass the BARE flag name to localValidationError — its kind:'flag' branch - // adds the `--` prefix, so '--produces' would render as '----produces'. - const flagList = depFlags.map(f => `--${f}`); - const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; - // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + - `Remove ${flagList.join('/')} or use --type backend`, + `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + + `Remove ${depFlags.join('/')} or use --type backend.`, ); } } @@ -847,11 +836,6 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); - // Surface backend advisories (e.g. a hardcoded-credential warning for BE - // tests) on stderr so they reach the agent without polluting stdout JSON. - // Emitted before the --run early return so they always show. - emitResponseWarnings(response.warnings, deps); - // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -1014,16 +998,6 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } -/** - * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), - * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. - */ -function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { - if (!warnings || warnings.length === 0) return; - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - for (const w of warnings) stderrFn(`[warn] ${w}`); -} - /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -3426,12 +3400,6 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; - /** - * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded - * credential in the replaced code). Rendered on stderr; the update still - * succeeded. - */ - warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3609,7 +3577,6 @@ export async function runCodePut( }, }, ); - emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -5836,7 +5803,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all tests in. */ + /** projectId to run all BE tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5973,7 +5940,7 @@ export async function runTestRunAll( stderrFn(`idempotency-key: ${idempotencyKey}`); } - // Resolve testIds: fetch all tests in the project, apply --filter. + // Resolve testIds: fetch all BE tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -6011,8 +5978,7 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL tests in the project - // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). + // When no --filter, omit testIds → server runs ALL BE tests in the project. const batchResp = await client.triggerBatchRunFresh( { @@ -8403,7 +8369,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -8431,7 +8397,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -8458,14 +8424,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all project tests in wave order\n' + + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).\n' + - '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + - 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + - "reported under skippedFrontend with an advisory; run those with 'test run '.", + '(see `testsprite test create --help` for details).', ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -8481,7 +8444,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all tests in a project', + 'provide a , or use --all --project to run all BE tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -8510,16 +8473,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: a BE test's base - // URL is baked into its code, and the unified engine resolves each - // project's configured environment server-side (per-run URL overrides - // are not applied to batch FE runs either). Silently dropping it could - // run the suite against an unintended environment in the caller's mind - // — reject loudly instead. + // --target-url has no effect on the --all batch path: it is BE-only + // (FE tests are skipped server-side) and a BE test's base URL is baked + // into its code. Silently dropping it could run the suite against an + // unintended environment in the caller's mind — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', + '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', ); } await runTestRunAll( @@ -10012,11 +9973,7 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option( - '--failed-only', - 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + - 'so this is usually a no-op; use `test steps ` for the full run trail.', - ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10046,11 +10003,7 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option( - '--failed-only', - 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + - 'so this is usually a no-op; use `test steps ` for the full run trail.', - ) + .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index 1980d26..a97dbba 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -154,12 +154,8 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); - // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). - // The API exposes no backend-specific per-run cost field, and it differs from - // the frontend rate, so state that it bills without asserting a possibly-wrong - // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: also consumes credits (exact amount not reported by the API)`, + `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, ); } diff --git a/src/index.ts b/src/index.ts index bf9a9aa..806f6e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', + 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', ) .option( '--request-timeout ', diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 2ac9158..51bf9b5 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -34,9 +34,7 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing - // \r on the description and break the byte-identical comparisons. - const lines = content.split(/\r?\n/); + const lines = content.split('\n'); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index ff20e8c..438092b 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -10,7 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { isAbsolute, join, resolve } from 'node:path'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -593,14 +593,18 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); - expect(isAbsolute(out)).toBe(true); + expect(out.endsWith('/tmp/x')).toBe(true); + expect(out.startsWith('/')).toBe(true); }); it('strips a trailing slash', () => { const out = resolveBundleDir('/tmp/x/'); expect(out).toBe('/tmp/x'); }); + + it('preserves the filesystem root path', () => { + expect(resolveBundleDir('/')).toBe('/'); + }); }); describe('streamUrlToFile retry', () => { diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index a3c0808..8d4eba0 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -325,7 +325,7 @@ export function resolveBundleDir(rawPath: string): string { }, }); } - const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + const trimmed = rawPath.endsWith('/') && rawPath.length > 1 ? rawPath.slice(0, -1) : rawPath; return isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed); } diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index d50ad52..896d057 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { homedir, tmpdir } from 'node:os'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { @@ -139,11 +139,8 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - // POSIX file modes don't exist on Windows (stat reports 0666). - if (process.platform !== 'win32') { - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); - } + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -191,8 +188,7 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. - it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { + it('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -203,7 +199,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); + expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); }); }); diff --git a/src/lib/http.ts b/src/lib/http.ts index e8d30a2..af1c25a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -339,7 +339,6 @@ export class HttpClient { * GET /api/cli/v1/tests/{testId}/runs * List a test's prior run history, newest-first. * - * Per the run-history read-endpoint spec §Route 1. * Limit-before-filter caveat: a `source`-filtered page may return fewer * than `pageSize` rows while still yielding a non-null `nextCursor`. * That means "none in THIS window", not end-of-history. @@ -638,7 +637,7 @@ export class HttpClient { accept: 'application/json', 'user-agent': `testsprite-cli/${VERSION}`, }; - // The CLI v1 facade authenticates via `x-api-key` per the CLI OpenAPI contract + // The CLI v1 facade authenticates via `x-api-key`. // (securitySchemes.ApiKeyAuth). Sending only Authorization Bearer would be // treated as a missing key by the backend. if (this.apiKey) headers['x-api-key'] = this.apiKey; @@ -810,7 +809,7 @@ export function parseRetryAfter(headerValue: string | null): number | undefined /** * HTTP statuses that arrive without our error envelope on transient * upstream outages (edge LB returning HTML, etc.). Treated as transport - * failures per the CLI error spec §7. + * failures. */ function isTransportEdgeStatus(status: number): boolean { return status === 408 || status === 502 || status === 504; diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 2b26c0e..c15b09d 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -13,30 +13,24 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- -// The implementation joins paths with the native separator; normalize so the -// fakes below match on Windows (backslashes) as well as POSIX. -const toPosix = (p: string) => p.replaceAll('\\', '/'); - describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => - toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => - toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); @@ -79,7 +73,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -204,6 +198,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); + expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/version.ts b/src/version.ts index f60fdfd..efa916d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.3.0'; +export const VERSION = '0.2.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 62efe5d..fcc3d85 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -133,29 +133,20 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - credential [options] Set the static backend credential injected - into every backend test - (Bearer token / API key / Basic token / - public). Free tier. - auto-auth [options] Configure the recurring-token - (auto-refresh login) for backend tests - (Pro). - A fresh token is fetched on each run and - injected into every backend test. - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + help [command] display help for command " `; @@ -251,7 +242,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all tests in a project (M4). + Use --all --project for a wave-ordered batch run of all BE tests (M4). Exit codes: 0 passed (or queued without --wait) @@ -371,9 +362,7 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Trim to the failed step ±1. The bundle is already - failure-focused server-side, so this is usually a no-op; use - \`test steps \` for the full run trail. + --failed-only Keep only the failed step plus its immediate neighbors (±1) -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -559,7 +548,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all tests in a project (M4). +Use --all --project for a wave-ordered batch run of all BE tests (M4). Exit codes: 0 passed (or queued without --wait) @@ -583,9 +572,9 @@ Options: 600) --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. - --all run all tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) --project project id (required with --all; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains @@ -600,17 +589,13 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all project tests in wave order + testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Frontend tests: the current unified engine runs FE tests too (they are billed -like any run). On the legacy backend-only engine FE tests cannot run — they are -reported under skippedFrontend with an advisory; run those with 'test run '. - Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " @@ -655,13 +640,10 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network and credentials; emit a canned - sample matching the OpenAPI contract. Useful for - learning the CLI surface without an API key. - Note: file inputs you pass - (--plan-from/--plans/--steps) are still read and - validated locally; only --code-file uses a - placeholder. + --dry-run Skip the network, credentials, and filesystem; + emit a canned sample matching the OpenAPI + contract. Useful for learning the CLI surface + without an API key. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 959f212..b0537b1 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; +import { existsSync, mkdtempSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; @@ -364,10 +364,7 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, - // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — - // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, - USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -900,10 +897,7 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - // POSIX file modes don't exist on Windows (stat reports 0666). - if (process.platform !== 'win32') { - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); - } + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -1050,7 +1044,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - rmSync(credPath, { force: true }); + if (existsSync(credPath)) execFileSync('rm', [credPath]); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts deleted file mode 100644 index c798bb3..0000000 --- a/test/helpers/hermetic-env.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). - * - * Two leaks this closes, both of which made results depend on the - * developer's machine: - * - * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` - * precedence over the credentials file, so a key exported in the - * developer's shell silently overrode test fixtures. - * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but - * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` - * recipe never isolated Windows runs. Both vars are redirected to a - * throwaway dir so no test can read or write `~/.testsprite`. - * - * Tests that need these vars set them explicitly (on `process.env` or via - * injected `env` deps) after this runs. - */ -import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; -import { homedir, tmpdir } from 'node:os'; -import { join } from 'node:path'; - -const realHome = homedir(); -const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); -if (process.platform === 'win32') { - // Node version shims (Volta) resolve LocalAppData under USERPROFILE and - // abort if it's missing, which would break the `npm run build` beforeAll - // in the subprocess/snapshot suites. - mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); -} -// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless -// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. -const realVoltaHome = join(realHome, '.volta'); -if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { - process.env.VOLTA_HOME = realVoltaHome; -} -process.env.HOME = hermeticHome; -process.env.USERPROFILE = hermeticHome; - -for (const key of Object.keys(process.env)) { - if (key.startsWith('TESTSPRITE_')) delete process.env[key]; -} diff --git a/vitest.config.ts b/vitest.config.ts index bd9b2ad..add8f0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,9 +4,6 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], - // Strip real TESTSPRITE_* env vars and redirect the home dir so results - // never depend on the developer's shell or ~/.testsprite (see the file). - setupFiles: ['./test/helpers/hermetic-env.ts'], // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). fileParallelism: false,