From 4b2e45beb704ec08d0a0d46b34409051d9319ec9 Mon Sep 17 00:00:00 2001 From: JerryNee Date: Mon, 6 Jul 2026 13:33:54 -0500 Subject: [PATCH 1/2] fix(dry-run): include failed run-scoped step sample --- src/commands/test.test.ts | 34 ++++++++++++++++++++++++++++ src/lib/dry-run/samples.test.ts | 37 ++++++++++++++++++++++++------- src/lib/dry-run/samples.ts | 39 ++++++++++++++++++++------------- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index c78c7a8..592657e 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -2667,6 +2667,40 @@ describe('runSteps', () => { expect(block.match(/error: /g)).toHaveLength(1); }); + it('--run-id dry-run sample maps the failed step error and failure contributor flag', async () => { + const out: string[] = []; + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_fe', + runId: 'run_dry', + }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: join(tmpdir(), 'testsprite-no-creds'), + stdout: line => out.push(line), + stderr: () => undefined, + }, + ); + + const failing = page.items.find(step => step.stepIndex === 3); + expect(failing).toMatchObject({ + status: 'failed', + error: expect.any(String), + stepType: 'assertion', + outcomeContributesToFailure: true, + }); + expect(page.items.find(step => step.stepIndex === 1)?.outcomeContributesToFailure).toBe(false); + + const printed = JSON.parse(out[0]!) as { items: Array<{ stepIndex: number; error?: string }> }; + expect(printed.items.some(step => step.stepIndex === 3 && typeof step.error === 'string')).toBe( + true, + ); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index a1838d6..2ff7578 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -407,20 +407,41 @@ describe('findSample', () => { expect(e).toBeUndefined(); }); - // defect-2 fix: getRun sample must return the passed shape (first-match-wins - // in findSample). Prior to fix, a duplicate failed-shape entry appeared - // before the passed-shape entry; `test wait --dry-run` always resolved to - // status: "failed", giving agents the wrong happy-path canned response. - it('GET /runs/{runId} resolves to the passed-shape getRun (not the failed shape)', () => { + it('GET /runs/{runId} sample includes a failed run-scoped step', () => { const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz'); expect(e?.operationId).toBe('getRun'); const body = e?.body() as { status: string; runId: string; - stepSummary: { failedCount: number }; + failedStepIndex: number | null; + failureKind: string | null; + error: string | null; + stepSummary: { total: number; completed: number; passedCount: number; failedCount: number }; + steps: Array<{ + stepIndex: string; + type: string; + status: string | null; + error: string | null; + }>; }; - expect(body.status).toBe('passed'); - expect(body.stepSummary.failedCount).toBe(0); + expect(body.status).toBe('failed'); + expect(body.failedStepIndex).toBe(3); + expect(body.failureKind).toBe('assertion'); + expect(body.error).toEqual(expect.any(String)); + expect(body.stepSummary).toMatchObject({ + total: 3, + completed: 3, + passedCount: 2, + failedCount: 1, + }); + + const failingStep = body.steps.find(step => step.stepIndex === '0003'); + expect(failingStep).toMatchObject({ + type: 'assertion', + status: 'failed', + error: expect.any(String), + }); + expect(failingStep?.error).not.toBe(''); }); it('getTest sample carries priority field (G1a)', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 4c7cbb0..9171610 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -691,17 +691,15 @@ const ENTRIES: DryRunSampleEntry[] = [ }, } satisfies BatchRerunResponse), // M3.3 piece-3 — GET /runs/{runId} (live status / long-poll). - // A terminal `passed` row is the most useful dry-run shape: agents see - // what a completed run looks like, and `--wait` terminates immediately. - // fix(2026-05-21): a duplicate failed-shape entry that appeared before - // this entry was removed; findSample first-match-wins was always - // returning status: "failed" for `test wait --dry-run`. + // Use a terminal failed row with a concrete failed step so + // `test steps --run-id --dry-run` demonstrates the run-scoped + // error + failedStepIndex mapping without needing live credentials. entry('getRun', 'GET', '/runs/{runId}', { runId: SAMPLE_RUN_ID, - testId: SAMPLE_TEST_ID_PASSED, + testId: SAMPLE_TEST_ID_FAILED, projectId: SAMPLE_PROJECT_ID, userId: SAMPLE_USER_ID, - status: 'passed', + status: 'failed', source: 'cli', createdAt: '2026-05-15T19:32:00.000Z', startedAt: '2026-05-15T19:32:05.000Z', @@ -709,19 +707,19 @@ const ENTRIES: DryRunSampleEntry[] = [ codeVersion: 'v1', targetUrl: SAMPLE_TARGET_URL, createdFrom: null, - failedStepIndex: null, - failureKind: null, - error: null, + failedStepIndex: 3, + failureKind: 'assertion', + error: 'Expected billing status badge to be visible, but it was not found.', videoUrl: null, stepSummary: { - total: 8, - completed: 8, - passedCount: 8, - failedCount: 0, + total: 3, + completed: 3, + passedCount: 2, + failedCount: 1, }, // Representative per-run steps so `test steps --run-id --dry-run` // demonstrates real output instead of an empty list (the generic - // `/runs/{runId}` sample is also used by `test wait`, which ignores steps). + // `/runs/{runId}` sample is also safe for wait flows, which ignore steps). steps: [ { stepIndex: '0001', @@ -745,6 +743,17 @@ const ENTRIES: DryRunSampleEntry[] = [ htmlSnapshotUrl: null, createdAt: '2026-05-15T19:32:20.000Z', }, + { + stepIndex: '0003', + type: 'assertion', + action: 'assert_visible', + status: 'failed', + description: 'Billing status badge is visible', + error: 'Expected billing status badge to be visible, but it was not found.', + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:30.000Z', + }, ], } satisfies RunResponse), ]; From 9179281e5ac54c386459a0b5c1bebc2195865ce5 Mon Sep 17 00:00:00 2001 From: JerryNee Date: Mon, 6 Jul 2026 16:11:58 -0500 Subject: [PATCH 2/2] fix(dry-run): use sentinel failed run sample --- src/commands/test.test.ts | 4 +- src/lib/dry-run/samples.test.ts | 19 +++- src/lib/dry-run/samples.ts | 193 +++++++++++++++++++++----------- 3 files changed, 147 insertions(+), 69 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 592657e..62949cc 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -2667,7 +2667,7 @@ describe('runSteps', () => { expect(block.match(/error: /g)).toHaveLength(1); }); - it('--run-id dry-run sample maps the failed step error and failure contributor flag', async () => { + it('--run-id run_failed_sample dry-run sample maps the failed step error and contributor flag', async () => { const out: string[] = []; const page = await runSteps( { @@ -2676,7 +2676,7 @@ describe('runSteps', () => { debug: false, dryRun: true, testId: 'test_fe', - runId: 'run_dry', + runId: 'run_failed_sample', }, { env: {} as NodeJS.ProcessEnv, diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index 2ff7578..413d2db 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -407,9 +407,25 @@ describe('findSample', () => { expect(e).toBeUndefined(); }); - it('GET /runs/{runId} sample includes a failed run-scoped step', () => { + // defect-2 fix: getRun sample must return the passed shape (first-match-wins + // in findSample). Prior to fix, a duplicate failed-shape entry appeared + // before the passed-shape entry; `test wait --dry-run` always resolved to + // status: "failed", giving agents the wrong happy-path canned response. + it('GET /runs/{runId} resolves to the passed-shape getRun (not the failed shape)', () => { const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz'); expect(e?.operationId).toBe('getRun'); + const body = e?.body() as { + status: string; + runId: string; + stepSummary: { failedCount: number }; + }; + expect(body.status).toBe('passed'); + expect(body.stepSummary.failedCount).toBe(0); + }); + + it('GET /runs/run_failed_sample resolves to the sentinel failed run-scoped step sample', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_failed_sample'); + expect(e?.operationId).toBe('getRun'); const body = e?.body() as { status: string; runId: string; @@ -424,6 +440,7 @@ describe('findSample', () => { error: string | null; }>; }; + expect(body.runId).toBe('run_failed_sample'); expect(body.status).toBe('failed'); expect(body.failedStepIndex).toBe(3); expect(body.failureKind).toBe('assertion'); diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 9171610..73bc4fe 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -47,6 +47,10 @@ const SAMPLE_TEST_ID_FAILED = 'test_8f2a4d10'; const SAMPLE_TEST_ID_PASSED = 'test_3a91bb02'; const SAMPLE_TEST_ID_BLOCKED = 'test_blocked_4f7a'; export const SAMPLE_RUN_ID = 'run_abc'; +// Documented sentinel for `test steps --run-id run_failed_sample --dry-run`: +// keeps wait flows on the default passed sample while still demonstrating a +// run-scoped failed step offline. +const SAMPLE_FAILED_RUN_ID = 'run_failed_sample'; // M3.4 rerun dry-run sample IDs const SAMPLE_RERUN_ID_BE_NAMED = 'run_rerun_be_named'; const SAMPLE_RERUN_ID_BE_PRODUCER = 'run_rerun_be_producer'; @@ -341,6 +345,118 @@ const failureSummary: CliFailureSummary = { recommendedFixTarget: failureContext.failure.recommendedFixTarget, }; +const passedRunSample: RunResponse = { + runId: SAMPLE_RUN_ID, + testId: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:34:00.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 8, + completed: 8, + passedCount: 8, + failedCount: 0, + }, + // Representative per-run steps so `test steps --run-id --dry-run` + // demonstrates real output instead of an empty list (the generic + // `/runs/{runId}` sample is also used by `test wait`, which ignores steps). + steps: [ + { + stepIndex: '0001', + type: 'action', + action: 'navigate', + status: 'passed', + description: 'Open the target URL', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:10.000Z', + }, + { + stepIndex: '0002', + type: 'assertion', + action: 'assert_visible', + status: 'passed', + description: 'Dashboard heading is visible', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:20.000Z', + }, + ], +}; + +const failedRunSample: RunResponse = { + runId: SAMPLE_FAILED_RUN_ID, + testId: SAMPLE_TEST_ID_FAILED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'failed', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:34:00.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: 3, + failureKind: 'assertion', + error: 'Expected billing status badge to be visible, but it was not found.', + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: 2, + failedCount: 1, + }, + steps: [ + { + stepIndex: '0001', + type: 'action', + action: 'navigate', + status: 'passed', + description: 'Open the target URL', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:10.000Z', + }, + { + stepIndex: '0002', + type: 'assertion', + action: 'assert_visible', + status: 'passed', + description: 'Dashboard heading is visible', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:20.000Z', + }, + { + stepIndex: '0003', + type: 'assertion', + action: 'assert_visible', + status: 'failed', + description: 'Billing status badge is visible', + error: 'Expected billing status badge to be visible, but it was not found.', + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:30.000Z', + }, + ], +}; + /** * Dry-run sample lookup keyed by OpenAPI operationId. Order matters in * {@link findSample}: more specific patterns must precede their generic @@ -691,71 +807,12 @@ const ENTRIES: DryRunSampleEntry[] = [ }, } satisfies BatchRerunResponse), // M3.3 piece-3 — GET /runs/{runId} (live status / long-poll). - // Use a terminal failed row with a concrete failed step so - // `test steps --run-id --dry-run` demonstrates the run-scoped - // error + failedStepIndex mapping without needing live credentials. - entry('getRun', 'GET', '/runs/{runId}', { - runId: SAMPLE_RUN_ID, - testId: SAMPLE_TEST_ID_FAILED, - projectId: SAMPLE_PROJECT_ID, - userId: SAMPLE_USER_ID, - status: 'failed', - source: 'cli', - createdAt: '2026-05-15T19:32:00.000Z', - startedAt: '2026-05-15T19:32:05.000Z', - finishedAt: '2026-05-15T19:34:00.000Z', - codeVersion: 'v1', - targetUrl: SAMPLE_TARGET_URL, - createdFrom: null, - failedStepIndex: 3, - failureKind: 'assertion', - error: 'Expected billing status badge to be visible, but it was not found.', - videoUrl: null, - stepSummary: { - total: 3, - completed: 3, - passedCount: 2, - failedCount: 1, - }, - // Representative per-run steps so `test steps --run-id --dry-run` - // demonstrates real output instead of an empty list (the generic - // `/runs/{runId}` sample is also safe for wait flows, which ignore steps). - steps: [ - { - stepIndex: '0001', - type: 'action', - action: 'navigate', - status: 'passed', - description: 'Open the target URL', - error: null, - screenshotUrl: null, - htmlSnapshotUrl: null, - createdAt: '2026-05-15T19:32:10.000Z', - }, - { - stepIndex: '0002', - type: 'assertion', - action: 'assert_visible', - status: 'passed', - description: 'Dashboard heading is visible', - error: null, - screenshotUrl: null, - htmlSnapshotUrl: null, - createdAt: '2026-05-15T19:32:20.000Z', - }, - { - stepIndex: '0003', - type: 'assertion', - action: 'assert_visible', - status: 'failed', - description: 'Billing status badge is visible', - error: 'Expected billing status badge to be visible, but it was not found.', - screenshotUrl: null, - htmlSnapshotUrl: null, - createdAt: '2026-05-15T19:32:30.000Z', - }, - ], - } satisfies RunResponse), + // A terminal `passed` row is the most useful dry-run shape: agents see + // what a completed run looks like, and `--wait` terminates immediately. + // fix(2026-05-21): a duplicate failed-shape entry that appeared before + // this entry was removed; findSample first-match-wins was always + // returning status: "failed" for `test wait --dry-run`. + entry('getRun', 'GET', '/runs/{runId}', passedRunSample), ]; function entry( @@ -807,11 +864,15 @@ export function findSample( const pathOnly = extractPath(url); for (const e of ENTRIES) { if (e.method === upper && e.pattern.test(pathOnly)) { + const body = + e.operationId === 'getRun' && pathOnly === `/runs/${SAMPLE_FAILED_RUN_ID}` + ? failedRunSample + : e.body(requestBody); // Rebind body so callers get the resolved value, not the factory. // We return a new object with `body` already applied so downstream // code can keep calling `e.body` as-before (no API break for tests // that call `findSample` directly). - return { ...e, body: () => e.body(requestBody) }; + return { ...e, body: () => body }; } } return undefined;