Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/commands/test.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2667,6 +2667,40 @@ describe('runSteps', () => {
expect(block.match(/error: /g)).toHaveLength(1);
});

it('--run-id run_failed_sample dry-run sample maps the failed step error and contributor flag', async () => {
const out: string[] = [];
const page = await runSteps(
{
profile: 'default',
output: 'json',
debug: false,
dryRun: true,
testId: 'test_fe',
runId: 'run_failed_sample',
},
{
env: {} as NodeJS.ProcessEnv,
credentialsPath: join(tmpdir(), 'testsprite-no-creds'),
stdout: line => out.push(line),
stderr: () => undefined,
},
);

const failing = page.items.find(step => step.stepIndex === 3);
expect(failing).toMatchObject({
status: 'failed',
error: expect.any(String),
stepType: 'assertion',
outcomeContributesToFailure: true,
});
expect(page.items.find(step => step.stepIndex === 1)?.outcomeContributesToFailure).toBe(false);

const printed = JSON.parse(out[0]!) as { items: Array<{ stepIndex: number; error?: string }> };
expect(printed.items.some(step => step.stepIndex === 3 && typeof step.error === 'string')).toBe(
true,
);
});

it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => {
const { credentialsPath } = makeCreds();
// The run-scoped endpoint returns a run whose testId differs from the
Expand Down
38 changes: 38 additions & 0 deletions src/lib/dry-run/samples.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,44 @@ describe('findSample', () => {
expect(body.stepSummary.failedCount).toBe(0);
});

it('GET /runs/run_failed_sample resolves to the sentinel failed run-scoped step sample', () => {
const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_failed_sample');
expect(e?.operationId).toBe('getRun');
const body = e?.body() as {
status: string;
runId: string;
failedStepIndex: number | null;
failureKind: string | null;
error: string | null;
stepSummary: { total: number; completed: number; passedCount: number; failedCount: number };
steps: Array<{
stepIndex: string;
type: string;
status: string | null;
error: string | null;
}>;
};
expect(body.runId).toBe('run_failed_sample');
expect(body.status).toBe('failed');
expect(body.failedStepIndex).toBe(3);
expect(body.failureKind).toBe('assertion');
expect(body.error).toEqual(expect.any(String));
expect(body.stepSummary).toMatchObject({
total: 3,
completed: 3,
passedCount: 2,
failedCount: 1,
});

const failingStep = body.steps.find(step => step.stepIndex === '0003');
expect(failingStep).toMatchObject({
type: 'assertion',
status: 'failed',
error: expect.any(String),
});
expect(failingStep?.error).not.toBe('');
});

it('getTest sample carries priority field (G1a)', () => {
const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc');
expect(e?.operationId).toBe('getTest');
Expand Down
174 changes: 122 additions & 52 deletions src/lib/dry-run/samples.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ const SAMPLE_TEST_ID_FAILED = 'test_8f2a4d10';
const SAMPLE_TEST_ID_PASSED = 'test_3a91bb02';
const SAMPLE_TEST_ID_BLOCKED = 'test_blocked_4f7a';
export const SAMPLE_RUN_ID = 'run_abc';
// Documented sentinel for `test steps --run-id run_failed_sample --dry-run`:
// keeps wait flows on the default passed sample while still demonstrating a
// run-scoped failed step offline.
const SAMPLE_FAILED_RUN_ID = 'run_failed_sample';
// M3.4 rerun dry-run sample IDs
const SAMPLE_RERUN_ID_BE_NAMED = 'run_rerun_be_named';
const SAMPLE_RERUN_ID_BE_PRODUCER = 'run_rerun_be_producer';
Expand Down Expand Up @@ -341,6 +345,118 @@ const failureSummary: CliFailureSummary = {
recommendedFixTarget: failureContext.failure.recommendedFixTarget,
};

const passedRunSample: RunResponse = {
runId: SAMPLE_RUN_ID,
testId: SAMPLE_TEST_ID_PASSED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'passed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: null,
failureKind: null,
error: null,
videoUrl: null,
stepSummary: {
total: 8,
completed: 8,
passedCount: 8,
failedCount: 0,
},
// Representative per-run steps so `test steps --run-id <id> --dry-run`
// demonstrates real output instead of an empty list (the generic
// `/runs/{runId}` sample is also used by `test wait`, which ignores steps).
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
],
};

const failedRunSample: RunResponse = {
runId: SAMPLE_FAILED_RUN_ID,
testId: SAMPLE_TEST_ID_FAILED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'failed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: 3,
failureKind: 'assertion',
error: 'Expected billing status badge to be visible, but it was not found.',
videoUrl: null,
stepSummary: {
total: 3,
completed: 3,
passedCount: 2,
failedCount: 1,
},
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
{
stepIndex: '0003',
type: 'assertion',
action: 'assert_visible',
status: 'failed',
description: 'Billing status badge is visible',
error: 'Expected billing status badge to be visible, but it was not found.',
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:30.000Z',
},
],
};

/**
* Dry-run sample lookup keyed by OpenAPI operationId. Order matters in
* {@link findSample}: more specific patterns must precede their generic
Expand Down Expand Up @@ -696,57 +812,7 @@ const ENTRIES: DryRunSampleEntry[] = [
// fix(2026-05-21): a duplicate failed-shape entry that appeared before
// this entry was removed; findSample first-match-wins was always
// returning status: "failed" for `test wait --dry-run`.
entry('getRun', 'GET', '/runs/{runId}', {
runId: SAMPLE_RUN_ID,
testId: SAMPLE_TEST_ID_PASSED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'passed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: null,
failureKind: null,
error: null,
videoUrl: null,
stepSummary: {
total: 8,
completed: 8,
passedCount: 8,
failedCount: 0,
},
// Representative per-run steps so `test steps --run-id <id> --dry-run`
// demonstrates real output instead of an empty list (the generic
// `/runs/{runId}` sample is also used by `test wait`, which ignores steps).
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
],
} satisfies RunResponse),
entry('getRun', 'GET', '/runs/{runId}', passedRunSample),
];

function entry(
Expand Down Expand Up @@ -798,11 +864,15 @@ export function findSample(
const pathOnly = extractPath(url);
for (const e of ENTRIES) {
if (e.method === upper && e.pattern.test(pathOnly)) {
const body =
e.operationId === 'getRun' && pathOnly === `/runs/${SAMPLE_FAILED_RUN_ID}`
? failedRunSample
: e.body(requestBody);
// Rebind body so callers get the resolved value, not the factory.
// We return a new object with `body` already applied so downstream
// code can keep calling `e.body` as-before (no API break for tests
// that call `findSample` directly).
return { ...e, body: () => e.body(requestBody) };
return { ...e, body: () => body };
}
}
return undefined;
Expand Down
Loading