diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 88b059a35a145..169b8c943f11e 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -515,6 +515,27 @@ export default defineConfig({ }); ``` +## property: TestConfig.retryStrategy +* since: v1.62 +- type: ?<[RetryStrategy]<"immediate"|"deferred">> + +Controls when failed tests are retried. Defaults to `'immediate'`. +* `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the run. This is the default. +* `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the configured number of [workers](#test-config-workers). + +Learn more about [test retries](../test-retries.md#retries). + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + retries: 2, + retryStrategy: 'deferred', +}); +``` + ## property: TestConfig.shard * since: v1.10 - type: ?<[null]|[Object]> diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index e521d61c49802..4dd22b11d8e81 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,6 +48,7 @@ export class FullConfigInternal { readonly projects: FullProjectInternal[] = []; readonly singleTSConfigPath?: string; readonly captureGitInfo: Config['captureGitInfo']; + readonly retryStrategy: 'immediate' | 'deferred'; defineConfigWasUsed = false; globalSetups: string[] = []; @@ -67,6 +68,7 @@ export class FullConfigInternal { this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig); this.captureGitInfo = userConfig.captureGitInfo; + this.retryStrategy = takeFirst(userConfig.retryStrategy, 'immediate'); this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 32f3a5621724f..9beee268caa5c 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -243,6 +243,11 @@ function validateConfig(file: string, config: Config) { throw errorWithFile(file, `config.updateSnapshots must be one of "all", "changed", "missing" or "none"`); } + if ('retryStrategy' in config && config.retryStrategy !== undefined) { + if (typeof config.retryStrategy !== 'string' || !['immediate', 'deferred'].includes(config.retryStrategy)) + throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "deferred"`); + } + if ('tsconfig' in config && config.tsconfig !== undefined) { if (typeof config.tsconfig !== 'string') throw errorWithFile(file, `config.tsconfig must be a string`); diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 29f4f4e84eb1a..77cb0aa86cdab 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -152,7 +152,10 @@ export class Dispatcher { // 5. Possibly queue a new job with leftover tests and/or retries. if (!this._isStopped && result.newJob) { - this._queue.unshift(result.newJob); + if (this._testRun.config.retryStrategy === 'deferred') + this._queue.push(result.newJob); + else + this._queue.unshift(result.newJob); this._updateCounterForWorkerHash(result.newJob.workerHash, +1); } } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c2037da8f806b..ab8160cd39182 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1640,6 +1640,30 @@ interface TestConfig { */ retries?: number; + /** + * Controls when failed tests are retried. Defaults to `'immediate'`. + * - `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the + * run. This is the default. + * - `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the + * configured number of [workers](#test-config-workers). + * + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * retries: 2, + * retryStrategy: 'deferred', + * }); + * ``` + * + */ + retryStrategy?: "immediate"|"deferred"; + /** * Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`. * diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index c82a61b20fb8d..1800eaded07d8 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -262,3 +262,39 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline expect(result.output).toContain('Failed on first run'); expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]); }); + +test('should defer retries to the end of the run', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { retries: 3, retryStrategy: 'deferred' }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('a', ({}, testInfo) => { + console.log('\\n%%a-' + testInfo.retry); + expect(testInfo.retry).toBe(3); + }); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test.describe.configure({ retries: 2 }); + test('b', ({}, testInfo) => { + console.log('\\n%%b-' + testInfo.retry); + expect(testInfo.retry).toBe(2); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(2); + expect(result.results.length).toBe(7); + // First attempts run before any retry, and each retry round is interleaved. + expect(result.outputLines).toEqual([ + 'a-0', + 'b-0', + 'a-1', + 'b-1', + 'a-2', + 'b-2', + 'a-3', + ]); +});