From d74805f11b4156c522c3ed745e6cc7a4ecffdfa3 Mon Sep 17 00:00:00 2001 From: vctqs1 Date: Sat, 13 Jun 2026 13:57:24 +0700 Subject: [PATCH 1/5] feat(test-runner): add `repeat` config in test --- docs/src/test-api/class-test.md | 19 +++++++ package-lock.json | 5 -- packages/playwright/src/common/testType.ts | 39 +++++++++---- packages/playwright/src/common/validators.ts | 7 +++ packages/playwright/types/test.d.ts | 42 ++++++++++++++ tests/playwright-test/test-repeat.spec.ts | 60 ++++++++++++++++++++ 6 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 tests/playwright-test/test-repeat.spec.ts diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 4992d168c67dc..e40e266dddecc 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -85,6 +85,25 @@ You can also add annotations during runtime by manipulating [`property: TestInfo Learn more about [test annotations](../test-annotations.md). +**Repeat** + +You can repeat an individual test multiple times by providing the `repeat` property in the test details. + +This is useful when investigating flaky tests, validating test stability, or stress-testing specific scenarios without repeating the entire test suite. + +Each repetition is treated as a separate test run and is reported independently. + +```js +import { test, expect } from '@playwright/test'; + +test('repeated test', { + repeat: 3, +}, async ({ page }) => { + await page.goto('https://playwright.dev/'); + // ... +}); +``` + ### param: Test.(call).title * since: v1.10 - `title` <[string]> diff --git a/package-lock.json b/package-lock.json index 3926f80ae718b..d05edce705a58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9191,11 +9191,6 @@ "@browser-logos/safari": "2.1.0" } }, - "packages/devtools": { - "name": "@playwright/devtools", - "version": "0.0.0", - "extraneous": true - }, "packages/extension": { "name": "@playwright/extension", "version": "0.2.1", diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 9911efe09c78d..d615ce65df169 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -108,18 +108,33 @@ export class TestTypeImpl { } const validatedDetails = validateTestDetails(details, location); - const test = new TestCase(title, body, this, location); - test._requireFile = suite._requireFile; - test.annotations.push(...validatedDetails.annotations); - test._tags.push(...validatedDetails.tags); - suite._addTest(test); - - if (type === 'only' || type === 'fail.only') - test._only = true; - if (type === 'skip' || type === 'fixme' || type === 'fail') - test.annotations.push({ type, location }); - else if (type === 'fail.only') - test.annotations.push({ type: 'fail', location }); + const _this = this; + function createTestInstance(title: string) { + if (!suite) + return; + + const test = new TestCase(title, body, _this, location); + test._requireFile = suite._requireFile; + test.annotations.push(...validatedDetails.annotations); + test._tags.push(...validatedDetails.tags); + suite._addTest(test); + + if (type === 'only' || type === 'fail.only') + test._only = true; + if (type === 'skip' || type === 'fixme' || type === 'fail') + test.annotations.push({ type, location }); + else if (type === 'fail.only') + test.annotations.push({ type: 'fail', location }); + + } + + // If repeat is specified, create multiple test instances with modified titles. + if (validatedDetails.repeat && validatedDetails.repeat > 1) { + for (let i = 0; i < validatedDetails.repeat; i++) + createTestInstance(`${title} (${i + 1}/${validatedDetails.repeat})`); + } else { + createTestInstance(title); + } } private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) { diff --git a/packages/playwright/src/common/validators.ts b/packages/playwright/src/common/validators.ts index 0dd13523657b1..ffce64c773996 100644 --- a/packages/playwright/src/common/validators.ts +++ b/packages/playwright/src/common/validators.ts @@ -38,6 +38,7 @@ const testDetailsSchema: JsonSchema = { { type: 'array', items: { type: 'string', pattern: '^@', patternError: "Tag must start with '@'" } }, ] }, + repeat: { type: 'number', pattern: '^[1-9][0-9]*$', patternError: 'repeat must be a positive integer' }, annotation: { oneOf: [ testAnnotationSchema, @@ -51,6 +52,7 @@ type ValidTestDetails = { tags: string[]; annotations: (TestDetailsAnnotation & { location: Location })[]; location: Location; + repeat?: number; }; export function validateTestDetails(details: unknown, location: Location): ValidTestDetails { @@ -65,9 +67,14 @@ export function validateTestDetails(details: unknown, location: Location): Valid const annotation = obj.annotation; const annotations: TestDetailsAnnotation[] = annotation === undefined ? [] : Array.isArray(annotation) ? annotation : [annotation as TestDetailsAnnotation]; + const repeat = typeof obj.repeat === 'number' ? obj.repeat : undefined; + if (repeat !== undefined) + tags.push(`@repeat=${repeat}`); + return { annotations: annotations.map(a => ({ ...a, location })), tags, location, + repeat, }; } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c2037da8f806b..43f713cfadd68 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -2784,6 +2784,27 @@ export interface TestType { * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). * * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + * + * **Repeat** + * + * You can repeat an individual test multiple times by providing the `repeat` property in the test details. + * + * This is useful when investigating flaky tests, validating test stability, or stress-testing specific scenarios + * without repeating the entire test suite. + * + * Each repetition is treated as a separate test run and is reported independently. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('repeated test', { + * repeat: 3, + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * * @param title Test title. * @param details Additional test details. * @param body Test body that takes one or two arguments: an object with fixtures and optional @@ -2861,6 +2882,27 @@ export interface TestType { * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). * * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + * + * **Repeat** + * + * You can repeat an individual test multiple times by providing the `repeat` property in the test details. + * + * This is useful when investigating flaky tests, validating test stability, or stress-testing specific scenarios + * without repeating the entire test suite. + * + * Each repetition is treated as a separate test run and is reported independently. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('repeated test', { + * repeat: 3, + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * * @param title Test title. * @param details Additional test details. * @param body Test body that takes one or two arguments: an object with fixtures and optional diff --git a/tests/playwright-test/test-repeat.spec.ts b/tests/playwright-test/test-repeat.spec.ts new file mode 100644 index 0000000000000..01494a6bec52a --- /dev/null +++ b/tests/playwright-test/test-repeat.spec.ts @@ -0,0 +1,60 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should create test n times', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + export default class Reporter { + onBegin(config, suite) { + const visit = suite => { + for (const test of suite.tests || []) + console.log('\\n%%title=' + test.title + ', tags=' + test.tags.join(',')); + for (const child of suite.suites || []) + visit(child); + }; + visit(suite); + } + onError(error) { + console.log(error); + } + } + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + tag: '@global', + }; + `, + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('no-repeat', () => { + }); + test('repeat-equal-1', { tag: '@foo' }, () => { + }); + test('repeat-equal-2', { tag: ['@foo', '@bar'] }, () => { + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `title=no-repeat, tags=@global`, + `title=repeat-equal-1, tags=@global,@inline,@foo,@repeat=1`, + `title=repeat-equal-2 (1/2), tags=@global,@foo,@bar,@repeat=2`, + `title=repeat-equal-2 (2/2), tags=@global,@foo,@bar,@repeat=2`, + ]); +}); From e6c3baf916ffd26c151a23b3ffced425bd26194f Mon Sep 17 00:00:00 2001 From: vctqs1 Date: Sat, 13 Jun 2026 13:58:59 +0700 Subject: [PATCH 2/5] chore: revert changes in package-lock.json --- package-lock.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package-lock.json b/package-lock.json index d05edce705a58..3926f80ae718b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9191,6 +9191,11 @@ "@browser-logos/safari": "2.1.0" } }, + "packages/devtools": { + "name": "@playwright/devtools", + "version": "0.0.0", + "extraneous": true + }, "packages/extension": { "name": "@playwright/extension", "version": "0.2.1", From 59de88d6ed87d2446d081758d705ec0abef39595 Mon Sep 17 00:00:00 2001 From: vctqs1 Date: Sat, 13 Jun 2026 14:14:16 +0700 Subject: [PATCH 3/5] chore(test-runner): update test-repeat --- packages/playwright/src/transform/esmLoaderSync.ts | 2 +- tests/playwright-test/test-repeat.spec.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/playwright/src/transform/esmLoaderSync.ts b/packages/playwright/src/transform/esmLoaderSync.ts index 18c9c93af8361..378ec8196923b 100644 --- a/packages/playwright/src/transform/esmLoaderSync.ts +++ b/packages/playwright/src/transform/esmLoaderSync.ts @@ -32,7 +32,7 @@ export function resolve(specifier: string, context: { parentURL?: string, condit // - the ESM resolver wants a file:// URL and, on Windows, mistakes an absolute path's // drive letter for a URL scheme ("Received protocol 'c:'"). // The `import` condition is present only for ESM resolution, so use it to pick the form. - specifier = context.conditions?.includes('import') ? url.pathToFileURL(resolved).toString() : resolved; + specifier = context.conditions?.includes?.('import') ? url.pathToFileURL(resolved).toString() : resolved; } } const result = nextResolve(specifier, context); diff --git a/tests/playwright-test/test-repeat.spec.ts b/tests/playwright-test/test-repeat.spec.ts index 01494a6bec52a..f7195a875b20d 100644 --- a/tests/playwright-test/test-repeat.spec.ts +++ b/tests/playwright-test/test-repeat.spec.ts @@ -44,16 +44,17 @@ test('should create test n times', async ({ runInlineTest }) => { import { test, expect } from '@playwright/test'; test('no-repeat', () => { }); - test('repeat-equal-1', { tag: '@foo' }, () => { + test('repeat-equal-1', { tag: '@foo', repeat: 1 }, () => { }); - test('repeat-equal-2', { tag: ['@foo', '@bar'] }, () => { + test('repeat-equal-2', { tag: ['@foo', '@bar'], repeat: 2 }, () => { }); ` }); expect(result.exitCode).toBe(0); + expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ `title=no-repeat, tags=@global`, - `title=repeat-equal-1, tags=@global,@inline,@foo,@repeat=1`, + `title=repeat-equal-1, tags=@global,@foo,@repeat=1`, `title=repeat-equal-2 (1/2), tags=@global,@foo,@bar,@repeat=2`, `title=repeat-equal-2 (2/2), tags=@global,@foo,@bar,@repeat=2`, ]); From d74c92ea930f398a9f5b16e6755ee7a8db4ee805 Mon Sep 17 00:00:00 2001 From: vctqs1 Date: Sat, 13 Jun 2026 14:14:43 +0700 Subject: [PATCH 4/5] chore(test-runner): update test-repeat --- tests/playwright-test/test-repeat.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/playwright-test/test-repeat.spec.ts b/tests/playwright-test/test-repeat.spec.ts index f7195a875b20d..fce8a6897ca60 100644 --- a/tests/playwright-test/test-repeat.spec.ts +++ b/tests/playwright-test/test-repeat.spec.ts @@ -51,7 +51,6 @@ test('should create test n times', async ({ runInlineTest }) => { ` }); expect(result.exitCode).toBe(0); - expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ `title=no-repeat, tags=@global`, `title=repeat-equal-1, tags=@global,@foo,@repeat=1`, From 3ee6bd51313bfe2cbdf5e44672b3ed9fde4bdad1 Mon Sep 17 00:00:00 2001 From: vctqs1 Date: Sat, 13 Jun 2026 14:18:53 +0700 Subject: [PATCH 5/5] chore(test-runner): remove @repeat tag to avoid messup stuffs --- packages/playwright/src/common/validators.ts | 2 -- tests/playwright-test/test-repeat.spec.ts | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/playwright/src/common/validators.ts b/packages/playwright/src/common/validators.ts index ffce64c773996..beefb9f8c2783 100644 --- a/packages/playwright/src/common/validators.ts +++ b/packages/playwright/src/common/validators.ts @@ -68,8 +68,6 @@ export function validateTestDetails(details: unknown, location: Location): Valid const annotations: TestDetailsAnnotation[] = annotation === undefined ? [] : Array.isArray(annotation) ? annotation : [annotation as TestDetailsAnnotation]; const repeat = typeof obj.repeat === 'number' ? obj.repeat : undefined; - if (repeat !== undefined) - tags.push(`@repeat=${repeat}`); return { annotations: annotations.map(a => ({ ...a, location })), diff --git a/tests/playwright-test/test-repeat.spec.ts b/tests/playwright-test/test-repeat.spec.ts index fce8a6897ca60..a879c32facc92 100644 --- a/tests/playwright-test/test-repeat.spec.ts +++ b/tests/playwright-test/test-repeat.spec.ts @@ -53,8 +53,8 @@ test('should create test n times', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ `title=no-repeat, tags=@global`, - `title=repeat-equal-1, tags=@global,@foo,@repeat=1`, - `title=repeat-equal-2 (1/2), tags=@global,@foo,@bar,@repeat=2`, - `title=repeat-equal-2 (2/2), tags=@global,@foo,@bar,@repeat=2`, + `title=repeat-equal-1, tags=@global,@foo`, + `title=repeat-equal-2 (1/2), tags=@global,@foo,@bar`, + `title=repeat-equal-2 (2/2), tags=@global,@foo,@bar`, ]); });