diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 146ff6698..1a3c55e94 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -409,9 +409,8 @@ export class BenchmarkRunner { const suites = await this._prepareAllSuites(); try { for (const suite of suites) { - if (suite.disabled) + if (!suite.enabled) continue; - try { await this._appendFrame(); this._page = new Page(this._frame); diff --git a/resources/developer-mode.mjs b/resources/developer-mode.mjs index 3aa89ce02..5b9f3270d 100644 --- a/resources/developer-mode.mjs +++ b/resources/developer-mode.mjs @@ -145,7 +145,7 @@ function createUIForSuites() { control.className = "suites"; const checkboxes = []; const setSuiteEnabled = (suiteIndex, enabled) => { - Suites[suiteIndex].disabled = !enabled; + Suites[suiteIndex].enabled = enabled; checkboxes[suiteIndex].checked = enabled; }; @@ -157,9 +157,9 @@ function createUIForSuites() { const checkbox = document.createElement("input"); checkbox.id = suite.name; checkbox.type = "checkbox"; - checkbox.checked = !suite.disabled; + checkbox.checked = suite.enabled; checkbox.onchange = () => { - suite.disabled = !checkbox.checked; + suite.enabled = checkbox.checked; updateURL(); }; checkboxes.push(checkbox); @@ -275,17 +275,17 @@ function updateParamsSuitesAndTags() { // If less than all suites are selected then change the URL "Suites" GET parameter // to comma separate only the selected - const selectedSuites = Suites.filter((suite) => !suite.disabled); + const selectedSuites = Suites.filter((suite) => suite.enabled); if (!selectedSuites.length) return; // Try finding common tags that would result in the current suite selection. let commonTags = new Set(selectedSuites[0].tags); for (const suite of Suites) { - if (suite.disabled) - suite.tags.forEach((tag) => commonTags.delete(tag)); - else + if (suite.enabled) commonTags = new Set(suite.tags.filter((tag) => commonTags.has(tag))); + else + suite.tags.forEach((tag) => commonTags.delete(tag)); } if (selectedSuites.length > 1 && commonTags.size) params.tags = [...commonTags]; diff --git a/resources/main.mjs b/resources/main.mjs index a39dd06e5..20fbd807e 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -66,7 +66,9 @@ class MainBenchmarkClient { if (this._isRunning) return false; - if (Suites.every((suite) => suite.disabled)) { + const enabledSuites = Suites.filter((suite) => suite.enabled); + const totalSuitesCount = enabledSuites.length; + if (totalSuitesCount === 0) { const message = `No suites selected - "${params.suites}" does not exist.`; alert(message); console.error( @@ -75,7 +77,6 @@ class MainBenchmarkClient { "\nValid values:", Suites.map((each) => each.name) ); - return false; } if (!this._isStepping()) @@ -93,8 +94,6 @@ class MainBenchmarkClient { this._metrics = Object.create(null); this._isRunning = true; - const enabledSuites = Suites.filter((suite) => !suite.disabled); - const totalSuitesCount = enabledSuites.length; this.stepCount = params.iterationCount * totalSuitesCount; this._progressCompleted.max = this.stepCount; this.suitesCount = enabledSuites.length; diff --git a/resources/tests.mjs b/resources/tests.mjs index d7cf6794a..0a3878423 100644 --- a/resources/tests.mjs +++ b/resources/tests.mjs @@ -16,10 +16,7 @@ Suites.enable = function (names, tags) { if (names?.length) { const lowerCaseNames = names.map((each) => each.toLowerCase()); this.forEach((suite) => { - if (lowerCaseNames.includes(suite.name.toLowerCase())) - suite.disabled = false; - else - suite.disabled = true; + suite.enabled = lowerCaseNames.includes(suite.name.toLowerCase()); }); } else if (tags?.length) { tags.forEach((tag) => { @@ -28,15 +25,15 @@ Suites.enable = function (names, tags) { }); const tagsSet = new Set(tags); this.forEach((suite) => { - suite.disabled = !suite.tags.some((tag) => tagsSet.has(tag)); + suite.enabled = suite.tags.some((tag) => tagsSet.has(tag)); }); } else { console.warn("Neither names nor tags provided. Enabling all default suites."); this.forEach((suite) => { - suite.disabled = !("default" in suite.tags); + suite.enabled = suite.tags.includes("default"); }); } - if (this.some((suite) => !suite.disabled)) + if (this.some((suite) => suite.enabled)) return; let message, debugInfo; if (names?.length) { @@ -60,7 +57,6 @@ Suites.push({ name: "TodoMVC-LocalStorage", url: "experimental/todomvc-localstorage/dist/index.html", tags: ["todomvc"], - disabled: true, async prepare(page) { (await page.waitForElement(".new-todo")).focus(); page.getLocalStorage().getItem("javascript-es5"); @@ -91,7 +87,6 @@ Suites.push({ name: "TodoMVC-Emoji", url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html", tags: ["todomvc", "experimental"], - disabled: true, async prepare(page) { await page.waitForElement("todo-app"); }, @@ -124,7 +119,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-JavaScript-ES5", url: "resources/todomvc/vanilla-examples/javascript-es5/dist/index.html", - tags: ["todomvc"], + tags: ["default", "todomvc"], async prepare(page) { (await page.waitForElement(".new-todo")).focus(); }, @@ -154,7 +149,6 @@ Suites.push({ name: "TodoMVC-JavaScript-ES5-Complex-DOM", url: "resources/todomvc/vanilla-examples/javascript-es5-complex/dist/index.html", tags: ["todomvc", "complex"], - disabled: true, async prepare(page) { (await page.waitForElement(".new-todo")).focus(); }, @@ -184,7 +178,6 @@ Suites.push({ name: "TodoMVC-JavaScript-ES6-Webpack", url: "resources/todomvc/vanilla-examples/javascript-es6-webpack/dist/index.html", tags: ["todomvc"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -214,7 +207,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-JavaScript-ES6-Webpack-Complex-DOM", url: "resources/todomvc/vanilla-examples/javascript-es6-webpack-complex/dist/index.html", - tags: ["todomvc", "complex", "complex-default"], + tags: ["default", "todomvc", "complex", "complex-default"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -244,7 +237,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-WebComponents", url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html", - tags: ["todomvc", "webcomponents"], + tags: ["default", "todomvc", "webcomponents"], async prepare(page) { await page.waitForElement("todo-app"); }, @@ -278,7 +271,6 @@ Suites.push({ name: "TodoMVC-WebComponents-Complex-DOM", url: "resources/todomvc/vanilla-examples/javascript-web-components-complex/dist/index.html", tags: ["todomvc", "webcomponents", "complex"], - disabled: true, async prepare(page) { await page.waitForElement("todo-app"); }, @@ -312,7 +304,6 @@ Suites.push({ name: "TodoMVC-React", url: "resources/todomvc/architecture-examples/react/dist/index.html#/home", tags: ["todomvc"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -342,7 +333,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-React-Complex-DOM", url: "resources/todomvc/architecture-examples/react-complex/dist/index.html#/home", - tags: ["todomvc", "complex", "complex-default"], + tags: ["default", "todomvc", "complex", "complex-default"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -372,7 +363,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-React-Redux", url: "resources/todomvc/architecture-examples/react-redux/dist/index.html", - tags: ["todomvc"], + tags: ["default", "todomvc"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -402,7 +393,6 @@ Suites.push({ name: "TodoMVC-React-Redux-Complex-DOM", url: "resources/todomvc/architecture-examples/react-redux-complex/dist/index.html", tags: ["todomvc", "complex"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -431,7 +421,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Backbone", url: "resources/todomvc/architecture-examples/backbone/dist/index.html", - tags: ["todomvc"], + tags: ["default", "todomvc"], async prepare(page) { await page.waitForElement("#appIsReady"); const newTodo = page.querySelector(".new-todo"); @@ -463,7 +453,6 @@ Suites.push({ name: "TodoMVC-Backbone-Complex-DOM", url: "resources/todomvc/architecture-examples/backbone-complex/dist/index.html", tags: ["todomvc", "complex"], - disabled: true, async prepare(page) { await page.waitForElement("#appIsReady"); const newTodo = page.querySelector(".new-todo"); @@ -495,7 +484,6 @@ Suites.push({ name: "TodoMVC-Angular", url: "resources/todomvc/architecture-examples/angular/dist/index.html", tags: ["todomvc"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -525,7 +513,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Angular-Complex-DOM", url: "resources/todomvc/architecture-examples/angular-complex/dist/index.html", - tags: ["todomvc", "complex", "complex-default"], + tags: ["default", "todomvc", "complex", "complex-default"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -555,7 +543,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Vue", url: "resources/todomvc/architecture-examples/vue/dist/index.html", - tags: ["todomvc"], + tags: ["default", "todomvc"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -586,7 +574,6 @@ Suites.push({ name: "TodoMVC-Vue-Complex-DOM", url: "resources/todomvc/architecture-examples/vue-complex/dist/index.html", tags: ["todomvc", "complex", "complex-default"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -616,7 +603,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-jQuery", url: "resources/todomvc/architecture-examples/jquery/dist/index.html", - tags: ["todomvc"], + tags: ["default", "todomvc"], async prepare(page) { await page.waitForElement("#appIsReady"); const newTodo = page.getElementById("new-todo"); @@ -645,7 +632,6 @@ Suites.push({ name: "TodoMVC-jQuery-Complex-DOM", url: "resources/todomvc/architecture-examples/jquery-complex/dist/index.html", tags: ["todomvc", "complex"], - disabled: true, async prepare(page) { await page.waitForElement("#appIsReady"); const newTodo = page.getElementById("new-todo"); @@ -674,7 +660,6 @@ Suites.push({ name: "TodoMVC-Preact", url: "resources/todomvc/architecture-examples/preact/dist/index.html#/home", tags: ["todomvc"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -703,7 +688,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Preact-Complex-DOM", url: "resources/todomvc/architecture-examples/preact-complex/dist/index.html#/home", - tags: ["todomvc", "complex", "complex-default"], + tags: ["default", "todomvc", "complex", "complex-default"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -733,7 +718,6 @@ Suites.push({ name: "TodoMVC-Svelte", url: "resources/todomvc/architecture-examples/svelte/dist/index.html", tags: ["todomvc"], - disabled: true, async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -762,7 +746,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Svelte-Complex-DOM", url: "resources/todomvc/architecture-examples/svelte-complex/dist/index.html", - tags: ["todomvc", "complex", "complex-default"], + tags: ["default", "todomvc", "complex", "complex-default"], async prepare(page) { const element = await page.waitForElement(".new-todo"); element.focus(); @@ -792,7 +776,6 @@ Suites.push({ name: "TodoMVC-Lit", url: "resources/todomvc/architecture-examples/lit/dist/index.html", tags: ["todomvc", "webcomponents"], - disabled: true, async prepare(page) { await page.waitForElement("todo-app"); }, @@ -824,7 +807,7 @@ Suites.push({ Suites.push({ name: "TodoMVC-Lit-Complex-DOM", url: "resources/todomvc/architecture-examples/lit-complex/dist/index.html", - tags: ["todomvc", "webcomponents", "complex", "complex-default"], + tags: ["default", "todomvc", "webcomponents", "complex", "complex-default"], async prepare(page) { await page.waitForElement("todo-app"); }, @@ -856,7 +839,7 @@ Suites.push({ Suites.push({ name: "NewsSite-Next", url: "resources/newssite/news-next/dist/index.html", - tags: ["newssite", "language"], + tags: ["default", "newssite", "language"], async prepare(page) { await page.waitForElement("#navbar-dropdown-toggle"); }, @@ -898,7 +881,6 @@ Suites.push({ name: "NewsSite-PostMessage", url: "resources/newssite/news-next/dist/index.html", tags: ["experimental", "newssite", "language"], - disabled: true, async prepare() {}, type: "remote", /* config: { @@ -909,7 +891,7 @@ Suites.push({ Suites.push({ name: "NewsSite-Nuxt", url: "resources/newssite/news-nuxt/dist/index.html", - tags: ["newssite"], + tags: ["default", "newssite"], async prepare(page) { await page.waitForElement("#navbar-dropdown-toggle"); }, @@ -950,7 +932,7 @@ Suites.push({ Suites.push({ name: "Editor-CodeMirror", url: "resources/editors/dist/codemirror.html", - tags: ["editor"], + tags: ["default", "editor"], async prepare(page) {}, tests: [ new BenchmarkTestStep("Long", (page) => { @@ -969,7 +951,7 @@ Suites.push({ Suites.push({ name: "Editor-TipTap", url: "resources/editors/dist/tiptap.html", - tags: ["editor"], + tags: ["default", "editor"], async prepare(page) {}, tests: [ new BenchmarkTestStep("Long", (page) => { @@ -988,7 +970,7 @@ Suites.push({ Suites.push({ name: "Charts-observable-plot", url: "resources/charts/dist/observable-plot.html", - tags: ["chart"], + tags: ["default", "chart"], async prepare(page) {}, tests: [ new BenchmarkTestStep("Stacked by 6", (page) => { @@ -1015,7 +997,7 @@ Suites.push({ Suites.push({ name: "Charts-chartjs", url: "resources/charts/dist/chartjs.html", - tags: ["chart"], + tags: ["default", "chart"], async prepare(page) {}, tests: [ new BenchmarkTestStep("Draw scatter", (page) => { @@ -1035,7 +1017,7 @@ Suites.push({ Suites.push({ name: "React-Stockcharts-SVG", url: "resources/react-stockcharts/build/index.html?type=svg", - tags: ["chart", "svg"], + tags: ["default", "chart", "svg"], async prepare(page) { await page.waitForElement("#render"); }, @@ -1075,7 +1057,7 @@ Suites.push({ Suites.push({ name: "Perf-Dashboard", url: "resources/perf.webkit.org/public/v3/#/charts/?since=1678991819934&paneList=((55-1974-null-null-(5-2.5-500)))", - tags: ["chart", "webcomponents"], + tags: ["default", "chart", "webcomponents"], async prepare(page) { await page.waitForElement("#app-is-ready"); page.call("startTest"); @@ -1119,10 +1101,9 @@ Suites.forEach((suite) => { suite.tags = []; if (suite.url.startsWith("experimental/")) suite.tags.unshift("all", "experimental"); - else if (suite.disabled) - suite.tags.unshift("all"); else - suite.tags.unshift("all", "default"); + suite.tags.unshift("all"); + suite.enabled = suite.tags.includes("default"); Object.freeze(suite.tags); Object.freeze(suite.steps); }); diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index b7e4886cd..c3b246266 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -73,8 +73,9 @@ async function testIterations() { const iterationCount = 2; const metrics = await testPage(`index.html?iterationCount=${iterationCount}`); Suites.forEach((suite) => { - if (!suite.disabled) { + if (suite.enabled) { const metric = metrics[suite.name]; + assert(metric, `Missing suite result for ${suite.name}`); assert(metric.values.length === iterationCount); } else { assert(!(suite.name in metrics)); diff --git a/tests/unittests/benchmark-runner.mjs b/tests/unittests/benchmark-runner.mjs index 94b3059ad..16d765e2a 100644 --- a/tests/unittests/benchmark-runner.mjs +++ b/tests/unittests/benchmark-runner.mjs @@ -14,11 +14,13 @@ const SUITES_FIXTURE = [ { name: "Suite 1", async prepare(page) {}, + enabled: true, tests: [TEST_FIXTURE("Test 1"), TEST_FIXTURE("Test 2"), TEST_FIXTURE("Test 3")], }, { name: "Suite 2", async prepare(page) {}, + enabled: true, tests: [TEST_FIXTURE("Test 1")], }, ]; @@ -138,7 +140,7 @@ describe("BenchmarkRunner", () => { }); describe("runSuite", () => { - let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteRessultsStub, _suitePrepareSpy, performanceMarkSpy; + let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteResultsStub, _suitePrepareSpy, performanceMarkSpy; const suite = SUITES_FIXTURE[0]; @@ -146,7 +148,7 @@ describe("BenchmarkRunner", () => { _prepareSuiteSpy = spy(SuiteRunner.prototype, "_prepareSuite"); _loadFrameStub = stub(SuiteRunner.prototype, "_loadFrame").callsFake(async () => null); _runTestStub = stub(TestRunner.prototype, "runTest").callsFake(async () => null); - _validateSuiteRessultsStub = stub(SuiteRunner.prototype, "_validateSuiteResults").callsFake(async () => null); + _validateSuiteResultsStub = stub(SuiteRunner.prototype, "_validateSuiteResults").callsFake(async () => null); performanceMarkSpy = spy(window.performance, "mark"); _suitePrepareSpy = spy(suite, "prepare"); @@ -161,7 +163,7 @@ describe("BenchmarkRunner", () => { it("should run and record results for every test in suite", async () => { assert.calledThrice(_runTestStub); - assert.calledOnce(_validateSuiteRessultsStub); + assert.calledOnce(_validateSuiteResultsStub); assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-start"); assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-end"); assert.calledWith(performanceMarkSpy, "suite-Suite 1-start");