diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4670c34c3..c9ca6dd1838 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -643,7 +643,23 @@ jobs: - run: pnpm install --frozen-lockfile - name: Unit Tests - run: pnpm run test.unit && echo ok > unit-tests-completed.txt + run: pnpm run test.unit + + - name: Benchmark Tests + if: matrix.settings.host == 'ubuntu-latest' + run: pnpm run test.bench + + save-unit-tests-cache: + name: Save Unit Tests Cache + runs-on: ubuntu-latest + needs: + - changes + - test-unit + if: always() && needs.changes.outputs.build-unit == 'true' && needs.test-unit.result == 'success' + + steps: + - name: Mark unit tests complete + run: echo ok > unit-tests-completed.txt - name: Save unit tests cache uses: actions/cache/save@v4 @@ -786,6 +802,42 @@ jobs: - name: CLI E2E Tests run: pnpm run test.e2e.cli.${{ matrix.settings.browser }} + save-e2e-tests-cache: + name: Save E2E Tests Cache + runs-on: ubuntu-latest + needs: + - changes + - test-e2e + if: always() && needs.changes.outputs.build-e2e == 'true' && needs.test-e2e.result == 'success' + + steps: + - name: Mark e2e tests complete + run: echo ok > e2e-tests-completed.txt + + - name: Save e2e tests cache + uses: actions/cache/save@v4 + with: + key: ${{ needs.changes.outputs.hash-e2e }} + path: e2e-tests-completed.txt + + save-cli-e2e-tests-cache: + name: Save CLI E2E Tests Cache + runs-on: ubuntu-latest + needs: + - changes + - test-cli-e2e + if: always() && needs.changes.outputs.build-cli-e2e == 'true' && needs.test-cli-e2e.result == 'success' + + steps: + - name: Mark cli e2e tests complete + run: echo ok > cli-e2e-tests-completed.txt + + - name: Save cli-e2e tests cache + uses: actions/cache/save@v4 + with: + key: ${{ needs.changes.outputs.hash-cli-e2e }} + path: cli-e2e-tests-completed.txt + ########### LINT PACKAGES ############ lint-package: name: Lint Package @@ -851,30 +903,14 @@ jobs: run: | if [ "${{ needs.test-e2e.result }}" != success ] ; then exit 1 - else - echo ok > e2e-tests-completed.txt fi - - name: Save e2e tests cache - if: needs.test-e2e.result != 'skipped' - uses: actions/cache/save@v4 - with: - key: ${{ needs.changes.outputs.hash-e2e }} - path: e2e-tests-completed.txt - name: Verify test-cli-e2e if: needs.test-cli-e2e.result != 'skipped' run: | if [ "${{ needs.test-cli-e2e.result }}" != success ] ; then exit 1 - else - echo ok > cli-e2e-tests-completed.txt fi - - name: Save cli-e2e tests cache - if: needs.test-cli-e2e.result != 'skipped' - uses: actions/cache/save@v4 - with: - key: ${{ needs.changes.outputs.hash-cli-e2e }} - path: cli-e2e-tests-completed.txt - name: Checkout uses: actions/checkout@v5 diff --git a/e2e/qwik-cli-e2e/tests/serve.spec.ts b/e2e/qwik-cli-e2e/tests/serve.spec.ts index 4e89bb665eb..6156978c65d 100644 --- a/e2e/qwik-cli-e2e/tests/serve.spec.ts +++ b/e2e/qwik-cli-e2e/tests/serve.spec.ts @@ -54,11 +54,16 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { ); assert.equal(existsSync(global.tmpDir), true); + const counterComponentPath = join( + global.tmpDir, + 'src/components/starter/counter/counter.tsx' + ); + const originalCounterContent = readFileSync(counterComponentPath, 'utf-8'); + const browser = await playwright[browserType].launch(); try { const page = await browser.newPage(); - // Collect console messages for debugging const consoleLogs: string[] = []; page.on('console', (msg) => { consoleLogs.push(`[${msg.type()}] ${msg.text()}`); @@ -75,49 +80,31 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { await page.goto(host); log('Page loaded'); - // The counter's + button (last button in the counter component) - // Counter uses CSS modules so we can't rely on class names. - // The buttons contain "-" and "+" text. const plusBtn = page.locator('button', { hasText: '+' }).first(); await plusBtn.waitFor({ timeout: 10000 }); log('Found + button'); - // Click the + button 3 times to change counter state (starts at 70) - await plusBtn.click(); - await plusBtn.click(); - await plusBtn.click(); - log('Clicked + button 3 times'); - - // Verify counter changed to 73 (the gauge renders value in a ) - await page.waitForFunction(() => document.body.textContent?.includes('73'), { - timeout: 10000, - }); + await clickUntilCounterReaches(page, plusBtn, 73, 6, 10000); log('Counter is at 73'); - // Modify the counter component (whose QRL segments are loaded on the client - // because we clicked the buttons). Add a visible marker to verify the update. - const counterComponentPath = join( - global.tmpDir, - 'src/components/starter/counter/counter.tsx' - ); - const counterContent = readFileSync(counterComponentPath, 'utf-8'); - writeFileSync( - counterComponentPath, - counterContent.replace( - ``, - `HMR-OK` - ) - ); - log('Modified counter.tsx'); - - // Wait for HMR to apply the update (new marker should appear without full reload) + const markerText = `HMR-OK-${Date.now()}`; + writeFileSync(counterComponentPath, withHmrMarker(originalCounterContent, markerText)); + log(`Modified counter.tsx with marker ${markerText}`); + try { await page.waitForFunction( - () => !!document.querySelector('[data-testid="hmr-marker"]'), - { timeout: 15000 } + (expectedMarker) => { + const marker = document.querySelector('[data-testid="hmr-marker"]'); + const gaugeValue = document.querySelector('._value_1v6hy_9, [class*="value"]'); + return ( + marker?.textContent === expectedMarker && + gaugeValue?.textContent?.trim() === '73' + ); + }, + markerText, + { timeout: 20000 } ); } catch (e) { - // Dump debug info on failure const hasMarker = await page.locator('[data-testid="hmr-marker"]').count(); const bodyText = await page.textContent('body'); log( @@ -127,18 +114,19 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { throw e; } - // Verify the HMR marker appeared - const markerText = await page.locator('[data-testid="hmr-marker"]').textContent(); - expect(markerText).toBe('HMR-OK'); + const markerValue = await page.locator('[data-testid="hmr-marker"]').textContent(); + expect(markerValue).toBe(markerText); - // Verify counter state is preserved (not reset to initial value 70) - // If HMR works correctly, the counter should still show 73 - // If a full-reload happened, it would reset to the initial value of 70 const bodyText = await page.textContent('body'); expect(bodyText).toContain('73'); log('HMR state preservation verified: content updated, counter state preserved'); } finally { + try { + writeFileSync(counterComponentPath, originalCounterContent); + } catch (e) { + log(`Error restoring counter.tsx: ${e.message}`); + } await browser.close(); try { await promisifiedTreeKill(p.pid!, 'SIGKILL'); @@ -154,7 +142,6 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { const host = `http://localhost:${SERVE_PORT}/`; await assertHostUnused(host); - // First build the app const buildProcess = await runCommandUntil(`npm run build`, global.tmpDir, (output) => { return output.includes('dist/build') || output.includes('built in'); }); @@ -165,7 +152,6 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { log(`Error terminating build process: ${e.message}`); } - // Now run the preview const p = await runCommandUntil( `npm run preview -- --no-open --port ${SERVE_PORT}`, global.tmpDir, @@ -176,13 +162,12 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { assert.equal(existsSync(global.tmpDir), true); - // Wait a bit for the server to fully start - await new Promise((resolve) => setTimeout(resolve, 2000)); - - const res = await fetch(host, { headers: { accept: 'text/html' } }).then((r) => r.text()); - console.log('** res', res); + const res = await waitForHttpText( + host, + (html) => (type === 'playground' ? html.includes('fantastic') : html.includes('Hi')), + 20000 + ); - // Check for the appropriate content based on template type if (type === 'playground') { expect(res).toContain('fantastic'); } else if (type === 'empty') { @@ -199,22 +184,75 @@ for (const type of ['empty', 'playground'] as QwikProjectType[]) { }); } -async function expectHtmlOnARootPage(host: string) { - expect((await getPageHtml(host)).querySelector('.container h1')?.textContent).toBe( - `So fantasticto have you here` +function withHmrMarker(counterContent: string, markerText: string) { + return counterContent.replace( + ``, + `${markerText}` ); - const heroComponentPath = join(global.tmpDir, `src/components/starter/hero/hero.tsx`); - const heroComponentTextContent = readFileSync(heroComponentPath, 'utf-8'); - writeFileSync( - heroComponentPath, - heroComponentTextContent.replace( - `to have you here`, - `to have e2e tests here` - ) - ); - // wait for the arbitrary amount of time before the app is reloaded - await new Promise((r) => setTimeout(r, 2000)); - expect((await getPageHtml(host)).querySelector('.container h1')?.textContent).toBe( - `So fantasticto have e2e tests here` +} + +async function clickUntilCounterReaches( + page: playwright.Page, + plusBtn: playwright.Locator, + expectedValue: number, + maxClicks: number, + timeoutMs: number +) { + const expectedText = String(expectedValue); + const deadline = Date.now() + timeoutMs; + for (let i = 0; i < maxClicks; i++) { + await plusBtn.click(); + try { + await page.waitForFunction( + (text) => { + const gaugeValue = document.querySelector('._value_1v6hy_9, [class*="value"]'); + return gaugeValue?.textContent?.trim() === text; + }, + expectedText, + { timeout: Math.max(Math.min(deadline - Date.now(), 2000), 1) } + ); + return; + } catch { + // ignore + } + } + await page.waitForFunction( + (text) => { + const gaugeValue = document.querySelector('._value_1v6hy_9, [class*="value"]'); + return gaugeValue?.textContent?.trim() === text; + }, + expectedText, + { timeout: Math.max(deadline - Date.now(), 1) } ); } + +async function waitForHttpText( + host: string, + matcher: (html: string) => boolean, + timeoutMs: number +): Promise { + let lastHtml = ''; + await waitFor(async () => { + const html = await fetch(host, { headers: { accept: 'text/html' } }).then((r) => r.text()); + lastHtml = html; + return matcher(html); + }, timeoutMs); + return lastHtml; +} + +async function waitFor( + callback: () => Promise, + timeoutMs: number, + intervalMs = 250 +): Promise { + const timeoutAt = Date.now() + timeoutMs; + while (Date.now() < timeoutAt) { + try { + if (await callback()) { + return; + } + } catch {} + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`Timed out after ${timeoutMs}ms`); +} diff --git a/package.json b/package.json index 6a5c6d1f87f..53d9d8bec8e 100644 --- a/package.json +++ b/package.json @@ -248,6 +248,12 @@ "test.rust": "make test", "test.rust.bench": "make benchmark", "test.rust.update": "make test-update", + "test.bench": "pnpm test.bench.timing && pnpm test.bench.memory", + "test.bench.memory": "node --expose-gc --require ./scripts/runBefore.ts scripts/validate-benchmark-memory.ts", + "test.bench.memory.update": "node --expose-gc --require ./scripts/runBefore.ts scripts/validate-benchmark-memory.ts --update", + "test.bench.timing": "node --require ./scripts/runBefore.ts scripts/validate-benchmarks.ts", + "test.bench.timing.update": "node --require ./scripts/runBefore.ts scripts/validate-benchmarks.ts --update", + "test.bench.update": "pnpm test.bench.timing.update && pnpm test.bench.memory.update", "test.unit": "vitest packages", "test.unit.debug": "vitest --inspect-brk packages", "test.vite": "playwright test e2e/qwik-e2e/tests/qwikrouter --browser=chromium --config e2e/qwik-e2e/playwright.config.ts", diff --git a/packages/qwik/src/core/bench/baseline.ts b/packages/qwik/src/core/bench/baseline.ts new file mode 100644 index 00000000000..5a303f17c57 --- /dev/null +++ b/packages/qwik/src/core/bench/baseline.ts @@ -0,0 +1,64 @@ +export const BASELINE_UNITS = 400; + +const MEMORY_SLOTS = 512; + +type MemCell = { a: number; b: number; c: number; d: number }; + +/** + * Shared workload used to normalize benchmark results across machines. It mixes branchy integer + * arithmetic with deterministic memory churn and returns a checksum to prevent dead-code + * elimination. + */ +export const sharedBaselineWorkload = (units = BASELINE_UNITS): number => { + let acc = 0x9e3779b9; + const ring = new Uint32Array(MEMORY_SLOTS); + const cells: MemCell[] = new Array(MEMORY_SLOTS); + + for (let i = 0; i < MEMORY_SLOTS; i++) { + cells[i] = { a: i, b: i << 1, c: i << 2, d: i << 3 }; + } + + for (let unit = 0; unit < units; unit++) { + for (let i = 0; i < MEMORY_SLOTS; i++) { + acc = (acc ^ (i + unit * 131)) >>> 0; + acc = Math.imul(acc, 2654435761) >>> 0; + acc ^= acc >>> ((i & 7) + 1); + + const idx = (acc + i * 17) & (MEMORY_SLOTS - 1); + ring[idx] = (ring[idx] + acc + i) >>> 0; + + const cell = cells[idx]; + cell.a = (cell.a + (ring[idx] & 0xff)) >>> 0; + cell.b = (cell.b ^ ((ring[idx] >>> 8) & 0xff)) >>> 0; + cell.c = (cell.c + cell.a + unit) >>> 0; + cell.d = (cell.d ^ cell.c ^ acc) >>> 0; + + // Keep unpredictable branches so this does not simplify into a fast path. + if ((cell.d & 3) === 0) { + acc ^= cell.b; + } else if ((cell.d & 3) === 1) { + acc = (acc + cell.c) >>> 0; + } else { + acc = Math.imul(acc ^ cell.a, 2246822519) >>> 0; + } + } + } + + return (acc ^ ring[acc & (MEMORY_SLOTS - 1)]) >>> 0; +}; + +export const formatRatio = (value: number): string => { + const step = value < 10 ? 5 : value < 100 ? 25 : value < 1000 ? 250 : 2000; + const lower = Math.floor(value / step) * step; + const upper = lower + step; + return `${lower}-${upper}x`; +}; + +export const median = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return (sorted[middle - 1] + sorted[middle]) / 2; + } + return sorted[middle]; +}; diff --git a/packages/qwik/src/core/bench/bench-memory-results.json b/packages/qwik/src/core/bench/bench-memory-results.json new file mode 100644 index 00000000000..5b86fb658a1 --- /dev/null +++ b/packages/qwik/src/core/bench/bench-memory-results.json @@ -0,0 +1,88 @@ +{ + "version": 1, + "generatedAt": "2026-03-19T14:47:29.774Z", + "sampleCount": 3, + "countPerScenario": 10000, + "scenarios": { + "function": { + "totalBytes": 1127464, + "bytesPerInstance": 112.7464 + }, + "qrl": { + "totalBytes": 2988184, + "bytesPerInstance": 298.8184 + }, + "qrl-copy": { + "totalBytes": 2021600, + "bytesPerInstance": 202.16 + }, + "lazy-ref": { + "totalBytes": 1051984, + "bytesPerInstance": 105.1984 + }, + "qrl-class": { + "totalBytes": 653104, + "bytesPerInstance": 65.3104 + }, + "signal": { + "totalBytes": 660304, + "bytesPerInstance": 66.0304 + }, + "ComputedSignal": { + "totalBytes": 901016, + "bytesPerInstance": 90.1016 + }, + "wrapped-signal": { + "totalBytes": 1460416, + "bytesPerInstance": 146.0416 + }, + "virtual-vnode": { + "totalBytes": 1375648, + "bytesPerInstance": 137.5648 + }, + "element-vnode": { + "totalBytes": 7289544, + "bytesPerInstance": 728.9544 + }, + "text-vnode": { + "totalBytes": 2542032, + "bytesPerInstance": 254.2032 + }, + "jsx-node": { + "totalBytes": 2815064, + "bytesPerInstance": 281.5064 + }, + "context-id": { + "totalBytes": 736696, + "bytesPerInstance": 73.6696 + }, + "subscription-data": { + "totalBytes": 929240, + "bytesPerInstance": 92.924 + }, + "effect-subscription": { + "totalBytes": 4823856, + "bytesPerInstance": 482.3856 + }, + "delete-operation": { + "totalBytes": 1663456, + "bytesPerInstance": 166.3456 + }, + "set-text-operation": { + "totalBytes": 1744488, + "bytesPerInstance": 174.4488 + }, + "insert-or-move-operation": { + "totalBytes": 6323104, + "bytesPerInstance": 632.3104 + }, + "set-attribute-operation": { + "totalBytes": 6722376, + "bytesPerInstance": 672.2376 + }, + "remove-all-children-operation": { + "totalBytes": 6158744, + "bytesPerInstance": 615.8744 + } + } +} diff --git a/packages/qwik/src/core/bench/bench-results.json b/packages/qwik/src/core/bench/bench-results.json new file mode 100644 index 00000000000..b13065ff378 --- /dev/null +++ b/packages/qwik/src/core/bench/bench-results.json @@ -0,0 +1,90 @@ +{ + "version": 1, + "generatedAt": "2026-03-19T07:49:32.878Z", + "benchmarks": { + "baseline.shared-workload": { + "mean": 3.088664493055571, + "median": 2.9838345000002846, + "p75": 3.1992929999996704, + "p99": 4.196222999999918, + "p995": 4.295756000000438, + "p999": 4.766966000000139, + "rme": 0.6081320295006954, + "sampleCount": 1296 + }, + "current.dom-table-10": { + "mean": 2.6569924736609103, + "median": 2.4209310000005644, + "p75": 2.6598269999994955, + "p99": 4.945772000002762, + "p995": 5.941971000000194, + "p999": 7.454273999999714, + "rme": 0.9972462077872076, + "sampleCount": 2259, + "factor": 0.8113489538378665 + }, + "current.dom-table-1k": { + "mean": 226.27416044444504, + "median": 223.8181110000005, + "p75": 228.70898399999714, + "p99": 269.1425390000004, + "p995": 269.1425390000004, + "p999": 269.1425390000004, + "rme": 1.7506185315204819, + "sampleCount": 27, + "factor": 75.01022962231288 + }, + "current.dom-update-table-1k": { + "mean": 0.16147774939449824, + "median": 0.14038699999946402, + "p75": 0.15433199999824865, + "p99": 0.33831800000189105, + "p995": 0.4541299999982584, + "p999": 3.4702279999983148, + "rme": 1.8118797821851593, + "sampleCount": 37158, + "factor": 0.04704919123344496 + }, + "current.serialize-state-1k": { + "mean": 8.639306661870325, + "median": 7.590303000004496, + "p75": 9.659450999999535, + "p99": 19.368933999998262, + "p995": 22.706399999995483, + "p999": 26.309580000001006, + "rme": 2.7205570476483603, + "sampleCount": 695, + "factor": 2.543808311085542 + }, + "current.ssr-table-10": { + "mean": 3.6770456717697337, + "median": 3.153889999999592, + "p75": 4.103724000000511, + "p99": 8.557779999999184, + "p995": 10.160756999999649, + "p999": 30.789652999999817, + "rme": 2.2655355423081396, + "sampleCount": 1633, + "factor": 1.0569922695106888 + }, + "current.ssr-table-1k": { + "mean": 348.1837142222226, + "median": 318.59908899999937, + "p75": 353.7774320000026, + "p99": 548.3319370000027, + "p995": 548.3319370000027, + "p999": 548.3319370000027, + "rme": 12.380673917171574, + "sampleCount": 18, + "factor": 106.77505371024063 + } + }, + "sizes": { + "dom-table-10": 172, + "dom-table-1k": 18553, + "dom-update-table-1k": 26522, + "serialize-state-1k": 96844, + "ssr-table-10": 2175, + "ssr-table-1k": 177401 + } +} diff --git a/packages/qwik/src/core/bench/core.bench.ts b/packages/qwik/src/core/bench/core.bench.ts new file mode 100644 index 00000000000..f812ca544da --- /dev/null +++ b/packages/qwik/src/core/bench/core.bench.ts @@ -0,0 +1,34 @@ +import { bench, describe } from 'vitest'; +import { sharedBaselineWorkload } from './baseline'; +import { scenarios } from './scenarios'; + +const SIZE_LOG_PREFIX = 'QWIK_BENCH_SIZE'; + +describe('qwik core relative benchmarks', () => { + bench( + 'baseline.shared-workload', + async () => { + sharedBaselineWorkload(); + }, + { warmupTime: 500, time: 4000 } + ); + + for (const scenario of scenarios) { + let lastSize: number | null = null; + bench( + `current.${scenario.id}`, + async () => { + const size = await scenario.run(); + if (lastSize === null) { + lastSize = size; + process.stderr.write(`${SIZE_LOG_PREFIX}\t${scenario.id}\t${size}\n`); + } else if (lastSize !== size) { + throw new Error( + `Scenario ${scenario.id} returned inconsistent sizes: ${lastSize} vs ${size}` + ); + } + }, + { warmupTime: 500, time: 6000 } + ); + } +}); diff --git a/packages/qwik/src/core/bench/memory-scenarios.ts b/packages/qwik/src/core/bench/memory-scenarios.ts new file mode 100644 index 00000000000..ac3048a7cde --- /dev/null +++ b/packages/qwik/src/core/bench/memory-scenarios.ts @@ -0,0 +1,357 @@ +import { + _fnSignal, + createComputedQrl, + createContextId, + createSignal, + inlinedQrl, +} from '@qwik.dev/core'; +import { SubscriptionData } from '../reactive-primitives/subscription-data'; +import { EffectProperty, EffectSubscription } from '../reactive-primitives/types'; +import { JSXNodeImpl } from '../shared/jsx/jsx-node'; +import { LazyRef, QRLClass } from '../shared/qrl/qrl-class'; +import { _createQRL as createQRL } from '@qwik.dev/core/internal'; +import { ElementVNode } from '../shared/vnode/element-vnode'; +import { TextVNode } from '../shared/vnode/text-vnode'; +import { VirtualVNode } from '../shared/vnode/virtual-vnode'; +import { + createDeleteOperation, + createInsertOrMoveOperation, + createRemoveAllChildrenOperation, + createSetAttributeOperation, + createSetTextOperation, +} from '../shared/vnode/types/dom-vnode-operation'; +import { createDocument } from '../../testing/document'; + +export interface MemoryBenchmarkScenario { + id: string; + title: string; + count: number; + allocate: () => unknown[]; +} + +export const memoryScenarios: MemoryBenchmarkScenario[] = []; + +const INSTANCE_COUNT = 10_000; +const VNODE_FLAGS_INDEX_SHIFT = 12; +const VIRTUAL_VNODE_FLAGS = 0b000_000000010 | (-1 << VNODE_FLAGS_INDEX_SHIFT); +const TEXT_VNODE_FLAGS = 0b000_000000100 | 0b000_000001000 | (-1 << VNODE_FLAGS_INDEX_SHIFT); + +memoryScenarios.push({ + id: 'function', + title: 'function instances', + count: INSTANCE_COUNT, + allocate: () => { + const fns = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + fns[i] = function fn() { + return i; + }; + } + return fns; + }, +}); + +memoryScenarios.push({ + id: 'qrl', + title: 'QRL instances', + count: INSTANCE_COUNT, + allocate: () => { + const qrls = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + qrls[i] = createQRL('chunk123', `qrl_${i}`, null, null, null); + } + return qrls; + }, +}); + +memoryScenarios.push({ + id: 'qrl-copy', + title: 'copied QRL instances', + count: INSTANCE_COUNT, + allocate: () => { + const baseQrl = createQRL('chunk123', 'qrl_copy_base', null, null, null); + const captures: unknown[] = []; + const copies = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + copies[i] = baseQrl.w(captures); + } + return copies; + }, +}); + +memoryScenarios.push({ + id: 'lazy-ref', + title: 'LazyRef instances', + count: INSTANCE_COUNT, + allocate: () => { + const copies = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + copies[i] = new LazyRef('chunk123', `qrl_${i}`, null); + } + return copies; + }, +}); + +memoryScenarios.push({ + id: 'qrl-class', + title: 'QRL class instances', + count: INSTANCE_COUNT, + allocate: () => { + const lazyRef = new LazyRef('chunk123', 'qrl_class_base', null); + const copies = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + copies[i] = new QRLClass(lazyRef); + } + return copies; + }, +}); + +memoryScenarios.push({ + id: 'signal', + title: 'Signal instances', + count: INSTANCE_COUNT, + allocate: () => { + const signals = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + signals[i] = createSignal(i); + } + return signals; + }, +}); + +const qrl = inlinedQrl(() => {}, 'myQrl'); +memoryScenarios.push({ + id: 'ComputedSignal', + title: 'ComputedSignal instances', + count: INSTANCE_COUNT, + allocate: () => { + const signals = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + signals[i] = createComputedQrl(qrl); + } + return signals; + }, +}); + +const noopFN = () => {}; +memoryScenarios.push({ + id: 'wrapped-signal', + title: 'Wrapped signal instances', + count: INSTANCE_COUNT, + allocate: () => { + const signals = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + signals[i] = _fnSignal(noopFN, []); + } + return signals; + }, +}); + +memoryScenarios.push({ + id: 'virtual-vnode', + title: 'Virtual VNode instances', + count: INSTANCE_COUNT, + allocate: () => { + const vnodes = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + vnodes[i] = new VirtualVNode(null, VIRTUAL_VNODE_FLAGS, null, null, null, null, null, null); + } + return vnodes; + }, +}); + +memoryScenarios.push({ + id: 'element-vnode', + title: 'Element VNode instances', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const vnodes = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + vnodes[i] = new ElementVNode( + null, + VIRTUAL_VNODE_FLAGS, + null, + null, + null, + null, + null, + null, + document.createElement('div'), + 'div' + ); + } + return vnodes; + }, +}); + +memoryScenarios.push({ + id: 'text-vnode', + title: 'Text VNode instances', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const vnodes = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + const text = `text-${i}`; + vnodes[i] = new TextVNode( + TEXT_VNODE_FLAGS, + null, + null, + null, + null, + document.createTextNode(text), + text + ); + } + return vnodes; + }, +}); + +memoryScenarios.push({ + id: 'jsx-node', + title: 'JSX node instances', + count: INSTANCE_COUNT, + allocate: () => { + const nodes = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + nodes[i] = new JSXNodeImpl( + 'div', + { class: `cls-${i}` }, + { id: `node-${i}` }, + `child-${i}`, + 0 as never, + i, + false + ); + } + return nodes; + }, +}); + +memoryScenarios.push({ + id: 'context-id', + title: 'context IDs', + count: INSTANCE_COUNT, + allocate: () => { + const contexts = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + contexts[i] = createContextId(`context-${i}`); + } + return contexts; + }, +}); + +memoryScenarios.push({ + id: 'subscription-data', + title: 'Subscription data instances', + count: INSTANCE_COUNT, + allocate: () => { + const subscriptions = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + subscriptions[i] = new SubscriptionData({ + $scopedStyleIdPrefix$: i % 2 === 0 ? null : `s${i}`, + $isConst$: (i & 1) === 0, + }); + } + return subscriptions; + }, +}); + +memoryScenarios.push({ + id: 'effect-subscription', + title: 'Effect subscription instances', + count: INSTANCE_COUNT, + allocate: () => { + const subscriptions = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + subscriptions[i] = new EffectSubscription( + createSignal(i) as never, + EffectProperty.VNODE, + new Set([createSignal(i) as never]), + new SubscriptionData({ + $scopedStyleIdPrefix$: null, + $isConst$: false, + }) + ); + } + return subscriptions; + }, +}); + +memoryScenarios.push({ + id: 'delete-operation', + title: 'Delete operations', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const operations = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + operations[i] = createDeleteOperation(document.createTextNode(`delete-${i}`)); + } + return operations; + }, +}); + +memoryScenarios.push({ + id: 'set-text-operation', + title: 'set-text operations', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const operations = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + operations[i] = createSetTextOperation(document.createTextNode(''), `text-${i}`); + } + return operations; + }, +}); + +memoryScenarios.push({ + id: 'insert-or-move-operation', + title: 'insert-or-move operations', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const parent = document.createElement('div'); + const operations = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + operations[i] = createInsertOrMoveOperation(document.createElement('span'), parent, null); + } + return operations; + }, +}); + +memoryScenarios.push({ + id: 'set-attribute-operation', + title: 'set-attribute operations', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const operations = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + operations[i] = createSetAttributeOperation( + document.createElement('div'), + 'data-id', + `${i}`, + i % 2 === 0 ? null : 'scoped', + false + ); + } + return operations; + }, +}); + +memoryScenarios.push({ + id: 'remove-all-children-operation', + title: 'remove-all-children operations', + count: INSTANCE_COUNT, + allocate: () => { + const document = createDocument(); + const operations = new Array(INSTANCE_COUNT); + for (let i = 0; i < INSTANCE_COUNT; i++) { + operations[i] = createRemoveAllChildrenOperation(document.createElement('div')); + } + return operations; + }, +}); diff --git a/packages/qwik/src/core/bench/scenarios.tsx b/packages/qwik/src/core/bench/scenarios.tsx new file mode 100644 index 00000000000..08b79640bca --- /dev/null +++ b/packages/qwik/src/core/bench/scenarios.tsx @@ -0,0 +1,188 @@ +import { component$, createSignal, type JSXOutput } from '@qwik.dev/core'; +import { getDomContainer } from '../client/dom-container'; +import { render } from '../client/dom-render'; +import { _serialize } from '../shared/serdes/serdes.public'; +import { renderToString } from '../../server/ssr-render'; +import { createDocument } from '../../testing/document'; +import { waitForDrain } from '../../testing/util'; + +export interface BenchmarkScenario { + id: string; + title: string; + /** Returns the size of the result (e.g. bytes of HTML), or 0 if not applicable */ + run: () => Promise; +} + +type TableRow = { + id: number; + label: string; + value: number; +}; + +const makeRows = (count: number): TableRow[] => { + let state = 0x1234abcd; + const rows = new Array(count); + for (let i = 0; i < count; i++) { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + rows[i] = { + id: i + 1, + label: `row-${i}-${state & 0xffff}`, + value: state & 0x3ff, + }; + } + return rows; +}; + +const makeUpdatedRows = (rows: TableRow[]): TableRow[] => { + const nextRows = rows.map((row, index) => ({ + id: row.id, + label: `${row.label}-next-${(index * 13 + row.value) % 17}`, + value: (row.value * 7 + index * 11) % 2048, + })); + + if (nextRows.length > 4) { + const reordered = [...nextRows]; + const moved = reordered.splice(1, 3); + reordered.splice(reordered.length - 1, 0, ...moved); + return reordered; + } + + return nextRows.reverse(); +}; + +const rows10 = makeRows(10); +const rows1000 = makeRows(1000); +const updatedRows1000 = makeUpdatedRows(rows1000); + +const sharedMeta = { + adjectives: ['pretty', 'large', 'small', 'helpful'], + colours: ['red', 'green', 'blue', 'black'], + nouns: ['table', 'chair', 'keyboard', 'mouse'], +}; + +const makeSerializableState = (count: number) => { + const items = new Array(count); + for (let i = 0; i < count; i++) { + items[i] = { + id: i + 1, + label: `item-${i}`, + flags: [i % 2 === 0, i % 3 === 0, i % 5 === 0], + meta: sharedMeta, + nested: { + score: (i * 17) % 97, + tags: [`tag-${i % 7}`, `tag-${i % 11}`], + }, + }; + } + + return { + items, + summary: { + count, + first: items[0], + last: items[items.length - 1], + }, + }; +}; + +const serializableState1k = makeSerializableState(1000); + +const renderTable = (rows: TableRow[]): JSXOutput => { + return ( + + + {rows.map((row) => ( + console.warn('hi', row.id)}> + + + + + ))} + +
{row.id}{row.label}{row.value}
+ ); +}; + +const estimateTableSize = (rows: TableRow[]): number => { + let size = 24; + for (const row of rows) { + size += String(row.id).length; + size += row.label.length; + size += String(row.value).length; + } + return size; +}; + +const renderSsr = async (jsx: JSXOutput): Promise => { + const result = await renderToString(jsx, { qwikLoader: 'never', containerTagName: 'div' }); + return result.html.length; +}; + +const renderDom = async (jsx: JSXOutput): Promise => { + const document = createDocument(); + await render(document.body, jsx); + return 0; +}; + +const renderDomUpdate = async (initialRows: TableRow[], nextRows: TableRow[]): Promise => { + const document = createDocument(); + const rows = createSignal(initialRows); + const App = component$(() => { + return renderTable(rows.value); + }); + await render(document.body, ); + rows.value = nextRows; + await waitForDrain(getDomContainer(document.body)); + return 0; +}; + +const makeScenario = (id: string, rowCount: number, rows: TableRow[]): BenchmarkScenario => { + return { + id, + title: `SSR table ${rowCount} rows`, + run: () => renderSsr(renderTable(rows)), + }; +}; + +const makeDomScenario = (id: string, rowCount: number, rows: TableRow[]): BenchmarkScenario => { + return { + id, + title: `DOM table ${rowCount} rows`, + run: async () => { + await renderDom(renderTable(rows)); + return estimateTableSize(rows); + }, + }; +}; + +const makeDomUpdateScenario = ( + id: string, + rowCount: number, + initialRows: TableRow[], + nextRows: TableRow[] +): BenchmarkScenario => { + return { + id, + title: `DOM update table ${rowCount} rows`, + run: async () => { + await renderDomUpdate(initialRows, nextRows); + return estimateTableSize(nextRows); + }, + }; +}; + +export const scenarios: BenchmarkScenario[] = [ + makeScenario('ssr-table-10', 10, rows10), + makeScenario('ssr-table-1k', 1000, rows1000), + makeDomScenario('dom-table-10', 10, rows10), + makeDomScenario('dom-table-1k', 1000, rows1000), + makeDomUpdateScenario('dom-update-table-1k', 1000, rows1000, updatedRows1000), + { + id: 'serialize-state-1k', + title: 'Serialize 1k-item state graph', + run: async () => { + const result = await _serialize(serializableState1k); + return result.length; + }, + }, +]; diff --git a/packages/qwik/src/core/internal.ts b/packages/qwik/src/core/internal.ts index 7c1fafc86be..aed071f342c 100644 --- a/packages/qwik/src/core/internal.ts +++ b/packages/qwik/src/core/internal.ts @@ -37,6 +37,7 @@ export type { VNode as _VNode } from './shared/vnode/vnode'; export { _EFFECT_BACK_REF } from './reactive-primitives/backref'; export { _hasStoreEffects, isStore as _isStore } from './reactive-primitives/impl/store'; +export { isSignal } from './reactive-primitives/utils'; export { _wrapProp, _wrapSignal } from './reactive-primitives/internal-api'; export { SubscriptionData as _SubscriptionData } from './reactive-primitives/subscription-data'; export { diff --git a/packages/qwik/src/core/shared/qrl/qrl-class.ts b/packages/qwik/src/core/shared/qrl/qrl-class.ts index 54c4246b2b5..8eef2b3f96b 100644 --- a/packages/qwik/src/core/shared/qrl/qrl-class.ts +++ b/packages/qwik/src/core/shared/qrl/qrl-class.ts @@ -167,31 +167,26 @@ export class LazyRef { } } +const QRL_STATE = Symbol('qrl-state'); + +type QRLCallable = QRLInternal & { + [QRL_STATE]: QRLClass; +}; + /** - * When a method is called on the qrlFn wrapper function, `this` is the function, not the QRLClass - * instance that holds the data. This helper returns the actual instance by checking whether `this` - * owns `resolved` (always set on the instance). + * QRL methods may run with `this` set either to the callable wrapper or directly to the backing + * state object. This helper normalizes both cases to the shared backing state. */ const getInstance = (instance: any): QRLClass => { - return Object.prototype.hasOwnProperty.call(instance, 'resolved') - ? instance - : (Object.getPrototypeOf(instance) as QRLClass); + return (instance?.[QRL_STATE] as QRLClass | undefined) ?? instance; }; /** - * We use a class here to avoid copying all the methods for every QRL instance. The QRL itself is a - * function that calls the internal $callFn$ method, and we set the prototype to the class instance - * so it has access to all the properties and methods. That's why we need to extend Function, so - * that `.apply()` etc work. - * - * So a QRL is a function that has a prototype of a QRLClass instance. This is unconventional, but - * it allows us to have a callable QRL that is also a class. - * - * Note the use of getInstance everywhere when writing to `this`. If you write to `this` directly, - * it will be stored on the function itself, and we don't want that because the QRLClass instance - * doesn't have access to it, and it uses more memory. + * QRL state lives in a plain object. The callable wrapper stores that state under a symbol and uses + * a shared prototype derived from Function.prototype for methods/getters. This keeps QRLs callable + * without using a unique state object as each function's prototype. */ -class QRLClass extends Function implements QRLInternalMethods { +export class QRLClass { resolved: undefined | TYPE = undefined; // This is defined or undefined for the lifetime of the QRL, so we set it lazily $captures$?: Readonly | string | null; @@ -202,7 +197,6 @@ class QRLClass extends Function implements QRLInternalMethods { $captures$?: Readonly | string | null, container?: Container | null ) { - super(); if ($captures$) { this.$captures$ = $captures$; if (typeof $captures$ === 'string') { @@ -221,100 +215,176 @@ class QRLClass extends Function implements QRLInternalMethods { // If it is plain value with deserialized or missing captures, resolve it immediately // Otherwise we keep using the async path so we can wait for qrls to load if ($lazy$.$ref$ != null && typeof this.$captures$ !== 'string' && !isPromise($lazy$.$ref$)) { - // we can pass this instead of using getInstance because we know we are not the qrlFn this.resolved = bindCaptures(this, $lazy$.$ref$ as TYPE); } } +} - w(captures: Readonly | string | null): QRLInternal { - const newQrl = new QRLClass( - this.$lazy$, - captures!, - this.$captures$ ? this.$container$ : undefined - ); - return makeQrlFn(newQrl); +const qrlCallFn = function ( + this: QRLClass | QRLCallable, + withThis: unknown, + ...args: QrlArgs +): ValueOrPromise> { + const qrl = getInstance(this); + if (qrl.resolved) { + return (qrl.resolved as any).apply(withThis, args); } - s(ref: ValueOrPromise) { - const qrl = getInstance(this); - qrl.$lazy$.$setRef$(ref); - qrl.resolved = bindCaptures(qrl, ref as TYPE); - } + // Not resolved yet: we'll return a promise - // --- Getter proxies for backward compat --- - get $chunk$(): string | null { - return this.$lazy$.$chunk$; - } - get $symbol$(): string { - return this.$lazy$.$symbol$; - } - get $hash$(): string { - return this.$lazy$.$hash$; - } - get dev(): QRLDev | null | undefined { - return this.$lazy$.dev; - } + // grab the context while we are sync + const ctx = tryGetInvokeContext(); - $callFn$(withThis: unknown, ...args: QrlArgs): ValueOrPromise> { - if (this.resolved) { - return (this.resolved as any).apply(withThis, args); - } + return qrlResolve + .call(qrl, ctx?.$container$) + .then(() => invokeApply.call(withThis, ctx, qrl.resolved as any, args)); +}; - // Not resolved yet: we'll return a promise +const qrlWithCaptures = function ( + this: QRLClass | QRLCallable, + captures: Readonly | string | null +): QRLInternal { + const qrl = getInstance(this); + const newQrl = new QRLClass( + qrl.$lazy$, + captures!, + qrl.$captures$ ? qrl.$container$ : undefined + ); + return makeQrlFn(newQrl); +}; - // grab the context while we are sync - const ctx = tryGetInvokeContext(); +const qrlSetRef = function ( + this: QRLClass | QRLCallable, + ref: ValueOrPromise +) { + const qrl = getInstance(this); + qrl.$lazy$.$setRef$(ref); + qrl.resolved = bindCaptures(qrl, ref as TYPE); +}; - return this.resolve(ctx?.$container$).then(() => - invokeApply.call(withThis, ctx, this.resolved as any, args) - ); - } +const qrlResolve = async function ( + this: QRLClass | QRLCallable, + container?: Container +): Promise { + const qrl = getInstance(this); + return maybeThen($resolve$(qrl, container), () => qrl.resolved!); +}; - async resolve(container?: Container): Promise { - // We need to write to the QRLClass instance, not the function - const qrl = getInstance(this); - return maybeThen($resolve$(qrl, container), () => qrl.resolved!); - } +const qrlGetSymbol = function (this: QRLClass | QRLCallable): string { + return getInstance(this).$lazy$.$symbol$; +}; - getSymbol(): string { - return this.$symbol$; - } +const qrlGetHash = function (this: QRLClass | QRLCallable): string { + return getInstance(this).$lazy$.$hash$; +}; - getHash(): string { - return this.$hash$; - } +const qrlGetCaptured = function (this: QRLClass | QRLCallable): unknown[] | null { + const qrl = getInstance(this); + ensureQrlCaptures(qrl); + return qrl.$captures$ as unknown[] | null; +}; - getCaptured(): unknown[] | null { - const qrl = getInstance(this); - ensureQrlCaptures(qrl); - return qrl.$captures$ as unknown[] | null; - } +const qrlGetFn = function ( + this: QRLClass | QRLCallable, + currentCtx?: InvokeContext, + beforeFn?: () => void | false +): TYPE extends (...args: any) => any + ? (...args: Parameters) => ValueOrPromise | undefined> + : // unknown instead of never so we allow assigning function QRLs to any + unknown { + const qrl = getInstance(this); + const bound = (...args: QrlArgs): unknown => { + if (!qrl.resolved) { + return qrlResolve.call(qrl).then((fn) => { + if (qDev && !isFunction(fn)) { + throw qError(QError.qrlIsNotFunction); + } + return bound(...args); + }); + } + if (beforeFn && beforeFn() === false) { + return undefined; + } + return invokeApply(currentCtx, qrl.resolved as any, args); + }; + return bound as any; +}; - getFn( - currentCtx?: InvokeContext, - beforeFn?: () => void | false - ): TYPE extends (...args: any) => any - ? (...args: Parameters) => ValueOrPromise | undefined> - : // unknown instead of never so we allow assigning function QRLs to any - unknown { - const qrl = getInstance(this); - const bound = (...args: QrlArgs): unknown => { - if (!qrl.resolved) { - return qrl.resolve().then((fn) => { - if (qDev && !isFunction(fn)) { - throw qError(QError.qrlIsNotFunction); - } - return bound(...args); - }); - } - if (beforeFn && beforeFn() === false) { - return undefined; - } - return invokeApply(currentCtx, qrl.resolved as any, args); - }; - return bound as any; - } -} +const QRL_FUNCTION_PROTO: QRLInternalMethods = Object.create(Function.prototype, { + resolved: { + get(this: QRLCallable) { + return this[QRL_STATE].resolved; + }, + set(this: QRLCallable, value: unknown) { + this[QRL_STATE].resolved = value; + }, + }, + $captures$: { + get(this: QRLCallable) { + return this[QRL_STATE].$captures$; + }, + set(this: QRLCallable, value: Readonly | string | null | undefined) { + this[QRL_STATE].$captures$ = value; + }, + }, + $container$: { + get(this: QRLCallable) { + return this[QRL_STATE].$container$; + }, + set(this: QRLCallable, value: Container | null | undefined) { + this[QRL_STATE].$container$ = value; + }, + }, + $lazy$: { + get(this: QRLCallable) { + return this[QRL_STATE].$lazy$; + }, + }, + $chunk$: { + get(this: QRLCallable) { + return this[QRL_STATE].$lazy$.$chunk$; + }, + }, + $symbol$: { + get(this: QRLCallable) { + return this[QRL_STATE].$lazy$.$symbol$; + }, + }, + $hash$: { + get(this: QRLCallable) { + return this[QRL_STATE].$lazy$.$hash$; + }, + }, + dev: { + get(this: QRLCallable) { + return this[QRL_STATE].$lazy$.dev; + }, + }, + $callFn$: { + value: qrlCallFn, + }, + w: { + value: qrlWithCaptures, + }, + s: { + value: qrlSetRef, + }, + resolve: { + value: qrlResolve, + }, + getSymbol: { + value: qrlGetSymbol, + }, + getHash: { + value: qrlGetHash, + }, + getCaptured: { + value: qrlGetCaptured, + }, + getFn: { + value: qrlGetFn, + }, +}); /** * The current captured scope during QRL invocation. This is used to provide the lexical scope for @@ -441,12 +511,12 @@ export const createQRL = ( }; const makeQrlFn = (qrl: QRLClass): QRLInternal => { - // The QRL has to be callable, so we create a function that calls the internal $callFn$ - const qrlFn: QRLInternal = async function (this: unknown, ...args: QrlArgs) { - return qrl.$callFn$(this, ...args); - } as QRLInternal; - // ...and set the prototype to the QRL instance so it has all the properties and methods without copying them - Object.setPrototypeOf(qrlFn, qrl); + // The QRL has to be callable, so we create a function and attach the per-instance state to it. + const qrlFn = async function (this: unknown, ...args: QrlArgs) { + return qrlCallFn.call(qrlFn as QRLCallable, this, ...args); + } as QRLCallable; + qrlFn[QRL_STATE] = qrl; + Object.setPrototypeOf(qrlFn, QRL_FUNCTION_PROTO); return qrlFn; }; diff --git a/packages/qwik/src/core/use/use-hmr.ts b/packages/qwik/src/core/use/use-hmr.ts index 81a3eee0485..0f7414d4dbd 100644 --- a/packages/qwik/src/core/use/use-hmr.ts +++ b/packages/qwik/src/core/use/use-hmr.ts @@ -34,9 +34,7 @@ export const _hmr = (event: Event, element: Element) => { // Maybe we should use a qrl registry to invalidate all QRLs from a parent? const qrl = container.getHostProp>>(host, OnRenderProp); if (qrl) { - // This code is highly coupled to the internal implementation of QRL - const instance = (qrl as any).__proto__ as typeof qrl; - const lazy = instance.$lazy$; + const lazy = qrl.$lazy$; const chunk = lazy.$chunk$!; if (chunk) { /** @@ -46,7 +44,7 @@ export const _hmr = (event: Event, element: Element) => { const bustUrl = chunk.split('?')[0] + '?t=' + Date.now(); (lazy as any).$chunk$ = bustUrl; lazy.$ref$ = undefined; - instance.resolved = undefined; + qrl.resolved = undefined; // Force rerender markVNodeDirty(container, host as VNode, ChoreBits.COMPONENT); } diff --git a/packages/qwik/src/optimizer/src/plugins/vite.ts b/packages/qwik/src/optimizer/src/plugins/vite.ts index 59d624b74c3..819ad3e0c23 100644 --- a/packages/qwik/src/optimizer/src/plugins/vite.ts +++ b/packages/qwik/src/optimizer/src/plugins/vite.ts @@ -139,7 +139,7 @@ export function qwikVite(qwikViteOpts: QwikVitePluginOptions = {}): any { target = 'lib'; } else if (viteConfig.build?.ssr || viteEnv.mode === 'ssr') { target = 'ssr'; - } else if (viteEnv.mode === 'test') { + } else if (viteEnv.mode === 'test' || viteEnv.mode === 'benchmark') { target = 'test'; } else { target = 'client'; @@ -638,7 +638,9 @@ export function qwikVite(qwikViteOpts: QwikVitePluginOptions = {}): any { if (hmrEnabled) { // Source files live in the SSR module graph. When they change, notify the // client's loaded QRL segments via the client environment's HMR channel. - const files = ctx.modules.map((m) => m.type === 'js' && m.url).filter(Boolean); + const files = ctx.modules + .map((m) => (m.type === 'js' ? m.url.split('?')[0] : null)) + .filter(Boolean); if (files.length > 0 && viteServer) { viteServer.environments.client.hot.send({ type: 'custom', diff --git a/packages/qwik/src/server/ssr-container.ts b/packages/qwik/src/server/ssr-container.ts index 11b453c798c..005c9697dcf 100644 --- a/packages/qwik/src/server/ssr-container.ts +++ b/packages/qwik/src/server/ssr-container.ts @@ -243,7 +243,7 @@ class SSRContainer extends _SharedContainer implements ISSRContainer { private componentStack: ISsrComponentFrame[] = []; private cleanupQueue: CleanupQueue = []; private emitContainerDataFrame: ElementFrame | null = null; - public $instanceHash$ = hash(); + public $instanceHash$ = randomStr(); // Temporary flag to find missing roots after the state was serialized private $noMoreRoots$ = false; private qlInclude: QwikLoaderInclude; @@ -1342,8 +1342,8 @@ function isSSRUnsafeAttr(name: string): boolean { return false; } -function hash() { - return Math.random().toString(36).slice(2); +function randomStr() { + return (Math.random().toString(36) + '000000').slice(2, 8); } function addPreventDefaultEventToSerializationContext( diff --git a/scripts/validate-benchmark-memory.ts b/scripts/validate-benchmark-memory.ts new file mode 100644 index 00000000000..42c35044890 --- /dev/null +++ b/scripts/validate-benchmark-memory.ts @@ -0,0 +1,423 @@ +import { execFile } from 'node:child_process'; +import { readFile, unlink, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; +import { build } from 'esbuild'; + +const RESULTS_PATH = 'packages/qwik/src/core/bench/bench-memory-results.json'; +const SCENARIOS_ENTRY = 'packages/qwik/src/core/bench/memory-scenarios.ts'; +const SAMPLE_COUNT = 3; +const COUNT_PER_SCENARIO = 10_000; +const VERSION = 1; +const RELATIVE_TOLERANCE = 0.01; +const GC_ROUNDS = 4; +const execFileAsync = promisify(execFile); +const CHILD_MEASURE_SOURCE = [ + "import { pathToFileURL } from 'node:url';", + `const GC_ROUNDS = ${GC_ROUNDS};`, + 'const [bundlePath, scenarioId, expectedCount] = process.argv.slice(1);', + "if (typeof global.gc !== 'function') throw new Error('Missing global.gc');", + 'const mod = await import(pathToFileURL(bundlePath).href);', + 'const scenario = mod.memoryScenarios.find((entry) => entry.id === scenarioId);', + 'if (!scenario) throw new Error(`Unknown memory benchmark scenario: ${scenarioId}`);', + 'const count = Number(expectedCount);', + 'if (scenario.count !== count) throw new Error(`Scenario ${scenario.id} expected count ${count}, got ${scenario.count}`);', + 'for (let i = 0; i < GC_ROUNDS; i++) global.gc();', + 'const baseline = process.memoryUsage().heapUsed;', + 'const retained = scenario.allocate();', + 'if (retained.length !== scenario.count) throw new Error(`Scenario ${scenario.id} allocated ${retained.length} entries, expected ${scenario.count}`);', + 'globalThis.__qwikMemoryRetention = retained;', + 'for (let i = 0; i < GC_ROUNDS; i++) global.gc();', + 'const totalBytes = Math.max(0, process.memoryUsage().heapUsed - baseline);', + 'process.stdout.write(`${JSON.stringify({ id: scenario.id, totalBytes, count: scenario.count })}\\n`);', + 'process.exit(0);', +].join('\n'); + +type StoredScenarioResult = { + totalBytes: number; + bytesPerInstance: number; +}; + +type StoredResults = { + version: number; + generatedAt: string; + sampleCount: number; + countPerScenario: number; + scenarios: Record; +}; + +type MeasurementResult = { + id: string; + totalBytes: number; + count: number; +}; + +type MemoryScenario = { + id: string; + title: string; + count: number; + allocate: () => unknown[]; +}; + +const args = new Set(process.argv.slice(2)); +const shouldUpdate = args.has('--update'); +const scenarioToMeasure = getArgValue('--measure'); +const bundlePathArg = getArgValue('--bundle'); + +async function main() { + requireGc(); + const ownBundlePath = bundlePathArg ? null : await bundleMemoryScenarios(); + const bundlePath = bundlePathArg || ownBundlePath!; + const scenarios = await loadMemoryScenarios(bundlePath); + + try { + if (args.has('--measure')) { + if (!scenarioToMeasure) { + throw new Error('Missing scenario id after --measure.'); + } + await runMeasurementWorker(scenarios, scenarioToMeasure); + return; + } + + const measuredResults = await measureAllScenarios(scenarios, bundlePath); + const measuredIds = Object.keys(measuredResults); + const expectedIds = scenarios.map((scenario) => scenario.id); + assertExactKeySet('measured memory scenarios', measuredIds, expectedIds); + + const storedPath = resolve(process.cwd(), RESULTS_PATH); + if (shouldUpdate) { + const next = buildStoredResults(scenarios, measuredResults); + await writeFile(storedPath, JSON.stringify(next, null, 2) + '\n', 'utf-8'); + console.log(`Updated memory benchmark baselines in ${RESULTS_PATH}`); + return; + } + + const stored = await readStoredResults(storedPath); + validateResults(scenarios, stored, measuredResults); + console.log('Memory benchmark validation passed.'); + } finally { + if (ownBundlePath) { + await unlink(ownBundlePath).catch(() => undefined); + } + } +} + +async function runMeasurementWorker(scenarios: MemoryScenario[], scenarioId: string) { + const scenario = scenarios.find((entry) => entry.id === scenarioId); + if (!scenario) { + throw new Error(`Unknown memory benchmark scenario: ${scenarioId}`); + } + if (scenario.count !== COUNT_PER_SCENARIO) { + throw new Error( + `Scenario ${scenario.id} expected count ${COUNT_PER_SCENARIO}, got ${scenario.count}` + ); + } + + forceGc(); + const baseline = process.memoryUsage().heapUsed; + const retained = scenario.allocate(); + if (retained.length !== scenario.count) { + throw new Error( + `Scenario ${scenario.id} allocated ${retained.length} entries, expected ${scenario.count}` + ); + } + + (globalThis as any).__qwikMemoryRetention = retained; + forceGc(); + const totalBytes = Math.max(0, process.memoryUsage().heapUsed - baseline); + + const result: MeasurementResult = { + id: scenario.id, + totalBytes, + count: scenario.count, + }; + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +async function measureAllScenarios(scenarios: MemoryScenario[], bundlePath: string) { + const results: Record = {}; + + for (const scenario of scenarios) { + const samples: number[] = []; + console.log(`Measuring ${scenario.id}`); + for (let sample = 0; sample < SAMPLE_COUNT; sample++) { + const measurement = await measureScenarioSample(scenario.id, bundlePath); + if (measurement.count !== scenario.count) { + throw new Error( + `Scenario ${scenario.id} measured ${measurement.count} instances, expected ${scenario.count}` + ); + } + samples.push(measurement.totalBytes); + console.log( + ` sample ${sample + 1}/${SAMPLE_COUNT}: ${formatBytes(measurement.totalBytes / measurement.count)}` + ); + } + + results[scenario.id] = { + id: scenario.id, + totalBytes: median(samples), + count: scenario.count, + }; + } + + return results; +} + +async function measureScenarioSample( + scenarioId: string, + bundlePath: string +): Promise { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [ + '--expose-gc', + '--require', + './scripts/runBefore.ts', + '--input-type=module', + '--eval', + CHILD_MEASURE_SOURCE, + bundlePath, + scenarioId, + String(COUNT_PER_SCENARIO), + ], + { + cwd: process.cwd(), + maxBuffer: 1024 * 1024 * 4, + } + ); + + if (stderr.trim()) { + process.stderr.write(stderr); + } + + const trimmed = stdout.trim(); + if (!trimmed) { + throw new Error(`No measurement output received for ${scenarioId}`); + } + + return JSON.parse(trimmed) as MeasurementResult; +} + +function buildStoredResults( + scenarios: MemoryScenario[], + measuredResults: Record +): StoredResults { + const storedScenarios = Object.fromEntries( + scenarios.map((scenario) => { + const measured = measuredResults[scenario.id]; + return [ + scenario.id, + { + totalBytes: measured.totalBytes, + bytesPerInstance: measured.totalBytes / measured.count, + }, + ]; + }) + ); + + return { + version: VERSION, + generatedAt: new Date().toISOString(), + sampleCount: SAMPLE_COUNT, + countPerScenario: COUNT_PER_SCENARIO, + scenarios: storedScenarios, + }; +} + +async function readStoredResults(storedPath: string): Promise { + try { + const contents = await readFile(storedPath, 'utf-8'); + return JSON.parse(contents) as StoredResults; + } catch (error) { + throw new Error( + `Unable to read ${RESULTS_PATH}. Run \`pnpm test.bench.update\` to create it.\n${String(error)}` + ); + } +} + +function validateResults( + scenarios: MemoryScenario[], + stored: StoredResults, + measuredResults: Record +) { + if (stored.countPerScenario !== COUNT_PER_SCENARIO) { + throw new Error( + `Stored countPerScenario ${stored.countPerScenario} does not match expected ${COUNT_PER_SCENARIO}` + ); + } + assertExactKeySet( + 'stored memory scenarios', + Object.keys(stored.scenarios), + Object.keys(measuredResults) + ); + + const failures: string[] = []; + for (const scenario of scenarios) { + const storedScenario = stored.scenarios[scenario.id]; + const measuredScenario = measuredResults[scenario.id]; + const allowedMax = Math.round(storedScenario.totalBytes * (1 + RELATIVE_TOLERANCE)); + const ok = measuredScenario.totalBytes <= allowedMax; + const measuredPerInstance = measuredScenario.totalBytes / measuredScenario.count; + const allowedMaxPerInstance = allowedMax / measuredScenario.count; + + console.log( + [ + `${scenario.id}:`, + `perInstance=${formatBytes(measuredPerInstance)}`, + `stored=${formatBytes(storedScenario.bytesPerInstance)}`, + `allowedMax=${formatBytes(allowedMaxPerInstance)}`, + ok ? 'OK' : 'FAIL', + ].join(' ') + ); + + if (!ok) { + failures.push( + `${scenario.id} retained ${formatBytes(measuredPerInstance)} per instance, above allowed max ${formatBytes(allowedMaxPerInstance)}` + ); + } + } + + if (failures.length > 0) { + throw new Error( + `Memory benchmark validation failed:\n${failures.map((failure) => `- ${failure}`).join('\n')}` + ); + } +} + +async function bundleMemoryScenarios(): Promise { + const outfile = resolve( + tmpdir(), + `qwik-memory-scenarios-${process.pid}-${Date.now().toString(36)}.mjs` + ); + + await build({ + entryPoints: [resolve(process.cwd(), SCENARIOS_ENTRY)], + outfile, + bundle: true, + format: 'esm', + platform: 'node', + target: ['node22'], + sourcemap: false, + conditions: ['development'], + logLevel: 'silent', + plugins: [ + { + name: 'qwik-memory-stubs', + setup(buildApi) { + buildApi.onResolve({ filter: /^@qwik\.dev\/core$/ }, () => ({ + path: resolve(process.cwd(), 'packages/qwik/dist/core.mjs'), + external: true, + })); + buildApi.onResolve({ filter: /^@qwik\.dev\/core\/internal$/ }, () => ({ + path: resolve(process.cwd(), 'packages/qwik/dist/core.mjs'), + external: true, + })); + buildApi.onResolve({ filter: /^@qwik-client-manifest$/ }, () => ({ + path: 'qwik-client-manifest-stub', + namespace: 'qwik-memory-stub', + })); + buildApi.onResolve({ filter: /^\.\.\/\.\.\/client\/vnode-utils$/ }, (args) => { + if (args.importer.endsWith('/shared/vnode/vnode.ts')) { + return { path: 'vnode-utils-stub', namespace: 'qwik-memory-stub' }; + } + return null; + }); + + buildApi.onLoad({ filter: /.*/, namespace: 'qwik-memory-stub' }, (args) => { + if (args.path === 'vnode-utils-stub') { + return { + contents: 'export const vnode_toString = () => "[VNode]";', + loader: 'js', + }; + } + if (args.path === 'qwik-client-manifest-stub') { + return { + contents: 'export const manifest = {};', + loader: 'js', + }; + } + return null; + }); + }, + }, + ], + }); + + return outfile; +} + +async function loadMemoryScenarios(bundlePath: string): Promise { + const mod = (await import(pathToFileURL(bundlePath).href)) as { + memoryScenarios: MemoryScenario[]; + }; + return mod.memoryScenarios; +} + +function requireGc() { + if (typeof global.gc !== 'function') { + throw new Error( + 'Memory benchmarks require `global.gc`. Run this script with `node --expose-gc --require ./scripts/runBefore.ts ...`.' + ); + } +} + +function forceGc() { + for (let i = 0; i < GC_ROUNDS; i++) { + global.gc!(); + } +} + +function median(values: number[]) { + if (values.length === 0) { + throw new Error('Cannot compute median of an empty sample set.'); + } + const sorted = [...values].sort((a, b) => a - b); + const midpoint = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? Math.round((sorted[midpoint - 1] + sorted[midpoint]) / 2) + : sorted[midpoint]; +} + +function assertExactKeySet(label: string, actual: string[], expected: string[]) { + const actualSorted = [...actual].sort(); + const expectedSorted = [...expected].sort(); + const missing = expectedSorted.filter((key) => !actualSorted.includes(key)); + const extra = actualSorted.filter((key) => !expectedSorted.includes(key)); + + if (missing.length === 0 && extra.length === 0) { + return; + } + + const parts: string[] = [`Unexpected ${label}.`]; + if (missing.length > 0) { + parts.push(`Missing: ${missing.join(', ')}`); + } + if (extra.length > 0) { + parts.push(`Extra: ${extra.join(', ')}`); + } + throw new Error(parts.join(' ')); +} + +function getArgValue(flag: string) { + const index = process.argv.indexOf(flag); + if (index === -1 || index + 1 >= process.argv.length) { + return undefined; + } + return process.argv[index + 1]; +} + +function formatBytes(value: number) { + const rounded = Math.round(value); + return `${rounded.toLocaleString('en-US')}B`; +} + +main() + .then(() => { + process.exit(0); + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); + }); diff --git a/scripts/validate-benchmarks.ts b/scripts/validate-benchmarks.ts new file mode 100644 index 00000000000..b8ff6be5883 --- /dev/null +++ b/scripts/validate-benchmarks.ts @@ -0,0 +1,432 @@ +import { execFile } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { promisify } from 'node:util'; + +const BENCH_ENTRY = 'packages/qwik/src/core/bench/core.bench.ts'; +const RESULTS_PATH = 'packages/qwik/src/core/bench/bench-results.json'; +const BASELINE_BENCHMARK_ID = 'baseline.shared-workload'; +const BENCHMARK_PREFIX = 'current.'; +const SIZE_LOG_PREFIX = 'QWIK_BENCH_SIZE'; +const MIN_TOLERANCE_PCT = 3; +const LOW_SAMPLE_MIN_TOLERANCE_PCT = 10; +const VERY_LOW_SAMPLE_MIN_TOLERANCE_PCT = 15; +const execFileAsync = promisify(execFile); + +type BenchmarkMetrics = { + mean: number; + median: number; + p75: number; + p99: number; + p995: number; + p999: number; + rme: number; + sampleCount: number; + factor?: number; +}; + +type StoredResults = { + version: number; + generatedAt: string; + benchmarks: Record; + sizes: Record; +}; + +const args = new Set(process.argv.slice(2)); +const shouldUpdate = args.has('--update'); + +async function main() { + const { measuredBenchmarks, measuredSizes } = await runBenchmarks(); + const scenarioIds = scenarioIdsFromBenchmarkNames(Object.keys(measuredBenchmarks)); + + const benchmarkNames = [ + BASELINE_BENCHMARK_ID, + ...scenarioIds.map((scenarioId) => `${BENCHMARK_PREFIX}${scenarioId}`), + ]; + + assertExactKeySet('measured benchmarks', Object.keys(measuredBenchmarks), benchmarkNames); + assertExactKeySet('measured sizes', Object.keys(measuredSizes), scenarioIds); + + const storedPath = resolve(process.cwd(), RESULTS_PATH); + + if (shouldUpdate) { + const nextResults = buildStoredResults(scenarioIds, measuredBenchmarks, measuredSizes); + await writeFile(storedPath, JSON.stringify(nextResults, null, 2) + '\n', 'utf-8'); + console.log(`Updated benchmark baselines in ${RESULTS_PATH}`); + return; + } + + const stored = await readStoredResults(storedPath); + validateResults(scenarioIds, stored, measuredBenchmarks, measuredSizes); + console.log('Benchmark validation passed.'); +} + +async function runBenchmarks(): Promise<{ + measuredBenchmarks: Record; + measuredSizes: Record; +}> { + const outputJsonPath = resolve( + tmpdir(), + `qwik-bench-${process.pid}-${Date.now().toString(36)}.json` + ); + + const { stdout, stderr } = await execFileAsync( + 'pnpm', + [ + 'vitest', + 'bench', + BENCH_ENTRY, + '--outputJson', + outputJsonPath, + '--maxWorkers=1', + '--no-file-parallelism', + '--run', + '--silent', + ], + { + cwd: process.cwd(), + maxBuffer: 1024 * 1024 * 16, + } + ); + + if (stdout.trim()) { + process.stdout.write(stdout); + } + + const measuredSizes = parseScenarioSizes(stderr); + const passthroughStderr = stripScenarioSizeLines(stderr); + if (passthroughStderr.trim()) { + process.stderr.write(passthroughStderr); + } + + const report = JSON.parse(await readFile(outputJsonPath, 'utf-8')) as { + files: Array<{ + groups: Array<{ + benchmarks: Array< + BenchmarkMetrics & { + name: string; + } + >; + }>; + }>; + }; + + const benchmarks: Record = {}; + for (const file of report.files) { + for (const group of file.groups) { + for (const benchmark of group.benchmarks) { + benchmarks[benchmark.name] = { + mean: benchmark.mean ?? 0, + median: benchmark.median ?? 0, + p75: benchmark.p75 ?? benchmark.median ?? 0, + p99: benchmark.p99 ?? 0, + p995: benchmark.p995 ?? benchmark.p99 ?? 0, + p999: benchmark.p999 ?? benchmark.p995 ?? benchmark.p99 ?? 0, + rme: benchmark.rme ?? 0, + sampleCount: benchmark.sampleCount ?? 0, + }; + } + } + } + + return { measuredBenchmarks: benchmarks, measuredSizes }; +} + +function buildStoredResults( + scenarioIds: string[], + measuredBenchmarks: Record, + measuredSizes: Record +): StoredResults { + const baseline = measuredBenchmarks[BASELINE_BENCHMARK_ID]; + if (!baseline) { + throw new Error(`Missing benchmark result for ${BASELINE_BENCHMARK_ID}`); + } + + const benchmarks: Record = {}; + for (const name of sortBenchmarkNames(Object.keys(measuredBenchmarks))) { + const benchmark = measuredBenchmarks[name]; + benchmarks[name] = + name === BASELINE_BENCHMARK_ID + ? benchmark + : { + ...benchmark, + factor: benchmark.median / baseline.median, + }; + } + + const sizes = Object.fromEntries( + scenarioIds.map((scenarioId) => [scenarioId, measuredSizes[scenarioId] ?? 0]) + ); + + return { + version: 1, + generatedAt: new Date().toISOString(), + benchmarks, + sizes, + }; +} + +async function readStoredResults(storedPath: string): Promise { + try { + const contents = await readFile(storedPath, 'utf-8'); + return JSON.parse(contents) as StoredResults; + } catch (error) { + throw new Error( + `Unable to read ${RESULTS_PATH}. Run \`pnpm test.bench.update\` to create it.\n${String(error)}` + ); + } +} + +function validateResults( + scenarioIds: string[], + stored: StoredResults, + measuredBenchmarks: Record, + measuredSizes: Record +) { + assertExactKeySet( + 'stored benchmarks', + Object.keys(stored.benchmarks), + Object.keys(measuredBenchmarks) + ); + assertExactKeySet('stored sizes', Object.keys(stored.sizes), Object.keys(measuredSizes)); + + const storedBaseline = stored.benchmarks[BASELINE_BENCHMARK_ID]; + const measuredBaseline = measuredBenchmarks[BASELINE_BENCHMARK_ID]; + + if (!storedBaseline || !measuredBaseline) { + throw new Error(`Baseline benchmark ${BASELINE_BENCHMARK_ID} is required.`); + } + + const failures: string[] = []; + const lines: string[] = []; + const warnings: string[] = []; + + addSampleCountWarning(warnings, BASELINE_BENCHMARK_ID, measuredBaseline.sampleCount); + + const baselineTolerancePct = tolerancePct(storedBaseline); + const baselineMax = maxAllowedValue(storedBaseline.median, baselineTolerancePct); + const baselineOk = measuredBaseline.median <= baselineMax; + lines.push( + `${BASELINE_BENCHMARK_ID}: median=${formatNumber( + measuredBaseline.median + )}ms stored=${formatNumber(storedBaseline.median)}ms max=${formatNumber( + baselineMax + )}ms tail=${formatPercent(baselineTolerancePct)} samples=${measuredBaseline.sampleCount} ${ + baselineOk ? 'OK' : 'FAIL' + }` + ); + if (!baselineOk) { + failures.push( + `${BASELINE_BENCHMARK_ID} median ${formatNumber( + measuredBaseline.median + )}ms exceeds ${formatNumber(baselineMax)}ms` + ); + } + + for (const scenarioId of scenarioIds) { + const benchmarkId = `${BENCHMARK_PREFIX}${scenarioId}`; + const storedBenchmark = stored.benchmarks[benchmarkId]; + const measuredBenchmark = measuredBenchmarks[benchmarkId]; + + if (!storedBenchmark || !measuredBenchmark) { + failures.push(`Missing benchmark entry for ${benchmarkId}`); + continue; + } + + addSampleCountWarning(warnings, benchmarkId, measuredBenchmark.sampleCount); + + const medianTolerancePct = tolerancePct(storedBenchmark); + const factorTolerancePct = tolerancePct(storedBenchmark) + tolerancePct(storedBaseline); + const storedFactor = storedBenchmark.factor; + const measuredFactor = measuredBenchmark.median / measuredBaseline.median; + const medianMax = maxAllowedValue(storedBenchmark.median, medianTolerancePct); + const factorMax = + storedFactor == null ? null : maxAllowedValue(storedFactor, factorTolerancePct); + const expectedSize = stored.sizes[scenarioId]; + const measuredSize = measuredSizes[scenarioId]; + + const medianOk = measuredBenchmark.median <= medianMax; + const factorOk = factorMax != null ? measuredFactor <= factorMax : true; + const sizeOk = measuredSize === expectedSize; + const status = factorOk && sizeOk ? (medianOk ? 'OK' : 'WARN') : 'FAIL'; + + lines.push( + [ + `${benchmarkId}:`, + `median=${formatNumber(measuredBenchmark.median)}ms`, + `stored=${formatNumber(storedBenchmark.median)}ms`, + `max=${formatNumber(medianMax)}ms`, + `factor=${formatNumber(measuredFactor)}x`, + storedFactor == null ? '' : `storedFactor=${formatNumber(storedFactor)}x`, + factorMax == null ? '' : `factorMax=${formatNumber(factorMax)}x`, + `size=${measuredSize}/${expectedSize}`, + `tail=${formatPercent(medianTolerancePct)}`, + `samples=${measuredBenchmark.sampleCount}`, + status, + ] + .filter(Boolean) + .join(' ') + ); + + if (!medianOk) { + warnings.push( + `${benchmarkId} median ${formatNumber(measuredBenchmark.median)}ms exceeds ${formatNumber( + medianMax + )}ms, but factor still passed; machine may be slower than the stored baseline` + ); + } + if (!factorOk && factorMax != null) { + failures.push( + `${benchmarkId} factor ${formatNumber(measuredFactor)}x exceeds ${formatNumber(factorMax)}x` + ); + } + if (!sizeOk) { + failures.push( + `${benchmarkId} size ${measuredSize} does not match stored size ${expectedSize}` + ); + } + } + + for (const warning of warnings) { + console.warn(`Warning: ${warning}`); + } + + for (const line of lines) { + console.log(line); + } + + if (failures.length > 0) { + throw new Error( + `Benchmark validation failed:\n${failures.map((failure) => `- ${failure}`).join('\n')}` + ); + } +} + +function scenarioIdsFromBenchmarkNames(names: string[]) { + const scenarioIds = names + .filter((name) => name.startsWith(BENCHMARK_PREFIX)) + .map((name) => name.slice(BENCHMARK_PREFIX.length)); + + return [...scenarioIds].sort((a, b) => a.localeCompare(b)); +} + +function parseScenarioSizes(stderr: string): Record { + const sizes: Record = {}; + + for (const line of stderr.split(/\r?\n/)) { + if (!line.startsWith(`${SIZE_LOG_PREFIX}\t`)) { + continue; + } + + const [, scenarioId, rawSize, ...rest] = line.split('\t'); + if (!scenarioId || !rawSize || rest.length > 0) { + throw new Error(`Malformed benchmark size line: ${line}`); + } + + const size = Number(rawSize); + if (!Number.isInteger(size) || size < 0) { + throw new Error(`Invalid benchmark size for ${scenarioId}: ${rawSize}`); + } + + const previousSize = sizes[scenarioId]; + if (previousSize != null && previousSize !== size) { + throw new Error(`Conflicting benchmark sizes for ${scenarioId}: ${previousSize} and ${size}`); + } + + sizes[scenarioId] = size; + } + + return sizes; +} + +function stripScenarioSizeLines(stderr: string) { + return stderr + .split(/\r?\n/) + .filter((line) => line && !line.startsWith(`${SIZE_LOG_PREFIX}\t`)) + .join('\n'); +} + +function tolerancePct( + metrics: Pick +) { + const minTolerance = + metrics.sampleCount <= 10 + ? VERY_LOW_SAMPLE_MIN_TOLERANCE_PCT + : metrics.sampleCount < 25 + ? LOW_SAMPLE_MIN_TOLERANCE_PCT + : MIN_TOLERANCE_PCT; + + const p75Skew = percentAbove(metrics.p75, metrics.median); + const p99Skew = percentAbove(metrics.p99, metrics.median); + const p995Skew = percentAbove(metrics.p995, metrics.median); + const p999Skew = percentAbove(metrics.p999, metrics.median); + + const skewTolerance = p75Skew * 2 + p99Skew * 0.2 + p995Skew * 0.1 + p999Skew * 0.05; + + return Math.max(skewTolerance, minTolerance); +} + +function percentAbove(value: number, baseline: number) { + if (baseline <= 0 || value <= baseline) { + return 0; + } + return ((value - baseline) / baseline) * 100; +} + +function maxAllowedValue(value: number, tolerancePct: number) { + return value * (1 + tolerancePct / 100); +} + +function addSampleCountWarning(warnings: string[], benchmarkId: string, sampleCount: number) { + if (sampleCount <= 10) { + warnings.push( + `${benchmarkId} only collected ${sampleCount} samples; consider increasing runtime` + ); + } +} + +function assertExactKeySet(label: string, actual: string[], expected: string[]) { + const actualSorted = [...actual].sort(); + const expectedSorted = [...expected].sort(); + const missing = expectedSorted.filter((key) => !actualSorted.includes(key)); + const extra = actualSorted.filter((key) => !expectedSorted.includes(key)); + + if (missing.length === 0 && extra.length === 0) { + return; + } + + const parts: string[] = [`Unexpected ${label}.`]; + if (missing.length > 0) { + parts.push(`Missing: ${missing.join(', ')}`); + } + if (extra.length > 0) { + parts.push(`Extra: ${extra.join(', ')}`); + } + throw new Error(parts.join(' ')); +} + +function sortBenchmarkNames(names: string[]) { + return [...names].sort((a, b) => { + if (a === BASELINE_BENCHMARK_ID) { + return -1; + } + if (b === BASELINE_BENCHMARK_ID) { + return 1; + } + return a.localeCompare(b); + }); +} + +function formatNumber(value: number) { + return value.toFixed(value >= 10 ? 2 : 4); +} + +function formatPercent(value: number) { + return `${value.toFixed(2)}%`; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/vitest.bench.config.ts b/vitest.bench.config.ts new file mode 100644 index 00000000000..8b434ca5a25 --- /dev/null +++ b/vitest.bench.config.ts @@ -0,0 +1,57 @@ +import { fileURLToPath } from 'node:url'; +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + resolve: { + alias: [ + { + find: '@qwik.dev/core/jsx-dev-runtime', + replacement: fileURLToPath( + new URL('./packages/qwik/src/core/shared/jsx/jsx-runtime.ts', import.meta.url) + ), + }, + { + find: '@qwik.dev/core/jsx-runtime', + replacement: fileURLToPath( + new URL('./packages/qwik/src/core/shared/jsx/jsx-runtime.ts', import.meta.url) + ), + }, + { + find: '@qwik.dev/core/build', + replacement: fileURLToPath( + new URL('./packages/qwik/src/build/index.dev.ts', import.meta.url) + ), + }, + { + find: '@qwik.dev/core/preloader', + replacement: fileURLToPath( + new URL('./packages/qwik/src/core/bench/preloader-stub.ts', import.meta.url) + ), + }, + { + find: '@qwik.dev/core/internal', + replacement: fileURLToPath( + new URL('./packages/qwik/src/core/internal.ts', import.meta.url) + ), + }, + { + find: '@qwik.dev/core', + replacement: fileURLToPath(new URL('./packages/qwik/src/core/index.ts', import.meta.url)), + }, + { + find: '@qwik-client-manifest', + replacement: fileURLToPath( + new URL('./packages/qwik/src/core/bench/manifest-stub.ts', import.meta.url) + ), + }, + ], + }, + test: { + root: 'packages', + include: ['**/*.bench.?(c|m)[jt]s?(x)'], + testTimeout: 120000, + }, + benchmark: { + include: ['**/*.bench.?(c|m)[jt]s?(x)'], + }, +}); diff --git a/vitest.config.ts b/vitest.config.ts index d8adf9c0085..1da2a78c1ba 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -14,8 +14,8 @@ export default defineConfig({ test: { root: 'packages', include: [ - '**/*.spec.?(c|m)[jt]s?(x)', - '**/*.unit.?(c|m)[jt]s?(x)', + '**/*.spec.*', + '**/*.unit.*', '!*/(lib|dist|build|server|target)/**', '!**/node_modules/**', ],