Skip to content

Commit f2c883d

Browse files
wmertenswmertens
committed
chore(bench): use a validation script
more precise than a vite snapshot
1 parent 6178981 commit f2c883d

6 files changed

Lines changed: 549 additions & 130 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,8 @@
248248
"test.rust": "make test",
249249
"test.rust.bench": "make benchmark",
250250
"test.rust.update": "make test-update",
251-
"test.bench": "RUN_BENCHMARK=true vitest --maxWorkers=1 --no-file-parallelism packages/qwik/src/core/bench",
251+
"test.bench": "node --require ./scripts/runBefore.ts scripts/validate-benchmarks.ts",
252+
"test.bench.update": "node --require ./scripts/runBefore.ts scripts/validate-benchmarks.ts --update",
252253
"test.unit": "vitest packages",
253254
"test.unit.debug": "vitest --inspect-brk packages",
254255
"test.vite": "playwright test e2e/qwik-e2e/tests/qwikrouter --browser=chromium --config e2e/qwik-e2e/playwright.config.ts",

‎packages/qwik/src/core/bench/bench-harness.spec.ts‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

‎packages/qwik/src/core/bench/bench-results.json‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
{
2+
"version": 1,
3+
"generatedAt": "2026-03-19T07:49:32.878Z",
4+
"benchmarks": {
5+
"baseline.shared-workload": {
6+
"mean": 3.088664493055571,
7+
"median": 2.9838345000002846,
8+
"p75": 3.1992929999996704,
9+
"p99": 4.196222999999918,
10+
"p995": 4.295756000000438,
11+
"p999": 4.766966000000139,
12+
"rme": 0.6081320295006954,
13+
"sampleCount": 1296
14+
},
15+
"current.dom-table-10": {
16+
"mean": 2.6569924736609103,
17+
"median": 2.4209310000005644,
18+
"p75": 2.6598269999994955,
19+
"p99": 4.945772000002762,
20+
"p995": 5.941971000000194,
21+
"p999": 7.454273999999714,
22+
"rme": 0.9972462077872076,
23+
"sampleCount": 2259,
24+
"factor": 0.8113489538378665
25+
},
26+
"current.dom-table-1k": {
27+
"mean": 226.27416044444504,
28+
"median": 223.8181110000005,
29+
"p75": 228.70898399999714,
30+
"p99": 269.1425390000004,
31+
"p995": 269.1425390000004,
32+
"p999": 269.1425390000004,
33+
"rme": 1.7506185315204819,
34+
"sampleCount": 27,
35+
"factor": 75.01022962231288
36+
},
37+
"current.dom-update-table-1k": {
38+
"mean": 0.16147774939449824,
39+
"median": 0.14038699999946402,
40+
"p75": 0.15433199999824865,
41+
"p99": 0.33831800000189105,
42+
"p995": 0.4541299999982584,
43+
"p999": 3.4702279999983148,
44+
"rme": 1.8118797821851593,
45+
"sampleCount": 37158,
46+
"factor": 0.04704919123344496
47+
},
48+
"current.serialize-state-1k": {
49+
"mean": 8.639306661870325,
50+
"median": 7.590303000004496,
51+
"p75": 9.659450999999535,
52+
"p99": 19.368933999998262,
53+
"p995": 22.706399999995483,
54+
"p999": 26.309580000001006,
55+
"rme": 2.7205570476483603,
56+
"sampleCount": 695,
57+
"factor": 2.543808311085542
58+
},
59+
"current.ssr-table-10": {
60+
"mean": 3.6770456717697337,
61+
"median": 3.153889999999592,
62+
"p75": 4.103724000000511,
63+
"p99": 8.557779999999184,
64+
"p995": 10.160756999999649,
65+
"p999": 30.789652999999817,
66+
"rme": 2.2655355423081396,
67+
"sampleCount": 1633,
68+
"factor": 1.0569922695106888
69+
},
70+
"current.ssr-table-1k": {
71+
"mean": 348.1837142222226,
72+
"median": 318.59908899999937,
73+
"p75": 353.7774320000026,
74+
"p99": 548.3319370000027,
75+
"p995": 548.3319370000027,
76+
"p999": 548.3319370000027,
77+
"rme": 12.380673917171574,
78+
"sampleCount": 18,
79+
"factor": 106.77505371024063
80+
}
81+
},
82+
"sizes": {
83+
"dom-table-10": 172,
84+
"dom-table-1k": 18553,
85+
"dom-update-table-1k": 26522,
86+
"serialize-state-1k": 96844,
87+
"ssr-table-10": 2175,
88+
"ssr-table-1k": 177401
89+
}
90+
}

‎packages/qwik/src/core/bench/core.bench.ts‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,34 @@ import { bench, describe } from 'vitest';
22
import { sharedBaselineWorkload } from './baseline';
33
import { scenarios } from './scenarios';
44

5-
const BASELINE_OPTIONS = {
6-
warmupTime: 50,
7-
warmupIterations: 5,
8-
time: 250,
9-
};
10-
11-
const SSR_OPTIONS = {
12-
warmupTime: 20,
13-
warmupIterations: 1,
14-
time: 200,
15-
};
5+
const SIZE_LOG_PREFIX = 'QWIK_BENCH_SIZE';
166

177
describe('qwik core relative benchmarks', () => {
188
bench(
199
'baseline.shared-workload',
2010
async () => {
2111
sharedBaselineWorkload();
2212
},
23-
BASELINE_OPTIONS
13+
{ warmupTime: 500, time: 4000 }
2414
);
2515

2616
for (const scenario of scenarios) {
17+
let lastSize: number | null = null;
2718
bench(
2819
`current.${scenario.id}`,
2920
async () => {
30-
await scenario.run();
21+
const size = await scenario.run();
22+
if (lastSize === null) {
23+
lastSize = size;
24+
process.stderr.write(`${SIZE_LOG_PREFIX}\t${scenario.id}\t${size}\n`);
25+
} else if (lastSize !== size) {
26+
throw new Error(
27+
`Scenario ${scenario.id} returned inconsistent sizes: ${lastSize} vs ${size}`
28+
);
29+
}
30+
return size;
3131
},
32-
SSR_OPTIONS
32+
{ warmupTime: 500, time: 6000 }
3333
);
3434
}
3535
});

‎packages/qwik/src/core/bench/scenarios.tsx‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,7 @@ const makeUpdatedRows = (rows: TableRow[]): TableRow[] => {
5252

5353
const rows10 = makeRows(10);
5454
const rows1000 = makeRows(1000);
55-
const rows10000 = makeRows(10000);
5655
const updatedRows1000 = makeUpdatedRows(rows1000);
57-
const updatedRows10000 = makeUpdatedRows(rows10000);
5856

5957
const sharedMeta = {
6058
adjectives: ['pretty', 'large', 'small', 'helpful'],
@@ -105,6 +103,16 @@ const renderTable = (rows: TableRow[]): JSXOutput => {
105103
);
106104
};
107105

106+
const estimateTableSize = (rows: TableRow[]): number => {
107+
let size = 24;
108+
for (const row of rows) {
109+
size += String(row.id).length;
110+
size += row.label.length;
111+
size += String(row.value).length;
112+
}
113+
return size;
114+
};
115+
108116
const renderSsr = async (jsx: JSXOutput): Promise<number> => {
109117
const result = await renderToString(jsx, { qwikLoader: 'never', containerTagName: 'div' });
110118
return result.html.length;
@@ -142,7 +150,7 @@ const makeDomScenario = (id: string, rowCount: number, rows: TableRow[]): Benchm
142150
title: `DOM table ${rowCount} rows`,
143151
run: async () => {
144152
await renderDom(renderTable(rows));
145-
return 0;
153+
return estimateTableSize(rows);
146154
},
147155
};
148156
};
@@ -158,20 +166,17 @@ const makeDomUpdateScenario = (
158166
title: `DOM update table ${rowCount} rows`,
159167
run: async () => {
160168
await renderDomUpdate(initialRows, nextRows);
161-
return 0;
169+
return estimateTableSize(nextRows);
162170
},
163171
};
164172
};
165173

166174
export const scenarios: BenchmarkScenario[] = [
167175
makeScenario('ssr-table-10', 10, rows10),
168176
makeScenario('ssr-table-1k', 1000, rows1000),
169-
makeScenario('ssr-table-10k', 10000, rows10000),
170177
makeDomScenario('dom-table-10', 10, rows10),
171178
makeDomScenario('dom-table-1k', 1000, rows1000),
172-
makeDomScenario('dom-table-10k', 10000, rows10000),
173179
makeDomUpdateScenario('dom-update-table-1k', 1000, rows1000, updatedRows1000),
174-
makeDomUpdateScenario('dom-update-table-10k', 10000, rows10000, updatedRows10000),
175180
{
176181
id: 'serialize-state-1k',
177182
title: 'Serialize 1k-item state graph',

0 commit comments

Comments
 (0)