Skip to content

Commit e6f1823

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
chore: base run with main impl
1 parent b232faf commit e6f1823

File tree

1 file changed

+175
-100
lines changed

1 file changed

+175
-100
lines changed

‎packages/tinybench-plugin/src/walltime.ts‎

Lines changed: 175 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,134 +1,209 @@
11
import {
2+
Benchmark,
23
calculateQuantiles,
34
mongoMeasurement,
45
msToNs,
56
msToS,
67
writeWalltimeResults,
7-
type Benchmark as CodspeedBenchmark,
88
type BenchmarkStats,
99
} from "@codspeed/core";
10-
import { Bench, Fn, Task, TaskResult } from "tinybench";
11-
import { BaseBenchRunner } from "./shared";
10+
import { Bench, TaskResult } from "tinybench";
11+
import { getTaskUri } from "./uri";
1212

1313
export function setupCodspeedWalltimeBench(
1414
bench: Bench,
1515
rootCallingFile: string
1616
): void {
17-
const runner = new WalltimeBenchRunner(bench, rootCallingFile);
18-
runner.setupBenchMethods();
19-
}
20-
21-
class WalltimeBenchRunner extends BaseBenchRunner {
22-
private codspeedBenchmarks: CodspeedBenchmark[] = [];
23-
24-
protected getModeName(): string {
25-
return "walltime mode";
26-
}
17+
// const runner = new WalltimeBenchRunner(bench, rootCallingFile);
18+
// runner.setupBenchMethods();
2719

28-
protected async runTaskAsync(task: Task, uri: string): Promise<void> {
29-
// Override the function under test to add a static frame
30-
this.wrapTaskFunction(task, true);
31-
32-
// run the warmup of the task right before its actual run
33-
if (this.bench.opts.warmup) {
34-
await task.warmup();
35-
}
36-
37-
await mongoMeasurement.start(uri);
38-
await this.wrapWithInstrumentHooksAsync(() => task.run(), uri);
39-
await mongoMeasurement.stop(uri);
40-
41-
this.registerCodspeedBenchmarkFromTask(task);
42-
}
43-
44-
protected runTaskSync(task: Task, uri: string): void {
45-
// Override the function under test to add a static frame
46-
this.wrapTaskFunction(task, false);
20+
bench.run = async () => {
21+
console.log(
22+
`[CodSpeed] running with @codspeed/tinybench v (walltime mode)`
23+
);
4724

48-
if (this.bench.opts.warmup) {
49-
task.warmup();
50-
}
25+
// Store the original run method before we override it
26+
const originalRun = bench.run;
5127

52-
this.wrapWithInstrumentHooks(() => task.runSync(), uri);
28+
// Temporarily restore the original run to get actual benchmark results
29+
const benchProto = Object.getPrototypeOf(bench);
30+
const prototypeRun = benchProto.run;
31+
bench.run = prototypeRun;
5332

54-
this.registerCodspeedBenchmarkFromTask(task);
55-
}
33+
const benchmarks: Benchmark[] = [];
5634

57-
protected finalizeAsyncRun(): Task[] {
58-
return this.finalizeWalltimeRun(true);
59-
}
35+
// Run the bench naturally to collect TaskResult data
36+
const results = [];
6037

61-
protected finalizeSyncRun(): Task[] {
62-
return this.finalizeWalltimeRun(false);
63-
}
38+
// Collect and report walltime data
39+
for (const task of bench.tasks) {
40+
const uri = getTaskUri(bench, task.name, rootCallingFile);
6441

65-
private wrapTaskFunction(task: Task, isAsync: boolean): void {
66-
const { fn } = task as unknown as { fn: Fn };
67-
if (isAsync) {
68-
// eslint-disable-next-line no-inner-declarations
69-
async function __codspeed_root_frame__() {
70-
await fn();
42+
// run the warmup of the task right before its actual run
43+
if (bench.opts.warmup) {
44+
await task.warmup();
7145
}
72-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
73-
(task as any).fn = __codspeed_root_frame__;
74-
} else {
75-
// eslint-disable-next-line no-inner-declarations
76-
function __codspeed_root_frame__() {
77-
fn();
46+
await mongoMeasurement.start(uri);
47+
const taskResult = await task.run();
48+
await mongoMeasurement.stop(uri);
49+
results.push(taskResult);
50+
51+
if (task.result) {
52+
// Convert tinybench result to BenchmarkStats format
53+
const stats = convertTinybenchResultToBenchmarkStats(
54+
task.result,
55+
bench.opts.warmup ? bench.opts.warmupIterations ?? 0 : 0
56+
);
57+
58+
const benchmark: Benchmark = {
59+
name: task.name,
60+
uri,
61+
config: {
62+
max_rounds: bench.opts.iterations ?? null,
63+
max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null,
64+
min_round_time_ns: null, // tinybench does not have an option for this
65+
warmup_time_ns:
66+
bench.opts.warmup && bench.opts.warmupTime
67+
? msToNs(bench.opts.warmupTime)
68+
: null,
69+
},
70+
stats,
71+
};
72+
73+
benchmarks.push(benchmark);
74+
75+
console.log(` ✔ Collected walltime data for ${uri}`);
76+
} else {
77+
console.warn(` ⚠ No result data available for ${uri}`);
7878
}
79-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
80-
(task as any).fn = __codspeed_root_frame__;
81-
}
82-
}
83-
84-
private registerCodspeedBenchmarkFromTask(task: Task): void {
85-
const uri = this.getTaskUri(task);
86-
87-
if (!task.result) {
88-
console.warn(` ⚠ No result data available for ${uri}`);
89-
return;
9079
}
9180

92-
const warmupIterations = this.bench.opts.warmup
93-
? this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT
94-
: 0;
95-
const stats = convertTinybenchResultToBenchmarkStats(
96-
task.result,
97-
warmupIterations
98-
);
99-
100-
this.codspeedBenchmarks.push({
101-
name: task.name,
102-
uri,
103-
config: {
104-
max_rounds: this.bench.opts.iterations ?? null,
105-
max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null,
106-
min_round_time_ns: null, // tinybench does not have an option for this
107-
warmup_time_ns:
108-
this.bench.opts.warmup && this.bench.opts.warmupTime
109-
? msToNs(this.bench.opts.warmupTime)
110-
: null,
111-
},
112-
stats,
113-
});
114-
115-
this.logTaskCompletion(uri, "Collected walltime data for");
116-
}
117-
118-
private finalizeWalltimeRun(isAsync: boolean): Task[] {
11981
// Write results to JSON file using core function
120-
if (this.codspeedBenchmarks.length > 0) {
121-
writeWalltimeResults(this.codspeedBenchmarks, isAsync);
82+
if (benchmarks.length > 0) {
83+
writeWalltimeResults(benchmarks);
12284
}
12385

12486
console.log(
125-
`[CodSpeed] Done collecting walltime data for ${this.bench.tasks.length} benches.`
87+
`[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.`
12688
);
127-
return this.bench.tasks;
128-
}
89+
// Restore our custom run method
90+
bench.run = originalRun;
91+
92+
return results;
93+
};
12994
}
13095

131-
const TINYBENCH_WARMUP_DEFAULT = 16;
96+
// class WalltimeBenchRunner extends BaseBenchRunner {
97+
// private codspeedBenchmarks: CodspeedBenchmark[] = [];
98+
//
99+
// protected getModeName(): string {
100+
// return "walltime mode";
101+
// }
102+
//
103+
// protected async runTaskAsync(task: Task, uri: string): Promise<void> {
104+
// // Override the function under test to add a static frame
105+
// this.wrapTaskFunction(task, true);
106+
//
107+
// // run the warmup of the task right before its actual run
108+
// if (this.bench.opts.warmup) {
109+
// await task.warmup();
110+
// }
111+
//
112+
// await mongoMeasurement.start(uri);
113+
// await this.wrapWithInstrumentHooksAsync(() => task.run(), uri);
114+
// await mongoMeasurement.stop(uri);
115+
//
116+
// this.registerCodspeedBenchmarkFromTask(task);
117+
// }
118+
//
119+
// protected runTaskSync(task: Task, uri: string): void {
120+
// // Override the function under test to add a static frame
121+
// this.wrapTaskFunction(task, false);
122+
//
123+
// if (this.bench.opts.warmup) {
124+
// task.warmup();
125+
// }
126+
//
127+
// this.wrapWithInstrumentHooks(() => task.runSync(), uri);
128+
//
129+
// this.registerCodspeedBenchmarkFromTask(task);
130+
// }
131+
//
132+
// protected finalizeAsyncRun(): Task[] {
133+
// return this.finalizeWalltimeRun(true);
134+
// }
135+
//
136+
// protected finalizeSyncRun(): Task[] {
137+
// return this.finalizeWalltimeRun(false);
138+
// }
139+
//
140+
// private wrapTaskFunction(task: Task, isAsync: boolean): void {
141+
// const { fn } = task as unknown as { fn: Fn };
142+
// if (isAsync) {
143+
// // eslint-disable-next-line no-inner-declarations
144+
// async function __codspeed_root_frame__() {
145+
// await fn();
146+
// }
147+
// // eslint-disable-next-line @typescript-eslint/no-explicit-any
148+
// (task as any).fn = __codspeed_root_frame__;
149+
// } else {
150+
// // eslint-disable-next-line no-inner-declarations
151+
// function __codspeed_root_frame__() {
152+
// fn();
153+
// }
154+
// // eslint-disable-next-line @typescript-eslint/no-explicit-any
155+
// (task as any).fn = __codspeed_root_frame__;
156+
// }
157+
// }
158+
//
159+
// private registerCodspeedBenchmarkFromTask(task: Task): void {
160+
// const uri = this.getTaskUri(task);
161+
//
162+
// if (!task.result) {
163+
// console.warn(` ⚠ No result data available for ${uri}`);
164+
// return;
165+
// }
166+
//
167+
// const warmupIterations = this.bench.opts.warmup
168+
// ? this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT
169+
// : 0;
170+
// const stats = convertTinybenchResultToBenchmarkStats(
171+
// task.result,
172+
// warmupIterations
173+
// );
174+
//
175+
// this.codspeedBenchmarks.push({
176+
// name: task.name,
177+
// uri,
178+
// config: {
179+
// max_rounds: this.bench.opts.iterations ?? null,
180+
// max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null,
181+
// min_round_time_ns: null, // tinybench does not have an option for this
182+
// warmup_time_ns:
183+
// this.bench.opts.warmup && this.bench.opts.warmupTime
184+
// ? msToNs(this.bench.opts.warmupTime)
185+
// : null,
186+
// },
187+
// stats,
188+
// });
189+
//
190+
// this.logTaskCompletion(uri, "Collected walltime data for");
191+
// }
192+
//
193+
// private finalizeWalltimeRun(isAsync: boolean): Task[] {
194+
// // Write results to JSON file using core function
195+
// if (this.codspeedBenchmarks.length > 0) {
196+
// writeWalltimeResults(this.codspeedBenchmarks, isAsync);
197+
// }
198+
//
199+
// console.log(
200+
// `[CodSpeed] Done collecting walltime data for ${this.bench.tasks.length} benches.`
201+
// );
202+
// return this.bench.tasks;
203+
// }
204+
// }
205+
206+
// const TINYBENCH_WARMUP_DEFAULT = 16;
132207

133208
function convertTinybenchResultToBenchmarkStats(
134209
result: TaskResult,

0 commit comments

Comments
 (0)