feat(tinybench-plugin): add support for perf profiling

Support is still far from perfect for async/heavy code.

Author: GuillaumeLagrange

packages/core/src/introspection.ts

@@ -1,27 +1,37 @@
 import { writeFileSync } from "fs";
+import { getCodspeedRunnerMode } from ".";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
 export const getV8Flags = () => {
   const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
+  const codspeedRunnerMode = getCodspeedRunnerMode();
 
-  const flags = [
-    "--hash-seed=1",
-    "--random-seed=1",
-    "--no-opt",
-    "--predictable",
-    "--predictable-gc-schedule",
-    "--interpreted-frames-native-stack",
-    "--allow-natives-syntax",
-    "--expose-gc",
-    "--no-concurrent-sweeping",
-    "--max-old-space-size=4096",
-  ];
-  if (nodeVersionMajor < 18) {
-    flags.push("--no-randomize-hashes");
+  const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
+
+  if (codspeedRunnerMode === "instrumented") {
+    flags.push(
+      ...[
+        "--hash-seed=1",
+        "--random-seed=1",
+        "--no-opt",
+        "--predictable",
+        "--predictable-gc-schedule",
+        "--expose-gc",
+        "--no-concurrent-sweeping",
+        "--max-old-space-size=4096",
+      ]
+    );
+    if (nodeVersionMajor < 18) {
+      flags.push("--no-randomize-hashes");
+    }
+    if (nodeVersionMajor < 20) {
+      flags.push("--no-scavenge-task");
+    }
   }
-  if (nodeVersionMajor < 20) {
-    flags.push("--no-scavenge-task");
+
+  if (codspeedRunnerMode === "walltime") {
+    flags.push(...["--perf-prof", "--perf-prof-unwinding-info"]);
   }
   return flags;
 };

packages/tinybench-plugin/benches/sample.ts

@@ -35,7 +35,7 @@ bench
   });
 
 (async () => {
-  await bench.run();
+  bench.runSync();
   console.table(bench.table());
 
   const timingBench = withCodSpeed(
@@ -44,6 +44,6 @@ bench
 
   registerTimingBenchmarks(timingBench);
 
-  await timingBench.run();
+  timingBench.runSync();
   console.table(timingBench.table());
 })();

packages/tinybench-plugin/benches/timing.ts

@@ -8,15 +8,15 @@ const busySleep = (ms: number): void => {
 };
 
 export function registerTimingBenchmarks(bench: Bench) {
-  bench.add("wait 1ms", async () => {
+  bench.add("wait 1ms", () => {
     busySleep(1);
   });
 
-  bench.add("wait 500ms", async () => {
+  bench.add("wait 500ms", () => {
     busySleep(500);
   });
 
-  bench.add("wait 1sec", async () => {
+  bench.add("wait 1sec", () => {
     busySleep(1000);
   });
 }

packages/tinybench-plugin/src/walltime.ts

@@ -1,5 +1,6 @@
 import {
   calculateQuantiles,
+  InstrumentHooks,
   mongoMeasurement,
   msToNs,
   msToS,
@@ -40,7 +41,11 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void {
         await task.warmup();
       }
       await mongoMeasurement.start(uri);
-      const taskResult = await task.run();
+      InstrumentHooks.startBenchmark();
+      const taskResult = await InstrumentHooks.__codspeed_root_frame__(() =>
+        task.run()
+      );
+      InstrumentHooks.stopBenchmark();
       await mongoMeasurement.stop(uri);
       results.push(taskResult);
 
@@ -67,8 +72,85 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void {
         };
 
         benchmarks.push(benchmark);
+        console.log(`    ✔ Collected walltime data for ${uri}`);
+        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+      } else {
+        console.warn(`    ⚠ No result data available for ${uri}`);
+      }
+    }
+
+    // Write results to JSON file using core function
+    if (benchmarks.length > 0) {
+      writeWalltimeResults(benchmarks);
+    }
+
+    console.log(
+      `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.`
+    );
+    // Restore our custom run method
+    bench.run = originalRun;
+
+    return results;
+  };
+
+  bench.runSync = () => {
+    console.log(
+      `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)`
+    );
+
+    // Store the original run method before we override it
+    const originalRun = bench.run;
 
+    // Temporarily restore the original run to get actual benchmark results
+    const benchProto = Object.getPrototypeOf(bench);
+    const prototypeRun = benchProto.run;
+    bench.run = prototypeRun;
+
+    const benchmarks: Benchmark[] = [];
+
+    // Run the bench naturally to collect TaskResult data
+    const results = [];
+
+    // Collect and report walltime data
+    for (const task of bench.tasks) {
+      const uri = getTaskUri(bench, task.name, rootCallingFile);
+
+      // run the warmup of the task right before its actual run
+      if (bench.opts.warmup) {
+        task.warmup();
+      }
+      // await mongoMeasurement.start(uri);
+      InstrumentHooks.startBenchmark();
+      InstrumentHooks.__codspeed_root_frame__(() => task.runSync());
+      InstrumentHooks.stopBenchmark();
+      // await mongoMeasurement.stop(uri);
+      results.push(task);
+
+      if (task.result) {
+        // Convert tinybench result to BenchmarkStats format
+        const stats = convertTinybenchResultToBenchmarkStats(
+          task.result,
+          bench.opts.warmup ? bench.opts.warmupIterations ?? 0 : 0
+        );
+
+        const benchmark: Benchmark = {
+          name: task.name,
+          uri,
+          config: {
+            max_rounds: bench.opts.iterations ?? null,
+            max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null,
+            min_round_time_ns: null, // tinybench does not have an option for this
+            warmup_time_ns:
+              bench.opts.warmup && bench.opts.warmupTime
+                ? msToNs(bench.opts.warmupTime)
+                : null,
+          },
+          stats,
+        };
+
+        benchmarks.push(benchmark);
         console.log(`    ✔ Collected walltime data for ${uri}`);
+        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
       } else {
         console.warn(`    ⚠ No result data available for ${uri}`);
       }