feat(tinybench-plugin): bump tinybench and add walltime support

Author: GuillaumeLagrange

examples/with-javascript-cjs/package.json

@@ -9,6 +9,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-javascript-esm/package.json

@@ -10,6 +10,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-typescript-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-esm/package.json

@@ -14,7 +14,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
     "vitest": "^1.2.2"
   }

examples/with-typescript-simple-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -12,7 +12,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

packages/core/src/index.ts

@@ -10,6 +10,26 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
+export enum MeasurementMode {
+  Instrumentation = "instrumentation",
+  WallTime = "walltime",
+}
+
+export function getMeasurementMode(): MeasurementMode {
+  const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
+  if (isCodSpeedEnabled) {
+    // If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
+    if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
+      return MeasurementMode.WallTime;
+    } else {
+      return MeasurementMode.Instrumentation;
+    }
+  }
+
+  // Default to walltime mode when CODSPEED_ENV is not set
+  return MeasurementMode.WallTime;
+}
+
 export const setupCore = () => {
   native_core.Measurement.stopInstrumentation(
     `Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +49,5 @@ export type {
 export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
+export * from "./walltime";
 export const Measurement = native_core.Measurement;

packages/tinybench-plugin/moon.yml

@@ -1,6 +1,6 @@
 tasks:
   bench:
-    command: node -r esbuild-register benches/sample.ts
+    command: node --loader esbuild-register/loader -r esbuild-register benches/sample.ts
     inputs:
       - "benches/**"
     local: true

packages/tinybench-plugin/package.json

@@ -26,14 +26,15 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@types/stack-trace": "^0.0.30",
-    "tinybench": "^2.5.0",
+    "esbuild-register": "^3.4.2",
+    "tinybench": "^4.0.1",
     "vitest": "^1.2.2"
   },
   "dependencies": {
     "@codspeed/core": "workspace:^4.0.1",
     "stack-trace": "1.0.0-pre2"
   },
   "peerDependencies": {
-    "tinybench": "^2.3.0"
+    "tinybench": "^4.0.1"
   }
 }

packages/tinybench-plugin/src/index.ts

@@ -1,94 +1,64 @@
 import {
   getGitDir,
+  getMeasurementMode,
   Measurement,
+  MeasurementMode,
   mongoMeasurement,
-  optimizeFunction,
-  setupCore,
   SetupInstrumentsRequestBody,
   SetupInstrumentsResponse,
-  teardownCore,
   tryIntrospect,
 } from "@codspeed/core";
 import path from "path";
 import { get as getStackTrace } from "stack-trace";
-import { Bench, Task } from "tinybench";
+import { Bench } from "tinybench";
 import { fileURLToPath } from "url";
-
-declare const __VERSION__: string;
+import { runInstrumentedBench } from "./instrumented";
+import { runWalltimeBench } from "./walltime";
 
 tryIntrospect();
 
-type CodSpeedBenchOptions = Task["opts"] & {
-  uri: string;
-};
-
-function isCodSpeedBenchOptions(
-  options: Task["opts"]
-): options is CodSpeedBenchOptions {
-  return "uri" in options;
-}
+// Store URI mapping externally since fnOpts is private
+export const taskUriMap = new WeakMap<Bench, Map<string, string>>();
 
 export function withCodSpeed(bench: Bench): Bench {
-  if (!Measurement.isInstrumented()) {
+  const measurementMode = getMeasurementMode();
+  const rootCallingFile = getCallingFile();
+
+  // Initialize URI mapping for this bench instance
+  if (!taskUriMap.has(bench)) {
+    taskUriMap.set(bench, new Map());
+  }
+  const uriMap = taskUriMap.get(bench)!;
+
+  // Setup URI generation for tasks
+  const rawAdd = bench.add;
+  bench.add = (name, fn, opts?) => {
+    const callingFile = getCallingFile();
+    const uri = `${callingFile}::${name}`;
+    // Store URI mapping
+    uriMap.set(name, uri);
+    return rawAdd.bind(bench)(name, fn, opts);
+  };
+
+  // Apply the appropriate measurement strategy based on mode and instrumentation
+  if (
+    measurementMode === MeasurementMode.Instrumentation &&
+    Measurement.isInstrumented()
+  ) {
+    runInstrumentedBench(bench, rootCallingFile);
+  } else if (measurementMode === MeasurementMode.WallTime) {
+    runWalltimeBench(bench, rootCallingFile);
+  } else {
+    // Fallback: instrumentation requested but not available
     const rawRun = bench.run;
     bench.run = async () => {
       console.warn(
         `[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench`
       );
       return await rawRun.bind(bench)();
     };
-    return bench;
   }
 
-  const rawAdd = bench.add;
-  bench.add = (name, fn, opts: CodSpeedBenchOptions) => {
-    const callingFile = getCallingFile();
-    const uri = `${callingFile}::${name}`;
-    const options = Object.assign({}, opts ?? {}, { uri });
-    return rawAdd.bind(bench)(name, fn, options);
-  };
-  const rootCallingFile = getCallingFile();
-
-  bench.run = async () => {
-    console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`);
-    setupCore();
-    for (const task of bench.tasks) {
-      const uri = isCodSpeedBenchOptions(task.opts)
-        ? task.opts.uri
-        : `${rootCallingFile}::${task.name}`;
-
-      await task.opts.beforeAll?.call(task);
-
-      // run optimizations
-      await optimizeFunction(async () => {
-        await task.opts.beforeEach?.call(task);
-        await task.fn();
-        await task.opts.afterEach?.call(task);
-      });
-
-      // run instrumented benchmark
-      await task.opts.beforeEach?.call(task);
-
-      await mongoMeasurement.start(uri);
-      global.gc?.();
-      await (async function __codspeed_root_frame__() {
-        Measurement.startInstrumentation();
-        await task.fn();
-        Measurement.stopInstrumentation(uri);
-      })();
-      await mongoMeasurement.stop(uri);
-
-      await task.opts.afterEach?.call(task);
-
-      await task.opts.afterAll?.call(task);
-
-      // print results
-      console.log(`    ✔ Measured ${uri}`);
-    }
-    teardownCore();
-    console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
-    return bench.tasks;
-  };
   return bench;
 }