feat: add memory profiling support

not-matthias · not-matthias · commit 5f0f8ccc97ac · 2026-01-29T19:44:11.000+01:00
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -23,18 +23,30 @@ jobs:
       - run: pnpm install --frozen-lockfile --prefer-offline
       - run: pnpm moon run :build
 
-      - name: Run benchmarks
-        # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
+      - name: Run simulation benchmarks
+        uses: CodSpeedHQ/action@main
+        with:
+          mode: simulation
+          run: |
+            pnpm moon run tinybench-plugin:bench
+            pnpm moon run vitest-plugin:bench
+            pnpm moon run benchmark.js-plugin:bench
+            pnpm --workspace-concurrency 1 -r bench-tinybench
+            pnpm --workspace-concurrency 1 -r bench-benchmark-js
+            pnpm --workspace-concurrency 1 -r bench-vitest
+
+      - name: Run memory benchmarks
         uses: CodSpeedHQ/action@main
         with:
-          mode: instrumentation
+          mode: memory
           run: |
             pnpm moon run tinybench-plugin:bench
             pnpm moon run vitest-plugin:bench
             pnpm moon run benchmark.js-plugin:bench
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-benchmark-js
             pnpm --workspace-concurrency 1 -r bench-vitest
+          upload-url: https://api.staging.preview.codspeed.io/upload
 
   codspeed-walltime:
     name: Run CodSpeed walltime
diff --git a/packages/core/binding.gyp b/packages/core/binding.gyp
@@ -14,7 +14,9 @@
         "-Wno-unused-variable",
         "-Wno-unused-parameter",
         "-Wno-unused-but-set-variable",
-        "-Wno-type-limits"
+        "-Wno-type-limits",
+        "-Wno-format",
+        "-Wno-format-security"
       ],
       "cflags_cc": [
         "-Wno-maybe-uninitialized",
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
@@ -10,7 +10,9 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
-type CodSpeedRunnerMode = "disabled" | "simulation" | "walltime";
+type CodSpeedRunnerMode = "disabled" | "simulation" | "memory" | "walltime";
+
+type BuildMode = "disabled" | "analysis" | "walltime";
 
 export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
   const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
@@ -25,6 +27,8 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
     codspeedRunnerMode === "simulation"
   ) {
     return "simulation";
+  } else if (codspeedRunnerMode === "memory") {
+    return "memory";
   } else if (codspeedRunnerMode === "walltime") {
     return "walltime";
   }
@@ -35,6 +39,15 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
   return "disabled";
 }
 
+export function getBuildMode(): BuildMode {
+  const runnerMode = getCodspeedRunnerMode();
+  // Both "simulation" and "memory" map to "analysis" build mode
+  if (runnerMode === "simulation" || runnerMode === "memory") {
+    return "analysis";
+  }
+  return runnerMode; // "disabled" or "walltime"
+}
+
 export const setupCore = () => {
   if (!native_core.isBound) {
     throw new Error(
@@ -55,6 +68,7 @@ export type {
   SetupInstrumentsRequestBody,
   SetupInstrumentsResponse,
 } from "./generated/openapi";
+export type { BuildMode };
 export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts
@@ -1,15 +1,15 @@
 import { writeFileSync } from "fs";
-import { getCodspeedRunnerMode } from ".";
+import { getBuildMode } from ".";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
 export const getV8Flags = () => {
   const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
-  const codspeedRunnerMode = getCodspeedRunnerMode();
+  const buildMode = getBuildMode();
 
   const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
 
-  if (codspeedRunnerMode === "simulation") {
+  if (buildMode === "analysis") {
     flags.push(
       ...[
         "--hash-seed=1",
diff --git a/packages/core/src/native_core/instruments/hooks b/packages/core/src/native_core/instruments/hooks
@@ -1 +1 @@
-Subproject commit b3d4b78ef68498d59afd45a9c8b2ef096652f034
+Subproject commit 69b88f784173a3a244f09d3ded127fc28cc49fb1
diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts
@@ -6,17 +6,17 @@ import {
 import { Bench, Fn, FnOptions, Task } from "tinybench";
 import { BaseBenchRunner } from "./shared";
 
-export function setupCodspeedSimulationBench(
+export function setupCodspeedAnalysisBench(
   bench: Bench,
   rootCallingFile: string
 ): void {
-  const runner = new SimulationBenchRunner(bench, rootCallingFile);
+  const runner = new AnalysisBenchRunner(bench, rootCallingFile);
   runner.setupBenchMethods();
 }
 
-class SimulationBenchRunner extends BaseBenchRunner {
+class AnalysisBenchRunner extends BaseBenchRunner {
   protected getModeName(): string {
-    return "simulation mode";
+    return "analysis mode";
   }
 
   private taskCompletionMessage() {
diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts
@@ -1,4 +1,5 @@
 import {
+  getBuildMode,
   getCodspeedRunnerMode,
   getGitDir,
   InstrumentHooks,
@@ -11,7 +12,7 @@ import path from "path";
 import { get as getStackTrace } from "stack-trace";
 import { Bench } from "tinybench";
 import { fileURLToPath } from "url";
-import { setupCodspeedSimulationBench } from "./simulation";
+import { setupCodspeedAnalysisBench } from "./analysis";
 import { getOrCreateUriMap } from "./uri";
 import { setupCodspeedWalltimeBench } from "./walltime";
 
@@ -39,9 +40,10 @@ export function withCodSpeed(bench: Bench): Bench {
     return rawAdd.bind(bench)(name, fn, opts);
   };
 
-  if (codspeedRunnerMode === "simulation") {
-    setupCodspeedSimulationBench(bench, rootCallingFile);
-  } else if (codspeedRunnerMode === "walltime") {
+  const buildMode = getBuildMode();
+  if (buildMode === "analysis") {
+    setupCodspeedAnalysisBench(bench, rootCallingFile);
+  } else if (buildMode === "walltime") {
     setupCodspeedWalltimeBench(bench, rootCallingFile);
   }
 
diff --git a/packages/tinybench-plugin/src/index.unit.test.ts b/packages/tinybench-plugin/src/index.unit.test.ts
@@ -2,12 +2,12 @@ import { Bench } from "tinybench";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { withCodSpeed } from ".";
 
-const mockSimulation = vi.hoisted(() => ({
-  setupCodspeedSimulationBench: vi.fn(),
+const mockAnalysis = vi.hoisted(() => ({
+  setupCodspeedAnalysisBench: vi.fn(),
 }));
 
-vi.mock("./simulation", () => ({
-  ...mockSimulation,
+vi.mock("./analysis", () => ({
+  ...mockAnalysis,
 }));
 
 const mockWalltime = vi.hoisted(() => ({
@@ -38,23 +38,33 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
     expect(shouldBeCalled.mock.calls.length).toBeGreaterThan(1000);
   });
 
-  it("should run in simulation mode when CODSPEED_RUNNER_MODE=instrumentation", async () => {
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=instrumentation", async () => {
     process.env.CODSPEED_ENV = "true";
     process.env.CODSPEED_RUNNER_MODE = "instrumentation";
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
   });
 
-  it("should run in simulation mode when CODSPEED_RUNNER_MODE=simulation", async () => {
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=simulation", async () => {
     process.env.CODSPEED_ENV = "true";
     process.env.CODSPEED_RUNNER_MODE = "simulation";
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
+    expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
+  });
+
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=memory", async () => {
+    process.env.CODSPEED_ENV = "true";
+    process.env.CODSPEED_RUNNER_MODE = "memory";
+
+    withCodSpeed(new Bench());
+
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
   });
 
@@ -64,7 +74,7 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).not.toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).not.toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).toHaveBeenCalled();
   });
 });
diff --git a/packages/vitest-plugin/rollup.config.ts b/packages/vitest-plugin/rollup.config.ts
@@ -21,8 +21,8 @@ export default defineConfig([
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/simulation.ts",
-    output: { file: "dist/simulation.mjs", format: "es" },
+    input: "src/analysis.ts",
+    output: { file: "dist/analysis.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },
diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts
@@ -107,7 +107,7 @@ describe("codSpeedPlugin", () => {
           },
         },
         runner: expect.stringContaining(
-          "packages/vitest-plugin/src/simulation.ts"
+          "packages/vitest-plugin/src/analysis.ts"
         ),
       },
     });
diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts
@@ -1,7 +1,7 @@
 import { fromPartial } from "@total-typescript/shoehorn";
 import { describe, expect, it, vi, type RunnerTestSuite } from "vitest";
 import { getBenchFn } from "vitest/suite";
-import { SimulationRunner as CodSpeedRunner } from "../simulation";
+import { AnalysisRunner as CodSpeedRunner } from "../analysis";
 
 const coreMocks = vi.hoisted(() => {
   return {
diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/analysis.ts
@@ -18,7 +18,7 @@ import {
 const currentFileName =
   typeof __filename === "string"
     ? __filename
-    : new URL("simulation.mjs", import.meta.url).pathname;
+    : new URL("analysis.mjs", import.meta.url).pathname;
 
 /**
  * @deprecated
@@ -29,7 +29,7 @@ function logCodSpeed(message: string) {
   console.log(`[CodSpeed] ${message}`);
 }
 
-async function runSimulationBench(
+async function runAnalysisBench(
   benchmark: Benchmark,
   suite: RunnerTestSuite,
   currentSuiteName: string
@@ -60,7 +60,7 @@ async function runSimulationBench(
   logCodSpeed(`${uri} done`);
 }
 
-async function runSimulationBenchmarkSuite(
+async function runAnalysisBenchmarkSuite(
   suite: RunnerTestSuite,
   parentSuiteName?: string
 ) {
@@ -74,28 +74,28 @@ async function runSimulationBenchmarkSuite(
     if (task.mode !== "run") continue;
 
     if (isVitestTaskBenchmark(task)) {
-      await runSimulationBench(task, suite, currentSuiteName);
+      await runAnalysisBench(task, suite, currentSuiteName);
     } else if (task.type === "suite") {
-      await runSimulationBenchmarkSuite(task, currentSuiteName);
+      await runAnalysisBenchmarkSuite(task, currentSuiteName);
     }
   }
 
   await callSuiteHook(suite, suite, "afterAll");
 }
 
-export class SimulationRunner extends NodeBenchmarkRunner {
+export class AnalysisRunner extends NodeBenchmarkRunner {
   async runSuite(suite: RunnerTestSuite): Promise<void> {
     logDebug(`PROCESS PID: ${process.pid} in ${currentFileName}`);
     setupCore();
 
     patchRootSuiteWithFullFilePath(suite);
 
     logCodSpeed(`running suite ${suite.name}`);
-    await runSimulationBenchmarkSuite(suite);
+    await runAnalysisBenchmarkSuite(suite);
     logCodSpeed(`running suite ${suite.name} done`);
 
     teardownCore();
   }
 }
 
-export default SimulationRunner;
+export default AnalysisRunner;
diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts
@@ -1,4 +1,5 @@
 import {
+  getBuildMode,
   getCodspeedRunnerMode,
   getV8Flags,
   InstrumentHooks,
@@ -21,12 +22,12 @@ function getCodSpeedFileFromName(name: string) {
 }
 
 function getRunnerFile(): string | undefined {
-  const codspeedRunnerMode = getCodspeedRunnerMode();
-  if (codspeedRunnerMode === "disabled") {
+  const buildMode = getBuildMode();
+  if (buildMode === "disabled") {
     return undefined;
   }
 
-  return getCodSpeedFileFromName(codspeedRunnerMode);
+  return getCodSpeedFileFromName(buildMode);
 }
 
 export default function codspeedPlugin(): Plugin {
@@ -37,7 +38,7 @@ export default function codspeedPlugin(): Plugin {
         return false;
       }
       if (
-        getCodspeedRunnerMode() == "simulation" &&
+        getBuildMode() == "analysis" &&
         !InstrumentHooks.isInstrumented()
       ) {
         console.warn("[CodSpeed] bench detected but no instrumentation found");
diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts
@@ -1,3 +1,3 @@
-import { SimulationRunner } from "./simulation";
+import { AnalysisRunner } from "./analysis";
 
-export default SimulationRunner;
+export default AnalysisRunner;
