--wip-- [skip ci]

Author: GuillaumeLagrange

packages/core/binding.gyp

@@ -16,8 +16,16 @@
         "-Wno-unused-but-set-variable",
         "-Wno-type-limits",
         "-Wno-format",
-        "-Wno-format-security"
+        "-Wno-format-security",
+        "-Wno-error=format-security"
       ],
+      "xcode_settings": {
+        "OTHER_CFLAGS": [
+          "-Wno-format",
+          "-Wno-format-security",
+          "-Wno-error=format-security"
+        ]
+      },
       "cflags_cc": [
         "-Wno-maybe-uninitialized",
         "-Wno-unused-variable",

packages/playwright/src/index.ts

@@ -9,6 +9,8 @@ import {
   type Benchmark,
   type BenchmarkStats,
 } from "@codspeed/core";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
 import { _electron as electron } from "playwright";
 import type { ElectronApplication, Page } from "playwright-core";
 
@@ -32,6 +34,11 @@ export interface BenchFixtures {
  */
 export type BenchFunction = (fixtures: BenchFixtures) => void | Promise<void>;
 
+/**
+ * A hook run around the measured region. Not included in the reported timing.
+ */
+export type BenchHook = (fixtures: BenchFixtures) => void | Promise<void>;
+
 /**
  * Minimal options for a benchmark. Inspired by Vitest's `bench`, but kept
  * deliberately small.
@@ -47,20 +54,45 @@ export interface BenchOptions {
    */
   appPath: string;
   /**
-   * Additional CLI flags forwarded to Electron.
+   * CLI flags forwarded to Electron.
    */
   electronArgs?: string[];
   /**
    * Working directory for the Electron process. Defaults to `process.cwd()`.
    */
   cwd?: string;
   /**
-   * Absolute path to the Electron executable. When omitted, Playwright resolves
-   * it via `require("electron")`. Set this when running under package managers
-   * (e.g. pnpm) where Playwright cannot resolve `electron` from its own
-   * directory.
+   * Absolute path to the Electron executable. When omitted, it is resolved from
+   * the `electron` package in `cwd`. Set this only to override that default.
+   */
+  electronExecutablePath?: string;
+  /**
+   * Run before each round, after the window opens. Use it to bring the app to
+   * a steady state (initial render done, data loaded, …). Not measured.
+   */
+  setup?: BenchHook;
+  /**
+   * Run after each round, before the app is closed. Use it for teardown that
+   * should not be measured.
    */
-  executablePath?: string;
+  teardown?: BenchHook;
+}
+
+let integrationInitialized = false;
+
+/**
+ * Register the integration and environment with the instrumentation. This is
+ * process-global, so it only needs to run once regardless of how many
+ * benchmarks are defined.
+ */
+function ensureIntegrationSetup(): void {
+  if (integrationInitialized) return;
+  integrationInitialized = true;
+
+  InstrumentHooks.setIntegration("node-custom", __VERSION__);
+  InstrumentHooks.setEnvironment("nodejs", "version", process.versions.node);
+  InstrumentHooks.setEnvironment("nodejs", "v8", process.versions.v8);
+  InstrumentHooks.writeEnvironment(process.pid);
 }
 
 function resolveRounds(optionRounds: number | undefined): number {
@@ -74,15 +106,32 @@ function resolveRounds(optionRounds: number | undefined): number {
   return n;
 }
 
+/**
+ * Resolve the path to the Electron binary.
+ *
+ * Playwright resolves Electron via `require("electron/index.js")` from inside
+ * its own package directory. Under isolated installs (e.g. pnpm), Playwright
+ * cannot see the project's `electron` dependency and bails out with
+ * "Electron executablePath not found!". We resolve it ourselves from the
+ * benchmark's working directory, where `electron` is a real dependency.
+ */
+function resolveElectronExecutable(cwd: string): string {
+  const require = createRequire(pathToFileURL(`${cwd}/`));
+  // `electron`'s main module exports the absolute path to its binary.
+  return require("electron") as string;
+}
+
 async function launchApp(options: BenchOptions): Promise<ElectronApplication> {
+  const cwd = options.cwd ?? process.cwd();
   return electron.launch({
     args: [
       options.appPath,
       ...(options.electronArgs ?? []),
       `--js-flags=${DEFAULT_PROFILING_JS_FLAGS}`,
     ],
-    cwd: options.cwd ?? process.cwd(),
-    executablePath: options.executablePath,
+    cwd,
+    executablePath:
+      options.electronExecutablePath ?? resolveElectronExecutable(cwd),
   });
 }
 
@@ -94,6 +143,10 @@ async function runOneSample(
   const page = await app.firstWindow();
 
   try {
+    if (options.setup) {
+      await options.setup({ page });
+    }
+
     const startTs = InstrumentHooks.currentTimestamp();
     await fn({ page });
     const endTs = InstrumentHooks.currentTimestamp();
@@ -105,6 +158,10 @@ async function runOneSample(
     );
     InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, endTs);
 
+    if (options.teardown) {
+      await options.teardown({ page });
+    }
+
     return endTs - startTs;
   } finally {
     await app.close();
@@ -177,10 +234,7 @@ export async function bench(
   const rounds = resolveRounds(options.rounds);
   const uri = `${getCallingFile(0)}::${name}`;
 
-  InstrumentHooks.setIntegration("node-custom", __VERSION__);
-  InstrumentHooks.setEnvironment("nodejs", "version", process.versions.node);
-  InstrumentHooks.setEnvironment("nodejs", "v8", process.versions.v8);
-  InstrumentHooks.writeEnvironment(process.pid);
+  ensureIntegrationSetup();
 
   InstrumentHooks.setExecutedBenchmark(process.pid, uri);
   InstrumentHooks.startBenchmark();