--wip-- [skip ci]

.github/workflows/benchmark.yml

@@ -0,0 +1,41 @@
+name: codspeed-benchmarks
+
+on:
+  push:
+    branches:
+      - "main" # or "master"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+env:
+  TURBOPACK_BENCH_COUNTS: 50
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup rust toolchain, cache and cargo-codspeed binary
+        uses: moonrepo/setup-rust@v1
+        with:
+          channel: stable
+          cache-target: release
+          bins: cargo-codspeed
+          cache-base: main
+
+      - name: Build the benchmark target(s)
+        run: |
+          cargo codspeed build -p turbo-tasks-fs
+          cargo codspeed build -p turbo-tasks-memory
+          cargo codspeed build -p turbopack
+          # cargo codspeed build -p turbopack-bench    # disabled for now because it uses a custom benchmark iterator
+          cargo codspeed build -p turbopack-cli
+          cargo codspeed build -p turbopack-ecmascript
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: cargo codspeed run

Cargo.toml

@@ -219,6 +219,7 @@ concurrent-queue = "2.1.0"
 console = "0.15.5"
 console-subscriber = "0.1.8"
 criterion = "0.4.0"
+codspeed-criterion-compat = "2.5.0"
 crossbeam-channel = "0.5.8"
 dashmap = "5.4.0"
 dialoguer = "0.10.3"

crates/turbo-tasks-fs/Cargo.toml

@@ -50,6 +50,7 @@ turbo-tasks-hash = { workspace = true }
 unicode-segmentation = { workspace = true }
 
 [dev-dependencies]
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 rstest = { workspace = true }
 sha2 = "0.10.2"

crates/turbo-tasks-fs/benches/mod.rs

@@ -5,7 +5,7 @@ use std::{
     time::{Duration, Instant},
 };
 
-use criterion::{
+use codspeed_criterion_compat::{
     criterion_group, criterion_main,
     measurement::{Measurement, WallTime},
     BenchmarkId, Criterion,

crates/turbo-tasks-memory/Cargo.toml

@@ -32,6 +32,7 @@ turbo-tasks-hash = { workspace = true }
 turbo-tasks-malloc = { workspace = true, default-features = false }
 
 [dev-dependencies]
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 indexmap = { workspace = true }
 lazy_static = { workspace = true }

crates/turbo-tasks-memory/benches/mod.rs

@@ -1,6 +1,6 @@
 #![feature(arbitrary_self_types)]
 
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
 
 pub(crate) mod scope_stress;
 pub(crate) mod stress;

crates/turbo-tasks-memory/benches/scope_stress.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
-use criterion::{BenchmarkId, Criterion};
+use codspeed_criterion_compat::{BenchmarkId, Criterion};
 use turbo_tasks::{Completion, TryJoinIterExt, TurboTasks, Vc};
 use turbo_tasks_memory::MemoryBackend;
 
@@ -19,7 +19,7 @@ pub fn scope_stress(c: &mut Criterion) {
     group.sample_size(20);
 
     for size in [5, 10, 15, 20, 25, 30, 100, 200, 300] {
-        group.throughput(criterion::Throughput::Elements(
+        group.throughput(codspeed_criterion_compat::Throughput::Elements(
             /* tasks for fib from 0 to size - 1 = */
             (size as u64) * (size as u64) +
             /* root tasks = */

crates/turbo-tasks-memory/benches/stress.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
-use criterion::{BenchmarkId, Criterion};
+use codspeed_criterion_compat::{BenchmarkId, Criterion};
 use turbo_tasks::{TryJoinIterExt, TurboTasks, Vc};
 use turbo_tasks_memory::MemoryBackend;
 
@@ -19,7 +19,7 @@ pub fn fibonacci(c: &mut Criterion) {
     group.sample_size(20);
 
     for size in [100, 200, 500, 1000, 1414] {
-        group.throughput(criterion::Throughput::Elements(
+        group.throughput(codspeed_criterion_compat::Throughput::Elements(
             /* tasks for fib from 0 to size - 1 = */
             size as u64 * (size as u64 + 1) / 2 +
             /* root task = */

crates/turbopack-bench/Cargo.toml

@@ -21,6 +21,7 @@ anyhow = { workspace = true, features = ["backtrace"] }
 chromiumoxide = { workspace = true, features = [
   "tokio-runtime",
 ], default-features = false }
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 futures = { workspace = true }
 once_cell = { workspace = true }

crates/turbopack-bench/benches/mod.rs

@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
 use turbopack_bench::bundlers::Bundler;
 
 fn get_bundlers() -> Vec<Box<dyn Bundler>> {

crates/turbopack-bench/src/lib.rs

@@ -10,7 +10,7 @@ use std::{
 };
 
 use anyhow::{anyhow, Context, Result};
-use criterion::{
+use codspeed_criterion_compat::{
     measurement::{Measurement, WallTime},
     BenchmarkGroup, BenchmarkId, Criterion,
 };

crates/turbopack-bench/src/util/mod.rs

@@ -12,7 +12,9 @@ use chromiumoxide::{
     browser::{Browser, BrowserConfig},
     error::CdpError::Ws,
 };
-use criterion::{async_executor::AsyncExecutor, black_box, measurement::WallTime, AsyncBencher};
+use codspeed_criterion_compat::{
+    async_executor::AsyncExecutor, black_box, measurement::WallTime, AsyncBencher,
+};
 use futures::{Future, StreamExt};
 pub use page_guard::PageGuard;
 use parking_lot::Mutex;
@@ -155,7 +157,7 @@ pub trait AsyncBencherExtension<A: AsyncExecutor> {
         TF: Future<Output = ()>;
 }
 
-impl<'a, 'b, A: AsyncExecutor> AsyncBencherExtension<A> for AsyncBencher<'a, 'b, A, WallTime> {
+impl<'a, 'b, A: AsyncExecutor> AsyncBencherExtension<A> for AsyncBencher<'a, 'b, A> {
     fn try_iter_custom<R, F>(&mut self, routine: R)
     where
         R: Fn(u64, WallTime) -> F,

crates/turbopack-cli/Cargo.toml

@@ -72,6 +72,7 @@ turbopack-trace-utils = { workspace = true }
 webbrowser = { workspace = true }
 
 [dev-dependencies]
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 regex = { workspace = true }
 turbopack-bench = { workspace = true }

crates/turbopack-cli/benches/mod.rs

@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
 use turbopack_bench::bundlers::Bundler;
 
 mod bundler;

crates/turbopack-ecmascript/Cargo.toml

@@ -66,6 +66,7 @@ swc_core = { workspace = true, features = [
 ] }
 
 [dev-dependencies]
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 rstest = { workspace = true }
 turbo-tasks-memory = { workspace = true }

crates/turbopack-ecmascript/benches/analyzer.rs

@@ -1,6 +1,6 @@
 use std::{fs, path::PathBuf, sync::Arc, time::Duration};
 
-use criterion::{Bencher, BenchmarkId, Criterion};
+use codspeed_criterion_compat::{Bencher, BenchmarkId, Criterion};
 use swc_core::{
     common::{FilePathMapping, Mark, SourceMap, GLOBALS},
     ecma::{

crates/turbopack-ecmascript/benches/mod.rs

@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main};
+use codspeed_criterion_compat::{criterion_group, criterion_main};
 
 mod analyzer;
 

crates/turbopack/Cargo.toml

@@ -42,6 +42,7 @@ turbopack-static = { workspace = true }
 turbopack-wasm = { workspace = true }
 
 [dev-dependencies]
+codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] }
 criterion = { workspace = true, features = ["async_tokio"] }
 difference = "2.0"
 futures = { workspace = true }

crates/turbopack/benches/mod.rs

@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main};
+use codspeed_criterion_compat::{criterion_group, criterion_main};
 
 mod node_file_trace;
 

crates/turbopack/benches/node_file_trace.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, fs, path::PathBuf};
 
-use criterion::{Bencher, BenchmarkId, Criterion};
+use codspeed_criterion_compat::{Bencher, BenchmarkId, Criterion};
 use regex::Regex;
 use turbo_tasks::{TurboTasks, Value, Vc};
 use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};