From d958153c9d6b4f52f8eb565b53bafe3498b5bfa4 Mon Sep 17 00:00:00 2001 From: Adrien Cacciaguerra Date: Tue, 23 Apr 2024 17:32:10 -0400 Subject: [PATCH] feat: add CodSpeed to the project --- .github/workflows/benchmark.yml | 41 ++++++++++ Cargo.lock | 80 +++++++++++++++++-- Cargo.toml | 1 + crates/turbo-tasks-fs/Cargo.toml | 1 + crates/turbo-tasks-fs/benches/mod.rs | 2 +- crates/turbo-tasks-memory/Cargo.toml | 1 + crates/turbo-tasks-memory/benches/mod.rs | 2 +- .../benches/scope_stress.rs | 4 +- crates/turbo-tasks-memory/benches/stress.rs | 4 +- crates/turbopack-bench/Cargo.toml | 1 + crates/turbopack-bench/benches/mod.rs | 2 +- crates/turbopack-bench/src/lib.rs | 2 +- crates/turbopack-bench/src/util/mod.rs | 6 +- crates/turbopack-cli/Cargo.toml | 1 + crates/turbopack-cli/benches/mod.rs | 2 +- crates/turbopack-ecmascript/Cargo.toml | 1 + .../turbopack-ecmascript/benches/analyzer.rs | 2 +- crates/turbopack-ecmascript/benches/mod.rs | 2 +- crates/turbopack/Cargo.toml | 1 + crates/turbopack/benches/mod.rs | 2 +- crates/turbopack/benches/node_file_trace.rs | 2 +- 21 files changed, 139 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000000..6b6b22eb31978 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,41 @@ +name: codspeed-benchmarks + +on: + push: + branches: + - "main" # or "master" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +env: + TURBOPACK_BENCH_COUNTS: 50 + +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + cache-base: main + + - name: Build the benchmark target(s) + run: | + cargo codspeed build -p turbo-tasks-fs + cargo codspeed build -p turbo-tasks-memory + cargo codspeed build -p turbopack + # cargo codspeed build -p turbopack-bench # disabled for now because it uses a custom benchmark iterator + cargo codspeed build -p turbopack-cli + cargo codspeed build -p turbopack-ecmascript + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: cargo codspeed run diff --git a/Cargo.lock b/Cargo.lock index e7927028539d9..3c919232c8c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1561,6 +1561,30 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +[[package]] +name = "codspeed" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735f16ee0fc63cb90596cd7b57ce481522adfe1714f95bc04a94d4f4b0a06a6d" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572ca9c8ad460591b40aad63c99d6746aa3c532f979175344eb015389499860c" +dependencies = [ + "codspeed", + "colored", + "criterion 0.5.1", + "futures 0.3.30", + "tokio", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -1573,6 +1597,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "combine" version = "4.6.6" @@ -2020,6 +2054,34 @@ dependencies = [ "walkdir", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap 4.5.2", + "criterion-plot", + "futures 0.3.30", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + [[package]] name = "criterion-plot" version = "0.5.0" @@ -10614,8 +10676,9 @@ dependencies = [ "auto-hash-map", "bitflags 1.3.2", "bytes 1.5.0", + "codspeed-criterion-compat", "concurrent-queue", - "criterion", + "criterion 0.4.0", "dashmap", "dunce", "futures 0.3.30", @@ -10701,8 +10764,9 @@ version = "0.1.0" dependencies = [ "anyhow", "auto-hash-map", + "codspeed-criterion-compat", "concurrent-queue", - "criterion", + "criterion 0.4.0", "dashmap", "indexmap 1.9.3", "lazy_static", @@ -10755,7 +10819,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-recursion", - "criterion", + "codspeed-criterion-compat", + "criterion 0.4.0", "difference", "futures 0.3.30", "indexmap 1.9.3", @@ -10791,7 +10856,8 @@ version = "0.1.0" dependencies = [ "anyhow", "chromiumoxide", - "criterion", + "codspeed-criterion-compat", + "criterion 0.4.0", "futures 0.3.30", "nix 0.26.2", "once_cell", @@ -10890,8 +10956,9 @@ version = "0.1.0" dependencies = [ "anyhow", "clap 4.5.2", + "codspeed-criterion-compat", "console-subscriber", - "criterion", + "criterion 0.4.0", "dunce", "futures 0.3.30", "mime 0.3.17", @@ -11051,7 +11118,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "criterion", + "codspeed-criterion-compat", + "criterion 0.4.0", "either", "indexmap 1.9.3", "indoc", diff --git a/Cargo.toml b/Cargo.toml index 1d5e62f472e00..87c38bcb42bc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,6 +219,7 @@ concurrent-queue = "2.1.0" console = "0.15.5" console-subscriber = "0.1.8" criterion = "0.4.0" +codspeed-criterion-compat = "2.5.0" crossbeam-channel = "0.5.8" dashmap = "5.4.0" dialoguer = "0.10.3" diff --git a/crates/turbo-tasks-fs/Cargo.toml b/crates/turbo-tasks-fs/Cargo.toml index 27602ad932669..d4e1ab9937bdb 100644 --- a/crates/turbo-tasks-fs/Cargo.toml +++ b/crates/turbo-tasks-fs/Cargo.toml @@ -50,6 +50,7 @@ turbo-tasks-hash = { workspace = true } unicode-segmentation = { workspace = true } [dev-dependencies] +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } rstest = { workspace = true } sha2 = "0.10.2" diff --git a/crates/turbo-tasks-fs/benches/mod.rs b/crates/turbo-tasks-fs/benches/mod.rs index abb374f460a97..a43976de1d0f8 100644 --- a/crates/turbo-tasks-fs/benches/mod.rs +++ b/crates/turbo-tasks-fs/benches/mod.rs @@ -5,7 +5,7 @@ use std::{ time::{Duration, Instant}, }; -use criterion::{ +use codspeed_criterion_compat::{ criterion_group, criterion_main, measurement::{Measurement, WallTime}, BenchmarkId, Criterion, diff --git a/crates/turbo-tasks-memory/Cargo.toml b/crates/turbo-tasks-memory/Cargo.toml index ae1acec5a021e..363811c89941e 100644 --- a/crates/turbo-tasks-memory/Cargo.toml +++ b/crates/turbo-tasks-memory/Cargo.toml @@ -32,6 +32,7 @@ turbo-tasks-hash = { workspace = true } turbo-tasks-malloc = { workspace = true, default-features = false } [dev-dependencies] +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } indexmap = { workspace = true } lazy_static = { workspace = true } diff --git a/crates/turbo-tasks-memory/benches/mod.rs b/crates/turbo-tasks-memory/benches/mod.rs index 6bbaf50152aa7..11585e9b01b31 100644 --- a/crates/turbo-tasks-memory/benches/mod.rs +++ b/crates/turbo-tasks-memory/benches/mod.rs @@ -1,6 +1,6 @@ #![feature(arbitrary_self_types)] -use criterion::{criterion_group, criterion_main, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion}; pub(crate) mod scope_stress; pub(crate) mod stress; diff --git a/crates/turbo-tasks-memory/benches/scope_stress.rs b/crates/turbo-tasks-memory/benches/scope_stress.rs index 3c8d47338bff4..62bcaefae170d 100644 --- a/crates/turbo-tasks-memory/benches/scope_stress.rs +++ b/crates/turbo-tasks-memory/benches/scope_stress.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use criterion::{BenchmarkId, Criterion}; +use codspeed_criterion_compat::{BenchmarkId, Criterion}; use turbo_tasks::{Completion, TryJoinIterExt, TurboTasks, Vc}; use turbo_tasks_memory::MemoryBackend; @@ -19,7 +19,7 @@ pub fn scope_stress(c: &mut Criterion) { group.sample_size(20); for size in [5, 10, 15, 20, 25, 30, 100, 200, 300] { - group.throughput(criterion::Throughput::Elements( + group.throughput(codspeed_criterion_compat::Throughput::Elements( /* tasks for fib from 0 to size - 1 = */ (size as u64) * (size as u64) + /* root tasks = */ diff --git a/crates/turbo-tasks-memory/benches/stress.rs b/crates/turbo-tasks-memory/benches/stress.rs index f0482ab83e0d7..f322ca675294d 100644 --- a/crates/turbo-tasks-memory/benches/stress.rs +++ b/crates/turbo-tasks-memory/benches/stress.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use criterion::{BenchmarkId, Criterion}; +use codspeed_criterion_compat::{BenchmarkId, Criterion}; use turbo_tasks::{TryJoinIterExt, TurboTasks, Vc}; use turbo_tasks_memory::MemoryBackend; @@ -19,7 +19,7 @@ pub fn fibonacci(c: &mut Criterion) { group.sample_size(20); for size in [100, 200, 500, 1000, 1414] { - group.throughput(criterion::Throughput::Elements( + group.throughput(codspeed_criterion_compat::Throughput::Elements( /* tasks for fib from 0 to size - 1 = */ size as u64 * (size as u64 + 1) / 2 + /* root task = */ diff --git a/crates/turbopack-bench/Cargo.toml b/crates/turbopack-bench/Cargo.toml index 3646b0bba790e..f18f4ee546a0e 100644 --- a/crates/turbopack-bench/Cargo.toml +++ b/crates/turbopack-bench/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true, features = ["backtrace"] } chromiumoxide = { workspace = true, features = [ "tokio-runtime", ], default-features = false } +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } futures = { workspace = true } once_cell = { workspace = true } diff --git a/crates/turbopack-bench/benches/mod.rs b/crates/turbopack-bench/benches/mod.rs index fbb66c36f6d96..0c4f29d8ee3dc 100644 --- a/crates/turbopack-bench/benches/mod.rs +++ b/crates/turbopack-bench/benches/mod.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion}; use turbopack_bench::bundlers::Bundler; fn get_bundlers() -> Vec> { diff --git a/crates/turbopack-bench/src/lib.rs b/crates/turbopack-bench/src/lib.rs index 4e3df12db039f..c24ac62ec798b 100644 --- a/crates/turbopack-bench/src/lib.rs +++ b/crates/turbopack-bench/src/lib.rs @@ -10,7 +10,7 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; -use criterion::{ +use codspeed_criterion_compat::{ measurement::{Measurement, WallTime}, BenchmarkGroup, BenchmarkId, Criterion, }; diff --git a/crates/turbopack-bench/src/util/mod.rs b/crates/turbopack-bench/src/util/mod.rs index be077c334bfc4..dbe97cbdf67dd 100644 --- a/crates/turbopack-bench/src/util/mod.rs +++ b/crates/turbopack-bench/src/util/mod.rs @@ -12,7 +12,9 @@ use chromiumoxide::{ browser::{Browser, BrowserConfig}, error::CdpError::Ws, }; -use criterion::{async_executor::AsyncExecutor, black_box, measurement::WallTime, AsyncBencher}; +use codspeed_criterion_compat::{ + async_executor::AsyncExecutor, black_box, measurement::WallTime, AsyncBencher, +}; use futures::{Future, StreamExt}; pub use page_guard::PageGuard; use parking_lot::Mutex; @@ -155,7 +157,7 @@ pub trait AsyncBencherExtension { TF: Future; } -impl<'a, 'b, A: AsyncExecutor> AsyncBencherExtension for AsyncBencher<'a, 'b, A, WallTime> { +impl<'a, 'b, A: AsyncExecutor> AsyncBencherExtension for AsyncBencher<'a, 'b, A> { fn try_iter_custom(&mut self, routine: R) where R: Fn(u64, WallTime) -> F, diff --git a/crates/turbopack-cli/Cargo.toml b/crates/turbopack-cli/Cargo.toml index 59eba5c7ad509..1fcdf84d40f27 100644 --- a/crates/turbopack-cli/Cargo.toml +++ b/crates/turbopack-cli/Cargo.toml @@ -72,6 +72,7 @@ turbopack-trace-utils = { workspace = true } webbrowser = { workspace = true } [dev-dependencies] +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } regex = { workspace = true } turbopack-bench = { workspace = true } diff --git a/crates/turbopack-cli/benches/mod.rs b/crates/turbopack-cli/benches/mod.rs index 7236f5cc340b7..aed0ac9cc31ab 100644 --- a/crates/turbopack-cli/benches/mod.rs +++ b/crates/turbopack-cli/benches/mod.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion}; use turbopack_bench::bundlers::Bundler; mod bundler; diff --git a/crates/turbopack-ecmascript/Cargo.toml b/crates/turbopack-ecmascript/Cargo.toml index 105d36b868d8d..53405711c19c1 100644 --- a/crates/turbopack-ecmascript/Cargo.toml +++ b/crates/turbopack-ecmascript/Cargo.toml @@ -66,6 +66,7 @@ swc_core = { workspace = true, features = [ ] } [dev-dependencies] +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } rstest = { workspace = true } turbo-tasks-memory = { workspace = true } diff --git a/crates/turbopack-ecmascript/benches/analyzer.rs b/crates/turbopack-ecmascript/benches/analyzer.rs index d7e97abab698c..27477ab917955 100644 --- a/crates/turbopack-ecmascript/benches/analyzer.rs +++ b/crates/turbopack-ecmascript/benches/analyzer.rs @@ -1,6 +1,6 @@ use std::{fs, path::PathBuf, sync::Arc, time::Duration}; -use criterion::{Bencher, BenchmarkId, Criterion}; +use codspeed_criterion_compat::{Bencher, BenchmarkId, Criterion}; use swc_core::{ common::{FilePathMapping, Mark, SourceMap, GLOBALS}, ecma::{ diff --git a/crates/turbopack-ecmascript/benches/mod.rs b/crates/turbopack-ecmascript/benches/mod.rs index 48208446e41e4..8956bd3c76b38 100644 --- a/crates/turbopack-ecmascript/benches/mod.rs +++ b/crates/turbopack-ecmascript/benches/mod.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main}; +use codspeed_criterion_compat::{criterion_group, criterion_main}; mod analyzer; diff --git a/crates/turbopack/Cargo.toml b/crates/turbopack/Cargo.toml index ff161cc2c61c2..5534ec589e097 100644 --- a/crates/turbopack/Cargo.toml +++ b/crates/turbopack/Cargo.toml @@ -42,6 +42,7 @@ turbopack-static = { workspace = true } turbopack-wasm = { workspace = true } [dev-dependencies] +codspeed-criterion-compat = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] } difference = "2.0" futures = { workspace = true } diff --git a/crates/turbopack/benches/mod.rs b/crates/turbopack/benches/mod.rs index b1d1f6d977a1f..83ee815539f19 100644 --- a/crates/turbopack/benches/mod.rs +++ b/crates/turbopack/benches/mod.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main}; +use codspeed_criterion_compat::{criterion_group, criterion_main}; mod node_file_trace; diff --git a/crates/turbopack/benches/node_file_trace.rs b/crates/turbopack/benches/node_file_trace.rs index 0dd71a849bca4..02c3198157292 100644 --- a/crates/turbopack/benches/node_file_trace.rs +++ b/crates/turbopack/benches/node_file_trace.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, fs, path::PathBuf}; -use criterion::{Bencher, BenchmarkId, Criterion}; +use codspeed_criterion_compat::{Bencher, BenchmarkId, Criterion}; use regex::Regex; use turbo_tasks::{TurboTasks, Value, Vc}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};