feat: allow defining targets in codspeed yaml

Author: GuillaumeLagrange

src/exec/mod.rs

@@ -9,6 +9,7 @@ use crate::run::uploader::UploadResult;
 use clap::Args;
 use std::path::Path;
 
+pub mod multi_targets;
 mod poll_results;
 
 /// We temporarily force this name for all exec runs
@@ -78,8 +79,22 @@ pub async fn run(
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
     let merged_args = args.merge_with_project_config(project_config);
-
     let config = crate::executor::Config::try_from(merged_args)?;
+
+    execute_with_harness(config, api_client, codspeed_config, setup_cache_dir).await
+}
+
+/// Core execution logic for exec-harness based runs.
+///
+/// This function handles exec-harness installation and benchmark execution with exec-style
+/// result polling. It is used by both `codspeed exec` directly and by `codspeed run` when
+/// executing targets defined in codspeed.yaml.
+pub async fn execute_with_harness(
+    config: crate::executor::Config,
+    api_client: &CodSpeedAPIClient,
+    codspeed_config: &CodSpeedConfig,
+    setup_cache_dir: Option<&Path>,
+) -> Result<()> {
     let mut execution_context = executor::ExecutionContext::try_from((config, codspeed_config))?;
     debug!("config: {:#?}", execution_context.config);
     let executor = executor::get_executor_from_mode(

src/exec/multi_targets.rs

@@ -0,0 +1,84 @@
+use super::EXEC_HARNESS_COMMAND;
+use crate::prelude::*;
+use crate::project_config::Target;
+use crate::project_config::WalltimeOptions;
+use exec_harness::BenchmarkCommand;
+
+/// Convert targets from project config to exec-harness JSON input format
+pub fn targets_to_exec_harness_json(
+    targets: &[Target],
+    default_walltime: Option<&WalltimeOptions>,
+) -> Result<String> {
+    let inputs: Vec<BenchmarkCommand> = targets
+        .iter()
+        .map(|target| {
+            // Parse the exec string into command parts
+            let command = shell_words::split(&target.exec)
+                .with_context(|| format!("Failed to parse command: {}", target.exec))?;
+
+            // Merge target-specific walltime options with defaults
+            let target_walltime = target.options.as_ref().and_then(|o| o.walltime.as_ref());
+            let walltime_args = merge_walltime_options(default_walltime, target_walltime);
+
+            Ok(BenchmarkCommand {
+                command,
+                name: target.name.clone(),
+                walltime_args,
+            })
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    serde_json::to_string(&inputs).context("Failed to serialize targets to JSON")
+}
+
+/// Merge default walltime options with target-specific overrides
+fn merge_walltime_options(
+    default: Option<&WalltimeOptions>,
+    target: Option<&WalltimeOptions>,
+) -> exec_harness::walltime::WalltimeExecutionArgs {
+    let default_args = default.map(walltime_options_to_args);
+    let target_args = target.map(walltime_options_to_args);
+
+    match (default_args, target_args) {
+        (None, None) => exec_harness::walltime::WalltimeExecutionArgs::default(),
+        (Some(d), None) => d,
+        (None, Some(t)) => t,
+        (Some(d), Some(t)) => exec_harness::walltime::WalltimeExecutionArgs {
+            warmup_time: t.warmup_time.or(d.warmup_time),
+            max_time: t.max_time.or(d.max_time),
+            min_time: t.min_time.or(d.min_time),
+            max_rounds: t.max_rounds.or(d.max_rounds),
+            min_rounds: t.min_rounds.or(d.min_rounds),
+        },
+    }
+}
+
+/// Convert project config WalltimeOptions to exec-harness WalltimeExecutionArgs
+fn walltime_options_to_args(
+    opts: &WalltimeOptions,
+) -> exec_harness::walltime::WalltimeExecutionArgs {
+    exec_harness::walltime::WalltimeExecutionArgs {
+        warmup_time: opts.warmup_time.clone(),
+        max_time: opts.max_time.clone(),
+        min_time: opts.min_time.clone(),
+        max_rounds: opts.max_rounds,
+        min_rounds: opts.min_rounds,
+    }
+}
+
+/// Build a command that pipes targets JSON to exec-harness via stdin
+pub fn build_pipe_command(
+    targets: &[Target],
+    default_walltime: Option<&WalltimeOptions>,
+) -> Result<Vec<String>> {
+    let json = targets_to_exec_harness_json(targets, default_walltime)?;
+    // Use a heredoc to safely pass the JSON to exec-harness
+    Ok(vec![
+        EXEC_HARNESS_COMMAND.to_owned(),
+        "-".to_owned(),
+        "<<".to_owned(),
+        "'CODSPEED_EOF'\n".to_owned(),
+        json,
+        "\nCODSPEED_EOF".to_owned(),
+    ])
+}

src/executor/config.rs

@@ -125,18 +125,16 @@ impl TryFrom<RunArgs> for Config {
     }
 }
 
-impl TryFrom<crate::exec::ExecArgs> for Config {
-    type Error = Error;
-    fn try_from(args: crate::exec::ExecArgs) -> Result<Self> {
+impl Config {
+    /// Create a Config from ExecArgs with a custom command (used for targets mode)
+    pub fn try_from_with_command(args: crate::exec::ExecArgs, command: String) -> Result<Self> {
         let raw_upload_url = args
             .shared
             .upload_url
             .unwrap_or_else(|| DEFAULT_UPLOAD_URL.into());
         let upload_url = Url::parse(&raw_upload_url)
             .map_err(|e| anyhow!("Invalid upload URL: {raw_upload_url}, {e}"))?;
 
-        let wrapped_command = wrap_with_exec_harness(&args.walltime_args, &args.command);
-
         Ok(Self {
             upload_url,
             token: args.shared.token,
@@ -150,7 +148,7 @@ impl TryFrom<crate::exec::ExecArgs> for Config {
             instruments: Instruments { mongodb: None }, // exec doesn't support MongoDB
             perf_unwinding_mode: args.shared.perf_run_args.perf_unwinding_mode,
             enable_perf: args.shared.perf_run_args.enable_perf,
-            command: wrapped_command,
+            command,
             profile_folder: args.shared.profile_folder,
             skip_upload: args.shared.skip_upload,
             skip_run: args.shared.skip_run,
@@ -160,6 +158,14 @@ impl TryFrom<crate::exec::ExecArgs> for Config {
     }
 }
 
+impl TryFrom<crate::exec::ExecArgs> for Config {
+    type Error = Error;
+    fn try_from(args: crate::exec::ExecArgs) -> Result<Self> {
+        let wrapped_command = wrap_with_exec_harness(&args.walltime_args, &args.command);
+        Self::try_from_with_command(args, wrapped_command)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::instruments::MongoDBConfig;

src/executor/shared/fifo.rs

@@ -156,7 +156,7 @@ impl RunnerFifo {
                 }
                 Err(_) => continue,
             };
-            trace!("Received command: {cmd:?}");
+            debug!("Received command: {cmd:?}");
 
             match &cmd {
                 FifoCommand::CurrentBenchmark { pid, uri } => {

src/project_config/interfaces.rs

@@ -10,6 +10,25 @@ use serde::{Deserialize, Serialize};
 pub struct ProjectConfig {
     /// Default options to apply to all benchmark runs
     pub options: Option<ProjectOptions>,
+    /// List of benchmark targets to execute
+    pub targets: Option<Vec<Target>>,
+}
+
+/// A benchmark target to execute
+#[derive(Debug, Deserialize, Serialize, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub struct Target {
+    /// Optional name for this target
+    pub name: Option<String>,
+    /// Command to execute
+    pub exec: String,
+    /// Target-specific options
+    pub options: Option<TargetOptions>,
+}
+
+#[derive(Debug, Deserialize, Serialize, PartialEq)]
+pub struct TargetOptions {
+    pub walltime: Option<WalltimeOptions>,
 }
 
 /// Root-level options that apply to all benchmark runs unless overridden by CLI

src/project_config/mod.rs

@@ -240,6 +240,7 @@ options:
                 working_directory: None,
                 mode: None,
             }),
+            targets: None,
         };
 
         let result = config.validate();
@@ -266,6 +267,7 @@ options:
                 working_directory: None,
                 mode: None,
             }),
+            targets: None,
         };
 
         let result = config.validate();
@@ -292,6 +294,7 @@ options:
                 working_directory: Some("./bench".to_string()),
                 mode: Some(RunnerMode::Walltime),
             }),
+            targets: None,
         };
 
         assert!(config.validate().is_ok());

src/run/mod.rs

@@ -195,6 +195,21 @@ impl RunArgs {
     }
 }
 
+use crate::project_config::Target;
+use crate::project_config::WalltimeOptions;
+/// Determines the execution mode based on CLI args and project config
+enum RunTarget<'a> {
+    /// Single command from CLI args
+    SingleCommand(RunArgs),
+    /// Multiple targets from project config
+    /// Note: for now, only `codspeed exec` targets are supported in the project config
+    ConfigTargets {
+        args: RunArgs,
+        targets: &'a [Target],
+        default_walltime: Option<&'a WalltimeOptions>,
+    },
+}
+
 pub async fn run(
     args: RunArgs,
     api_client: &CodSpeedAPIClient,
@@ -204,35 +219,75 @@ pub async fn run(
 ) -> Result<()> {
     let output_json = args.message_format == Some(MessageFormat::Json);
 
-    let merged_args = args.merge_with_project_config(project_config);
-
-    let config = Config::try_from(merged_args)?;
+    let args = args.merge_with_project_config(project_config);
+
+    let run_target = if args.command.is_empty() {
+        // No command provided - check for targets in project config
+        let targets = project_config
+            .and_then(|c| c.targets.as_ref())
+            .filter(|t| !t.is_empty())
+            .ok_or_else(|| {
+                anyhow!("No command provided and no targets defined in codspeed.yaml")
+            })?;
+
+        let default_walltime = project_config
+            .and_then(|c| c.options.as_ref())
+            .and_then(|o| o.walltime.as_ref());
+
+        RunTarget::ConfigTargets {
+            args,
+            targets,
+            default_walltime,
+        }
+    } else {
+        RunTarget::SingleCommand(args)
+    };
 
-    // Create execution context
-    let mut execution_context = executor::ExecutionContext::try_from((config, codspeed_config))?;
+    match run_target {
+        RunTarget::SingleCommand(args) => {
+            let config = Config::try_from(args)?;
+
+            // Create execution context
+            let mut execution_context =
+                executor::ExecutionContext::try_from((config, codspeed_config))?;
+
+            if !execution_context.is_local() {
+                show_banner();
+            }
+            debug!("config: {:#?}", execution_context.config);
+
+            // Execute benchmarks
+            let executor = executor::get_executor_from_mode(
+                &execution_context.config.mode,
+                executor::ExecutorCommand::Run,
+            );
+
+            let poll_results_fn = async |upload_result: &UploadResult| {
+                poll_results::poll_results(api_client, upload_result, output_json).await
+            };
+            executor::execute_benchmarks(
+                executor.as_ref(),
+                &mut execution_context,
+                setup_cache_dir,
+                poll_results_fn,
+                api_client,
+            )
+            .await?;
+        }
 
-    if !execution_context.is_local() {
-        show_banner();
+        RunTarget::ConfigTargets {
+            mut args,
+            targets,
+            default_walltime,
+        } => {
+            args.command =
+                crate::exec::multi_targets::build_pipe_command(targets, default_walltime)?;
+            let config = Config::try_from(args)?;
+
+            crate::exec::execute_with_harness(config, api_client, codspeed_config, setup_cache_dir)
+                .await?;
+        }
     }
-    debug!("config: {:#?}", execution_context.config);
-
-    // Execute benchmarks
-    let executor = executor::get_executor_from_mode(
-        &execution_context.config.mode,
-        executor::ExecutorCommand::Run,
-    );
-
-    let poll_results_fn = async |upload_result: &UploadResult| {
-        poll_results::poll_results(api_client, upload_result, output_json).await
-    };
-    executor::execute_benchmarks(
-        executor.as_ref(),
-        &mut execution_context,
-        setup_cache_dir,
-        poll_results_fn,
-        api_client,
-    )
-    .await?;
 
     Ok(())
 }