feat(exec-harness): add support for analysis mode with memory instrument

Author: GuillaumeLagrange

crates/exec-harness/src/analysis.rs

@@ -0,0 +1,24 @@
+use crate::prelude::*;
+
+use crate::uri::NameAndUri;
+use codspeed::instrument_hooks::InstrumentHooks;
+use std::process::Command;
+
+pub fn perform(name_and_uri: NameAndUri, command: Vec<String>) -> Result<()> {
+    let hooks = InstrumentHooks::instance();
+
+    let mut cmd = Command::new(&command[0]);
+    cmd.args(&command[1..]);
+    hooks.start_benchmark().unwrap();
+    let status = cmd.status();
+    hooks.stop_benchmark().unwrap();
+    let status = status.context("Failed to execute command")?;
+
+    if !status.success() {
+        bail!("Command exited with non-zero status: {status}");
+    }
+
+    hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+
+    Ok(())
+}

crates/exec-harness/src/lib.rs

@@ -1,3 +1,15 @@
+use clap::ValueEnum;
+use serde::{Deserialize, Serialize};
+
+pub mod analysis;
 mod prelude;
 pub mod uri;
 pub mod walltime;
+
+#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum MeasurementMode {
+    Walltime,
+    Memory,
+    Simulation,
+}

crates/exec-harness/src/main.rs

@@ -1,8 +1,11 @@
 use anyhow::Result;
 use anyhow::bail;
 use clap::Parser;
+use exec_harness::MeasurementMode;
+use exec_harness::analysis;
 use exec_harness::uri;
 use exec_harness::walltime;
+use log::debug;
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
@@ -15,6 +18,10 @@ struct Args {
     #[arg(long)]
     name: Option<String>,
 
+    /// Set by the runner, should be coherent with the executor being used
+    #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE", hide = true)]
+    measurement_mode: Option<MeasurementMode>,
+
     #[command(flatten)]
     execution_args: walltime::WalltimeExecutionArgs,
 
@@ -29,6 +36,8 @@ fn main() -> Result<()> {
         .format_timestamp(None)
         .init();
 
+    debug!("Starting exec-harness with pid {}", std::process::id());
+
     let args = Args::parse();
 
     if args.command.is_empty() {
@@ -37,10 +46,19 @@ fn main() -> Result<()> {
 
     let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command);
 
-    // Build execution options from CLI args
-    let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
+    match args.measurement_mode {
+        Some(MeasurementMode::Walltime) | None => {
+            let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
 
-    walltime::perform(bench_name_and_uri, args.command, &execution_options)?;
+            walltime::perform(bench_name_and_uri, args.command, &execution_options)?;
+        }
+        Some(MeasurementMode::Memory) => {
+            analysis::perform(bench_name_and_uri, args.command)?;
+        }
+        Some(MeasurementMode::Simulation) => {
+            bail!("Simulation measurement mode is not yet supported by exec-harness");
+        }
+    }
 
     Ok(())
 }