chore: add executor tests

Author: not-matthias

Cargo.lock

@@ -62,6 +62,7 @@ procfs = "0.17.0"
 temp-env = { version = "0.3.6", features = ["async_closure"] }
 insta = { version = "1.29.0", features = ["json", "redactions"] }
 test-with = { version = "0.15", default-features = false, features = [] }
+rstest = { version = "0.25.0", default-features = false }
 
 [workspace.metadata.release]
 sign-tag = true

Cargo.toml

@@ -5,91 +5,189 @@ use crate::run::runner::interfaces::RunData;
 use crate::run::runner::valgrind::executor::ValgrindExecutor;
 use crate::run::{RunnerMode, runner::wall_time::executor::WallTimeExecutor};
 use tempfile::TempDir;
+use tokio::sync::{OnceCell, Semaphore, SemaphorePermit};
 
+const SIMPLE_ECHO_SCRIPT: &str = "echo 'Hello, World!'";
 const MULTILINE_ECHO_SCRIPT: &str = "echo \"Working\"
 echo \"with\"
 echo \"multiple lines\"";
-
 const MULTILINE_ECHO_WITH_SEMICOLONS: &str = "echo \"Working\";
 echo \"with\";
 echo \"multiple lines\";";
-
 const DIRECTORY_CHECK_SCRIPT: &str = "cd /tmp
 # Check that the directory is actually changed
 if [ $(basename $(pwd)) != \"tmp\" ]; then
   exit 1
 fi";
+const ENV_VAR_VALIDATION_SCRIPT: &str = "
+output=$(echo \"$MY_ENV_VAR\")
+if [ \"$output\" != \"Hello\" ]; then
+  echo \"Assertion failed: Expected 'Hello' but got '$output'\"
+  exit 1
+fi";
+
+async fn get_valgrind_executor() -> &'static ValgrindExecutor {
+    static VALGRIND_EXECUTOR: OnceCell<ValgrindExecutor> = OnceCell::const_new();
 
-const BENCHMARK_COMMANDS: [&str; 5] = [
-    "echo 'Hello, World!'",
-    MULTILINE_ECHO_WITH_SEMICOLONS,
-    MULTILINE_ECHO_WITH_SEMICOLONS,
-    MULTILINE_ECHO_SCRIPT,
-    DIRECTORY_CHECK_SCRIPT,
-];
+    VALGRIND_EXECUTOR
+        .get_or_init(|| async {
+            let executor = ValgrindExecutor;
+            let system_info = SystemInfo::new().unwrap();
+            executor.setup(&system_info).await.unwrap();
+            executor
+        })
+        .await
+}
+
+async fn get_walltime_executor() -> (SemaphorePermit<'static>, WallTimeExecutor) {
+    static WALLTIME_INIT: OnceCell<()> = OnceCell::const_new();
+    static WALLTIME_SEMAPHORE: OnceCell<Semaphore> = OnceCell::const_new();
+
+    WALLTIME_INIT
+        .get_or_init(|| async {
+            let executor = WallTimeExecutor::new();
+            let system_info = SystemInfo::new().unwrap();
+            executor.setup(&system_info).await.unwrap();
+        })
+        .await;
+
+    // We can't execute multiple walltime executors in parallel because perf isn't thread-safe (yet). We have to
+    // use a semaphore to limit concurrent access.
+    let semaphore = WALLTIME_SEMAPHORE
+        .get_or_init(|| async { Semaphore::new(1) })
+        .await;
+    let permit = semaphore.acquire().await.unwrap();
+
+    (permit, WallTimeExecutor::new())
+}
 
 async fn create_test_setup() -> (SystemInfo, RunData, TempDir) {
     let temp_dir = TempDir::new().unwrap();
 
-    let system_info = SystemInfo::test();
+    let system_info = SystemInfo::new().unwrap();
     let run_data = RunData {
         profile_folder: temp_dir.path().to_path_buf(),
     };
 
     (system_info, run_data, temp_dir)
 }
 
+#[rstest::rstest]
+#[case(SIMPLE_ECHO_SCRIPT)]
+#[case(MULTILINE_ECHO_WITH_SEMICOLONS)]
+#[case(MULTILINE_ECHO_SCRIPT)]
+#[case(DIRECTORY_CHECK_SCRIPT)]
 #[tokio::test]
-async fn test_valgrind_executor() {
+async fn test_valgrind_executor(#[case] cmd: &str) {
     let (system_info, run_data, _temp_dir) = create_test_setup().await;
+    let executor = get_valgrind_executor().await;
+
+    eprintln!("Running command: {cmd}");
+
+    let config = Config {
+        mode: RunnerMode::Instrumentation,
+        command: cmd.to_string(),
+        ..Config::test()
+    };
+
+    executor
+        .run(&config, &system_info, &run_data, &None)
+        .await
+        .unwrap();
+    executor
+        .teardown(&config, &system_info, &run_data)
+        .await
+        .unwrap();
+}
+
+#[rstest::rstest]
+#[case(SIMPLE_ECHO_SCRIPT, false)]
+#[case(SIMPLE_ECHO_SCRIPT, true)]
+#[case(MULTILINE_ECHO_WITH_SEMICOLONS, false)]
+#[case(MULTILINE_ECHO_WITH_SEMICOLONS, true)]
+#[case(MULTILINE_ECHO_SCRIPT, false)]
+#[case(MULTILINE_ECHO_SCRIPT, true)]
+#[case(DIRECTORY_CHECK_SCRIPT, false)]
+#[case(DIRECTORY_CHECK_SCRIPT, true)]
+#[tokio::test]
+async fn test_walltime_executor(#[case] cmd: &str, #[case] enable_perf: bool) {
+    let (system_info, run_data, _temp_dir) = create_test_setup().await;
+    let (_permit, executor) = get_walltime_executor().await;
+    let executor = executor;
+
+    eprintln!("Running command: {cmd}");
+
+    let config = Config {
+        mode: RunnerMode::Walltime,
+        command: cmd.to_string(),
+        enable_perf,
+        ..Config::test()
+    };
+
+    executor
+        .run(&config, &system_info, &run_data, &None)
+        .await
+        .unwrap();
 
-    let executor = ValgrindExecutor;
-    executor.setup(&system_info).await.unwrap();
+    // Don't execute teardown since the cmds dont use the FIFO to set the integration metadata.
+}
+
+#[rstest::rstest]
+#[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT)]
+#[tokio::test]
+async fn test_valgrind_executor_with_env(
+    #[case] env_var: &str,
+    #[case] env_value: &str,
+    #[case] cmd: &str,
+) {
+    let (system_info, run_data, _temp_dir) = create_test_setup().await;
+    let executor = get_valgrind_executor().await;
 
-    for cmd in BENCHMARK_COMMANDS {
-        eprintln!("Running command: {cmd}");
+    let config = Config {
+        mode: RunnerMode::Instrumentation,
+        command: cmd.to_string(),
+        ..Config::test()
+    };
 
-        let config = Config {
-            mode: RunnerMode::Instrumentation,
-            command: cmd.to_string(),
-            ..Config::test()
-        };
+    eprintln!("Running command: {cmd}");
 
+    temp_env::async_with_vars(&[(env_var, Some(env_value))], async {
         executor
             .run(&config, &system_info, &run_data, &None)
             .await
             .unwrap();
-        executor
-            .teardown(&config, &system_info, &run_data)
-            .await
-            .unwrap();
-    }
+    })
+    .await;
 }
 
+#[rstest::rstest]
+#[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT, false)]
+#[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT, true)]
 #[tokio::test]
-async fn test_walltime_executor() {
+async fn test_walltime_executor_with_env(
+    #[case] env_var: &str,
+    #[case] env_value: &str,
+    #[case] cmd: &str,
+    #[case] enable_perf: bool,
+) {
     let (system_info, run_data, _temp_dir) = create_test_setup().await;
+    let (_permit, executor) = get_walltime_executor().await;
+    let executor = executor;
+
+    let config = Config {
+        mode: RunnerMode::Walltime,
+        command: cmd.to_string(),
+        enable_perf,
+        ..Config::test()
+    };
 
-    let executor = WallTimeExecutor::new();
-    executor.setup(&system_info).await.unwrap();
-
-    for enable_perf in [false, true] {
-        for cmd in BENCHMARK_COMMANDS {
-            eprintln!("Running command: {cmd}");
-
-            let config = Config {
-                mode: RunnerMode::Walltime,
-                command: cmd.to_string(),
-                enable_perf,
-                ..Config::test()
-            };
-
-            executor
-                .run(&config, &system_info, &run_data, &None)
-                .await
-                .unwrap();
+    eprintln!("Running command: {cmd}");
 
-            // Don't execute teardown since the cmds dont use the FIFO to set the integration metadata.
-        }
-    }
+    temp_env::async_with_vars(&[(env_var, Some(env_value))], async {
+        executor
+            .run(&config, &system_info, &run_data, &None)
+            .await
+            .unwrap();
+    })
+    .await;
 }