timeout for python3 tests

Author: Heliozoa

tmc-langs-framework/src/domain.rs

@@ -128,14 +128,21 @@ impl ExercisePackagingConfiguration {
 pub struct TmcProjectYml {
     #[serde(default)]
     pub extra_student_files: Vec<PathBuf>,
+
     #[serde(default)]
     pub extra_exercise_files: Vec<PathBuf>,
+
     #[serde(default)]
     pub force_update: Vec<PathBuf>,
+
+    #[serde(default)]
+    pub tests_timeout_ms: Option<u64>,
+
     #[serde(default)]
     #[serde(rename = "no-tests")]
     pub no_tests: Option<NoTests>,
 
+    #[serde(default)]
     pub fail_on_valgrind_error: Option<bool>,
 }
 

tmc-langs-framework/src/lib.rs

@@ -11,6 +11,9 @@ pub use policy::StudentFilePolicy;
 use domain::TmcProjectYml;
 use io::zip;
 use std::path::PathBuf;
+use std::process::{Command, Output};
+use std::thread;
+use std::time::{Duration, Instant};
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -37,6 +40,11 @@ pub enum Error {
     #[error("Running command '{0}' failed")]
     CommandFailed(&'static str),
 
+    #[error("Failed to spawn command: {0}")]
+    CommandSpawn(std::io::Error),
+    #[error("Test timed out")]
+    TestTimeout,
+
     #[error("Error in plugin: {0}")]
     Plugin(Box<dyn std::error::Error + 'static + Send + Sync>),
 
@@ -49,3 +57,60 @@ pub enum Error {
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
+
+pub struct CommandWithTimeout<'a>(pub &'a mut Command);
+
+impl CommandWithTimeout<'_> {
+    pub fn wait_with_timeout(
+        &mut self,
+        name: &'static str,
+        timeout: Option<Duration>,
+    ) -> Result<Output> {
+        match timeout {
+            Some(timeout) => {
+                // spawn process and init timer
+                let mut child = self.0.spawn().map_err(|e| Error::CommandSpawn(e))?;
+                let timer = Instant::now();
+                loop {
+                    match child.try_wait()? {
+                        Some(_exit_status) => {
+                            // done, get output
+                            return child
+                                .wait_with_output()
+                                .map_err(|_| Error::CommandFailed(name));
+                        }
+                        None => {
+                            // still running, check timeout
+                            if timer.elapsed() > timeout {
+                                child.kill()?;
+                                return Err(Error::TestTimeout);
+                            }
+
+                            // TODO: gradually increase sleep duration?
+                            thread::sleep(Duration::from_millis(100));
+                        }
+                    }
+                }
+            }
+            // no timeout, block forever
+            None => self.0.output().map_err(|_| Error::CommandFailed(name)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn timeout() {
+        let mut command = Command::new("sleep");
+        let mut command = command.arg("1");
+        let mut out = CommandWithTimeout(&mut command);
+        let res = out.wait_with_timeout("sleep", Some(Duration::from_millis(100)));
+        if let Err(Error::TestTimeout) = res {
+        } else {
+            panic!("unexpected result: {:?}", res);
+        }
+    }
+}

tmc-langs-framework/src/plugin.rs

@@ -10,6 +10,7 @@ use super::Result;
 use log::debug;
 use std::collections::HashSet;
 use std::path::{Path, PathBuf};
+use std::time::Duration;
 use walkdir::WalkDir;
 
 /// The trait that each language plug-in must implement.
@@ -61,7 +62,17 @@ pub trait LanguagePlugin {
     fn scan_exercise(&self, path: &Path, exercise_name: String) -> Result<ExerciseDesc>;
 
     /// Runs the tests for the exercise.
-    fn run_tests(&self, path: &Path) -> Result<RunResult>;
+    fn run_tests(&self, path: &Path) -> Result<RunResult> {
+        let timeout = self
+            .get_student_file_policy(path)
+            .get_tmc_project_yml()
+            .ok()
+            .and_then(|t| t.tests_timeout_ms.map(Duration::from_millis));
+        self.run_tests_with_timeout(path, timeout)
+    }
+
+    /// Runs the tests for the exercise with the given timeout.
+    fn run_tests_with_timeout(&self, path: &Path, timeout: Option<Duration>) -> Result<RunResult>;
 
     /// Prepares a submission for processing in the sandbox.
     ///
@@ -207,7 +218,11 @@ mod test {
             unimplemented!()
         }
 
-        fn run_tests(&self, _path: &Path) -> Result<RunResult> {
+        fn run_tests_with_timeout(
+            &self,
+            _path: &Path,
+            _timeout: Option<Duration>,
+        ) -> Result<RunResult> {
             unimplemented!()
         }
 

tmc-langs-java/src/ant.rs

@@ -1,6 +1,7 @@
 pub mod policy;
 
 use super::{error::JavaError, plugin::JavaPlugin, CompileResult, TestRun, SEPARATOR};
+
 use isolang::Language;
 use j4rs::Jvm;
 use policy::AntStudentFilePolicy;
@@ -9,6 +10,7 @@ use std::fs::{self, File};
 use std::io::Write;
 use std::path::Path;
 use std::process::Command;
+use std::time::Duration;
 use tmc_langs_abstraction::ValidationResult;
 use tmc_langs_framework::{
     domain::{ExerciseDesc, RunResult},
@@ -73,7 +75,11 @@ impl LanguagePlugin for AntPlugin {
         Ok(self.scan_exercise_with_compile_result(path, exercise_name, compile_result)?)
     }
 
-    fn run_tests(&self, project_root_path: &Path) -> Result<RunResult, Error> {
+    fn run_tests_with_timeout(
+        &self,
+        project_root_path: &Path,
+        _timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
         Ok(self.run_java_tests(project_root_path)?)
     }
 

tmc-langs-java/src/maven.rs

@@ -1,12 +1,14 @@
 pub mod policy;
 
 use super::{error::JavaError, plugin::JavaPlugin, CompileResult, TestRun, SEPARATOR};
+
 use isolang::Language;
 use j4rs::Jvm;
 use policy::MavenStudentFilePolicy;
 use std::fs;
 use std::path::Path;
 use std::process::Command;
+use std::time::Duration;
 use tmc_langs_abstraction::ValidationResult;
 use tmc_langs_framework::{
     domain::{ExerciseDesc, RunResult},
@@ -44,7 +46,11 @@ impl LanguagePlugin for MavenPlugin {
         Ok(self.scan_exercise_with_compile_result(path, exercise_name, compile_result)?)
     }
 
-    fn run_tests(&self, project_root_path: &Path) -> Result<RunResult, Error> {
+    fn run_tests_with_timeout(
+        &self,
+        project_root_path: &Path,
+        _timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
         Ok(self.run_java_tests(project_root_path)?)
     }
 

tmc-langs-make/src/plugin.rs

@@ -4,6 +4,7 @@ use crate::check_log::CheckLog;
 use crate::error::MakeError;
 use crate::policy::MakeStudentFilePolicy;
 use crate::valgrind_log::ValgrindLog;
+
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::collections::HashMap;
@@ -12,6 +13,7 @@ use std::io;
 use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::process::Command;
+use std::time::Duration;
 use tmc_langs_framework::{
     domain::{ExerciseDesc, RunResult, RunStatus, TestDesc, TmcProjectYml},
     plugin::LanguagePlugin,
@@ -135,7 +137,11 @@ impl LanguagePlugin for MakePlugin {
         Ok(self.parse_exercise_desc(&available_points_path, exercise_name)?)
     }
 
-    fn run_tests(&self, path: &Path) -> Result<RunResult, Error> {
+    fn run_tests_with_timeout(
+        &self,
+        path: &Path,
+        _timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
         if !self.builds(path)? {
             return Ok(RunResult {
                 status: RunStatus::CompileFailed,

tmc-langs-notests/src/lib.rs

@@ -6,10 +6,15 @@ use tmc_langs_framework::{
 
 use std::collections::HashMap;
 use std::path::Path;
+use std::time::Duration;
 
 pub struct NoTestsPlugin {}
 
 impl NoTestsPlugin {
+    pub fn new() -> Self {
+        Self {}
+    }
+
     fn get_points(&self, path: &Path) -> Vec<String> {
         self.get_student_file_policy(path)
             .get_tmc_project_yml()
@@ -35,7 +40,11 @@ impl LanguagePlugin for NoTestsPlugin {
         })
     }
 
-    fn run_tests(&self, path: &Path) -> Result<RunResult, Error> {
+    fn run_tests_with_timeout(
+        &self,
+        path: &Path,
+        _timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
         Ok(RunResult {
             status: RunStatus::Passed,
             test_results: vec![TestResult {

tmc-langs-python3/src/plugin.rs

@@ -7,11 +7,12 @@ use std::fs::{self, File};
 use std::io::BufReader;
 use std::path::Path;
 use std::process::Command;
+use std::time::Duration;
 use tmc_langs_framework::{
     domain::{ExerciseDesc, RunResult, RunStatus, TestDesc, TestResult},
     plugin::LanguagePlugin,
     policy::StudentFilePolicy,
-    Error,
+    CommandWithTimeout, Error,
 };
 use walkdir::WalkDir;
 
@@ -33,7 +34,7 @@ impl LanguagePlugin for Python3Plugin {
     }
 
     fn scan_exercise(&self, path: &Path, exercise_name: String) -> Result<ExerciseDesc, Error> {
-        let run_result = run_tmc_command(path, &["available_points"]);
+        let run_result = run_tmc_command(path, &["available_points"], None);
 
         if let Err(error) = run_result {
             log::error!("Failed to scan exercise. {}", error);
@@ -43,8 +44,12 @@ impl LanguagePlugin for Python3Plugin {
         Ok(ExerciseDesc::new(exercise_name, test_descs))
     }
 
-    fn run_tests(&self, path: &Path) -> Result<RunResult, Error> {
-        let run_result = run_tmc_command(path, &[]);
+    fn run_tests_with_timeout(
+        &self,
+        path: &Path,
+        timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
+        let run_result = run_tmc_command(path, &[], timeout);
 
         if let Err(error) = run_result {
             log::error!("Failed to parse exercise description. {}", error);
@@ -87,37 +92,45 @@ impl LanguagePlugin for Python3Plugin {
     }
 }
 
-fn run_tmc_command(path: &Path, extra_args: &[&str]) -> Result<std::process::Output, PythonError> {
+fn run_tmc_command(
+    path: &Path,
+    extra_args: &[&str],
+    timeout: Option<Duration>,
+) -> Result<std::process::Output, PythonError> {
     let path = path
         .canonicalize()
         .map_err(|e| PythonError::Path(path.to_path_buf(), e))?;
     log::debug!("running tmc command at {}", path.display());
     let common_args = ["-m", "tmc"];
-    let result = match &*LOCAL_PY {
-        LocalPy::Unix => Command::new("python3")
+
+    let (name, mut command) = match &*LOCAL_PY {
+        LocalPy::Unix => ("python3", Command::new("python3")),
+        LocalPy::Windows => ("py", Command::new("py")),
+        //.map_err(|e| PythonError::Command("py", e))?,
+        LocalPy::WindowsConda { conda_path } => ("conda", Command::new(conda_path)),
+    };
+    let command = match &*LOCAL_PY {
+        LocalPy::Unix => command
             .args(&common_args)
             .args(extra_args)
-            .current_dir(path)
-            .output()
-            .map_err(|e| PythonError::Command("python3", e))?,
-        LocalPy::Windows => Command::new("py")
+            .current_dir(path),
+        LocalPy::Windows => command
             .args(&["-3"])
             .args(&common_args)
             .args(extra_args)
-            .current_dir(path)
-            .output()
-            .map_err(|e| PythonError::Command("py", e))?,
-        LocalPy::WindowsConda { conda_path } => Command::new(conda_path)
+            .current_dir(path),
+        LocalPy::WindowsConda { .. } => command
             .args(&common_args)
             .args(extra_args)
-            .current_dir(path)
-            .output()
-            .map_err(|e| PythonError::Command(conda_path, e))?,
+            .current_dir(path),
     };
+    let output = CommandWithTimeout(command)
+        .wait_with_timeout(name, timeout)
+        .unwrap();
 
-    log::debug!("stdout: {}", String::from_utf8_lossy(&result.stdout));
-    log::debug!("stderr: {}", String::from_utf8_lossy(&result.stderr));
-    Ok(result)
+    log::debug!("stdout: {}", String::from_utf8_lossy(&output.stdout));
+    log::debug!("stderr: {}", String::from_utf8_lossy(&output.stderr));
+    Ok(output)
 }
 
 fn parse_exercise_description(path: &Path) -> Result<Vec<TestDesc>, PythonError> {

tmc-langs-r/src/plugin.rs

@@ -12,9 +12,16 @@ use std::collections::HashMap;
 use std::fs::{self, File};
 use std::path::Path;
 use std::process::Command;
+use std::time::Duration;
 
 pub struct RPlugin {}
 
+impl RPlugin {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
 impl LanguagePlugin for RPlugin {
     fn get_plugin_name(&self) -> &str {
         "r"
@@ -54,7 +61,11 @@ impl LanguagePlugin for RPlugin {
         })
     }
 
-    fn run_tests(&self, path: &Path) -> Result<RunResult, Error> {
+    fn run_tests_with_timeout(
+        &self,
+        path: &Path,
+        _timeout: Option<Duration>,
+    ) -> Result<RunResult, Error> {
         // delete results json
         let results_path = path.join(".results.json");
         if results_path.exists() {