Skip to content

Commit 1cada6a

Browse files
HeliozoaHeliozoa
committed
improved timeout wrapper for commands
1 parent ffdc89b commit 1cada6a

File tree

17 files changed

+295
-27
lines changed

17 files changed

+295
-27
lines changed

‎tmc-langs-csharp/src/plugin.rs‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ pub use policy::CSharpStudentFilePolicy;
66
use crate::{CSTestResult, CSharpError};
77

88
use tmc_langs_framework::{
9-
domain::{ExerciseDesc, RunResult, RunStatus, TestDesc},
9+
domain::{ExerciseDesc, RunResult, RunStatus, TestDesc, TestResult},
1010
plugin::{Language, Strategy, ValidationResult},
11-
CommandWithTimeout, LanguagePlugin, TmcError,
11+
CommandWithTimeout, LanguagePlugin, OutputWithTimeout, TmcError,
1212
};
1313

1414
use std::collections::HashMap;
@@ -190,19 +190,37 @@ impl LanguagePlugin for CSharpPlugin {
190190
});
191191
}
192192
};
193-
log::trace!("stdout: {}", String::from_utf8_lossy(&output.stdout));
194-
log::debug!("stderr: {}", String::from_utf8_lossy(&output.stderr));
195-
if !output.status.success() {
196-
let mut logs = HashMap::new();
197-
logs.insert("stdout".to_string(), output.stdout);
198-
logs.insert("stderr".to_string(), output.stderr);
199-
return Ok(RunResult {
200-
status: RunStatus::CompileFailed,
201-
test_results: vec![],
202-
logs,
203-
});
193+
log::trace!("stdout: {}", String::from_utf8_lossy(output.stdout()));
194+
log::debug!("stderr: {}", String::from_utf8_lossy(output.stderr()));
195+
196+
match output {
197+
OutputWithTimeout::Output(output) => {
198+
if !output.status.success() {
199+
let mut logs = HashMap::new();
200+
logs.insert("stdout".to_string(), output.stdout);
201+
logs.insert("stderr".to_string(), output.stderr);
202+
return Ok(RunResult {
203+
status: RunStatus::CompileFailed,
204+
test_results: vec![],
205+
logs,
206+
});
207+
}
208+
Self::parse_test_results(&test_results_path).map_err(|e| e.into())
209+
}
210+
OutputWithTimeout::Timeout { .. } => Ok(RunResult {
211+
status: RunStatus::TestsFailed,
212+
test_results: vec![TestResult {
213+
name: "Timeout test".to_string(),
214+
successful: false,
215+
points: vec![],
216+
message:
217+
"Tests timed out.\nMake sure you don't have an infinite loop in your code."
218+
.to_string(),
219+
exception: vec![],
220+
}],
221+
logs: HashMap::new(),
222+
}),
204223
}
205-
Self::parse_test_results(&test_results_path).map_err(|e| e.into())
206224
}
207225

208226
// no checkstyle for C#

‎tmc-langs-framework/src/error.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,6 @@ pub enum TmcError {
6161

6262
#[error("Failed to spawn command: {0}")]
6363
CommandSpawn(&'static str, #[source] std::io::Error),
64-
#[error("Test timed out after {} seconds", .0.as_secs())]
65-
TestTimeout(Duration),
6664

6765
#[error("Error in plugin: {0}")]
6866
Plugin(#[source] Box<dyn std::error::Error + 'static + Send + Sync>),

‎tmc-langs-framework/src/lib.rs‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,74 @@ pub use plugin::LanguagePlugin;
1111
pub use policy::StudentFilePolicy;
1212

1313
use domain::TmcProjectYml;
14-
use std::process::{Command, Output};
14+
use std::io::Read;
15+
use std::process::{Command, Output, Stdio};
1516
use std::thread;
1617
use std::time::{Duration, Instant};
1718

1819
pub type Result<T> = std::result::Result<T, TmcError>;
1920

2021
pub struct CommandWithTimeout<'a>(pub &'a mut Command);
2122

23+
pub enum OutputWithTimeout {
24+
Output(Output),
25+
Timeout { stdout: Vec<u8>, stderr: Vec<u8> },
26+
}
27+
28+
impl OutputWithTimeout {
29+
pub fn stdout(&self) -> &[u8] {
30+
match self {
31+
Self::Output(output) => &output.stdout,
32+
Self::Timeout { stdout, .. } => &stdout,
33+
}
34+
}
35+
pub fn stderr(&self) -> &[u8] {
36+
match self {
37+
Self::Output(output) => &output.stderr,
38+
Self::Timeout { stderr, .. } => &stderr,
39+
}
40+
}
41+
}
42+
2243
impl CommandWithTimeout<'_> {
2344
pub fn wait_with_timeout(
2445
&mut self,
2546
name: &'static str,
2647
timeout: Option<Duration>,
27-
) -> Result<Output> {
48+
) -> Result<OutputWithTimeout> {
2849
match timeout {
2950
Some(timeout) => {
3051
// spawn process and init timer
3152
let mut child = self
3253
.0
54+
.stdout(Stdio::piped())
55+
.stderr(Stdio::piped())
3356
.spawn()
3457
.map_err(|e| TmcError::CommandSpawn(name, e))?;
3558
let timer = Instant::now();
59+
3660
loop {
3761
match child.try_wait().map_err(TmcError::Process)? {
3862
Some(_exit_status) => {
3963
// done, get output
4064
return child
4165
.wait_with_output()
66+
.map(OutputWithTimeout::Output)
4267
.map_err(|e| TmcError::CommandFailed(name, e));
4368
}
4469
None => {
4570
// still running, check timeout
4671
if timer.elapsed() > timeout {
72+
log::warn!("command {} timed out", name);
4773
child.kill().map_err(TmcError::Process)?;
48-
return Err(TmcError::TestTimeout(timer.elapsed()));
74+
75+
let mut stdout = vec![];
76+
let mut stderr = vec![];
77+
let stdout_handle = child.stdout.as_mut().unwrap();
78+
let stderr_handle = child.stderr.as_mut().unwrap();
79+
stdout_handle.read_to_end(&mut stdout).unwrap();
80+
stderr_handle.read_to_end(&mut stderr).unwrap();
81+
return Ok(OutputWithTimeout::Timeout { stdout, stderr });
4982
}
5083

5184
// TODO: gradually increase sleep duration?
@@ -58,6 +91,7 @@ impl CommandWithTimeout<'_> {
5891
None => self
5992
.0
6093
.output()
94+
.map(OutputWithTimeout::Output)
6195
.map_err(|e| TmcError::CommandFailed(name, e)),
6296
}
6397
}
@@ -72,10 +106,12 @@ mod test {
72106
let mut command = Command::new("sleep");
73107
let mut command = command.arg("1");
74108
let mut out = CommandWithTimeout(&mut command);
75-
let res = out.wait_with_timeout("sleep", Some(Duration::from_millis(100)));
76-
if let Err(TmcError::TestTimeout(_)) = res {
109+
let res = out
110+
.wait_with_timeout("sleep", Some(Duration::from_millis(100)))
111+
.unwrap();
112+
if let OutputWithTimeout::Timeout { .. } = res {
77113
} else {
78-
panic!("unexpected result: {:?}", res);
114+
panic!("unexpected result");
79115
}
80116
}
81117
}

‎tmc-langs-python3/src/plugin.rs‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use std::time::Duration;
1313
use tmc_langs_framework::{
1414
domain::{ExerciseDesc, RunResult, RunStatus, TestDesc, TestResult},
1515
plugin::LanguagePlugin,
16-
CommandWithTimeout, TmcError,
16+
CommandWithTimeout, OutputWithTimeout, TmcError,
1717
};
1818
use walkdir::WalkDir;
1919

@@ -49,7 +49,22 @@ impl LanguagePlugin for Python3Plugin {
4949
path: &Path,
5050
timeout: Option<Duration>,
5151
) -> Result<RunResult, TmcError> {
52-
run_tmc_command(path, &[], timeout)?;
52+
let output = run_tmc_command(path, &[], timeout)?;
53+
if let OutputWithTimeout::Timeout { .. } = output {
54+
return Ok(RunResult {
55+
status: RunStatus::TestsFailed,
56+
test_results: vec![TestResult {
57+
name: "Timeout test".to_string(),
58+
successful: false,
59+
points: vec![],
60+
message:
61+
"Tests timed out.\nMake sure you don't have an infinite loop in your code."
62+
.to_string(),
63+
exception: vec![],
64+
}],
65+
logs: HashMap::new(),
66+
});
67+
}
5368
Ok(parse_test_result(path)?)
5469
}
5570

@@ -82,7 +97,7 @@ fn run_tmc_command(
8297
path: &Path,
8398
extra_args: &[&str],
8499
timeout: Option<Duration>,
85-
) -> Result<std::process::Output, PythonError> {
100+
) -> Result<OutputWithTimeout, PythonError> {
86101
let path = path
87102
.canonicalize()
88103
.map_err(|e| PythonError::Canonicalize(path.to_path_buf(), e))?;
@@ -107,8 +122,8 @@ fn run_tmc_command(
107122
.current_dir(path);
108123
let output = CommandWithTimeout(command).wait_with_timeout(name, timeout)?;
109124

110-
log::trace!("stdout: {}", String::from_utf8_lossy(&output.stdout));
111-
log::debug!("stderr: {}", String::from_utf8_lossy(&output.stderr));
125+
log::trace!("stdout: {}", String::from_utf8_lossy(output.stdout()));
126+
log::debug!("stderr: {}", String::from_utf8_lossy(output.stderr()));
112127
Ok(output)
113128
}
114129

@@ -304,4 +319,16 @@ mod test {
304319
.exists());
305320
assert!(temp_path.join("leave").exists());
306321
}
322+
323+
#[test]
324+
fn timeout() {
325+
init();
326+
let plugin = Python3Plugin::new();
327+
328+
let temp = copy_test("tests/data/timeout");
329+
let timeout = plugin
330+
.run_tests_with_timeout(temp.path(), Some(std::time::Duration::from_millis(1)))
331+
.unwrap();
332+
assert_eq!(timeout.test_results[0].name, "Timeout test");
333+
}
307334
}

‎tmc-langs-python3/tests/data/timeout/test/__init__.py‎

Whitespace-only changes.

‎tmc-langs-python3/tests/data/timeout/test/test_points.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import unittest
2+
from tmc import points
3+
4+
5+
@points('1.1')
6+
class TestEverything(unittest.TestCase):
7+
8+
@points('1.2', '2.2')
9+
def test_new(self):
10+
while True:
11+
pass
12+
self.assertEqual("a", "a")
13+
14+
if __name__ == '__main__':
15+
unittest.main()

‎tmc-langs-python3/tests/data/timeout/tmc/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from .points import points
2+
from .runner import TMCTestRunner

‎tmc-langs-python3/tests/data/timeout/tmc/__main__.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from unittest import TestProgram
2+
from .runner import TMCTestRunner
3+
import sys
4+
5+
6+
if sys.argv.__len__() > 1 and sys.argv[1] == 'available_points':
7+
TMCTestRunner().available_points()
8+
sys.exit()
9+
10+
main = TestProgram
11+
main(testRunner=TMCTestRunner, module=None, failfast=False, buffer=True)

‎tmc-langs-python3/tests/data/timeout/tmc/__pycache__/__init__.cpython-38.pyc‎

249 Bytes
Binary file not shown.

‎tmc-langs-python3/tests/data/timeout/tmc/__pycache__/__main__.cpython-38.pyc‎

452 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)