diff --git a/Changelog.md b/Changelog.md index fe1119ba..15d26abc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented here. - Add remote URL whitelist for AI tester to restrict allowed endpoints (#693) - Increased default settings job timeout from 600s to 1200s (#707) - Disable pytest cacheprovider to avoid creating .pytest_cache in isolated runs (#709) +- Fixed Python tester to correctly report marks when `markus_marks_earned` equals total or zero (#716) ## [v2.9.0] - Install stack with GHCup (#626) diff --git a/server/autotest_server/testers/py/py_tester.py b/server/autotest_server/testers/py/py_tester.py index 9ffb7520..5f0e468b 100644 --- a/server/autotest_server/testers/py/py_tester.py +++ b/server/autotest_server/testers/py/py_tester.py @@ -251,11 +251,8 @@ def run(self) -> str: """ Return a json string containing all test result information. """ - if self.points_earned is not None and 0 < self.points_earned < self.points_total: - return self.partially_passed(points_earned=self.points_earned, message=self.message) - elif self.points_earned is not None and self.points_earned > self.points_total: - bonus = self.points_earned - self.points_total - return self.passed_with_bonus(points_bonus=bonus, message=self.message) + if self.points_earned is not None: + return self.done(points_earned=self.points_earned, message=self.message) elif self.status == "success": return self.passed(message=self.message) elif self.status == "failure": diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py new file mode 100644 index 00000000..1d25e812 --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.mark.parametrize("marks_earned", [0, 1, 2]) +def test_partial_marks(request, marks_earned: int) -> None: + request.node.add_marker(pytest.mark.markus_marks_total(2)) + request.node.add_marker(pytest.mark.markus_marks_earned(marks_earned)) + assert False, f"Should be {marks_earned}/2" diff --git a/server/autotest_server/tests/testers/py/test_py_tester.py b/server/autotest_server/tests/testers/py/test_py_tester.py index 1dee43e9..73deab71 100644 --- a/server/autotest_server/tests/testers/py/test_py_tester.py +++ b/server/autotest_server/tests/testers/py/test_py_tester.py @@ -1,5 +1,6 @@ from ....testers.specs import TestSpecs -from ....testers.py.py_tester import PyTester +from ....testers.py.py_tester import PyTester, PyTest +import json import re @@ -54,3 +55,49 @@ def test_skip(request, monkeypatch) -> None: """)) results = tester.run_python_tests() assert results == {"fixtures/sample_tests_skip.py": []} + + +def test_marks_earned_respected_when_equal_to_total(request, monkeypatch) -> None: + """Test that markus_marks_earned is respected even when earned == total and test fails (TICKET-602).""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester(specs=TestSpecs.from_json(""" + { + "test_data": { + "script_files": ["fixtures/sample_tests_marks_earned.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """)) + results = tester.run_python_tests() + test_results = results["fixtures/sample_tests_marks_earned.py"] + assert len(test_results) == 3 + + # Build PyTest instances and run them to get final marks + outputs = [] + for res in test_results: + test = PyTest(tester, "fixtures/sample_tests_marks_earned.py", res) + outputs.append(json.loads(test.run())) + + assert len(outputs) == 3 + + # Sort by marks_earned so assertions are deterministic + outputs.sort(key=lambda o: o["marks_earned"]) + + # marks_earned=0: should get 0 marks (TICKET-603) + assert outputs[0]["marks_earned"] == 0 + assert outputs[0]["marks_total"] == 2 + + # marks_earned=1: should get 1 mark (partial) + assert outputs[1]["marks_earned"] == 1 + assert outputs[1]["marks_total"] == 2 + + # marks_earned=2: should get 2 marks, not 0 (TICKET-602) + assert outputs[2]["marks_earned"] == 2 + assert outputs[2]["marks_total"] == 2