diff --git a/.gitignore b/.gitignore index 2b51df24..155619dd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__ .eggs venv venv2 +.venv build docker-compose.override.yml /workspace +.vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 893f2a57..0a390168 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.11.0 + rev: 26.3.1 hooks: - id: black - repo: https://github.com/pycqa/flake8 diff --git a/Changelog.md b/Changelog.md index fc46e74f..a269bade 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,21 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v2.10.0] +- Fixed Haskell test results to only include the function name (#687) +- Improved robustness of tester installation scripts and Docker configuration (#688) +- Moved tidyverse installation steps from server Dockerfile into R tester requirements.system (#688) +- Fixed Haskell tester installation using ghcup to install stack system-wide (#688) +- Updated tester schema generation to use msgspec datatypes (#689) +- Specify dockerfile frontend version (#691) +- Add a new Javascript (Jest) tester to support Javascript autotesting (#698) +- Add remote URL whitelist for AI tester to restrict allowed endpoints (#693) +- Increased default settings job timeout from 600s to 1200s (#707) +- Disable pytest cacheprovider to avoid creating .pytest_cache in isolated runs (#709) +- Fixed Python tester to correctly report marks when `markus_marks_earned` equals total or zero (#716) +- Fix worker orphan processes and add server-side max test timeout cap (#710) +- Resolve ghcup isolation installation failure in Haskell autotester (#725) + ## [v2.9.0] - Install stack with GHCup (#626) - Fixed AI tester to report error when the specified `submission` file is not found (#663) @@ -39,13 +54,13 @@ All notable changes to this project will be documented here. - Update R tester to allow a renv.lock file (#581) - Improve display of Python package installation errors when creating environment (#585) - Update "setting up test environment" message with http response of status code 503 (#589) -- Change rlimit resource settings to apply each worker individually (#587) +- Change rlimit resource settings to apply each worker individually (#587) - Drop support for Python 3.8 (#590) - Use Python 3.13 in development (#590) - Update Docker configuration to install dependencies in a separate service (#590) - Improve error reporting with handled assertion errors (#591) - Add custom pytest markers to Python tester to record MarkUs metadata (#592) -- Stop the autotester from running tests if there are errors in test settings (#593) +- Stop the autotester from running tests if there are errors in test settings (#593) - Implement Redis backoff strategy (#594) ## [v2.6.0] @@ -119,7 +134,7 @@ All notable changes to this project will be documented here. - Add ability to clean up test scripts that haven't been used for X days (#379) ## [v2.1.2] -- Support dependencies on specific package versions and non-CRAN sources for R tester (#323) +- Support dependencies on specific package versions and non-CRAN sources for R tester (#323) ## [v2.1.1] - Remove the requirement for clients to send unique user name (#318) @@ -140,7 +155,7 @@ All notable changes to this project will be documented here. - Add Jupyter tester (#284) ## [v1.10.3] -- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) +- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) - Pass PGHOST and PGINFO environment variables to tests (#272) - Update to new version of markus-api that supports uploading binary files (#273) - Fix bug where environment variables were not string types (#274) diff --git a/README.md b/README.md index 6282da23..6628d1cc 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ The autotester currently supports testers for the following languages and testin - [`tasty-quickcheck`](https://hackage.haskell.org/package/tasty-quickcheck) - `java` - [JUnit](https://junit.org/junit4/) +- `js` (JavaScript) + - [Jest](https://jestjs.io/) - `py` (python3) - [Unittest](https://docs.python.org/3/library/unittest.html) - [Pytest](https://docs.pytest.org/en/latest/) @@ -155,6 +157,8 @@ Installing each tester will also install the following additional packages (syst - tasty-quickcheck (cabal package) - `java` - openjdk-8-jdk +- `js` (JavaScript) + - none - `py` (python3) - none - `pyta` @@ -202,6 +206,10 @@ supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:900 worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs. +max_test_timeout: # maximum number of seconds a single test is allowed to run before being killed. + # When set, any per-test timeout exceeding this value is capped to it, and tests + # with no timeout default to this value. default is: 3600 + rlimit_settings: # RLIMIT settings (see details below) nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300 - 300 diff --git a/client/.dockerfiles/Dockerfile b/client/.dockerfiles/Dockerfile index ebc4f1ca..efc4aed8 100644 --- a/client/.dockerfiles/Dockerfile +++ b/client/.dockerfiles/Dockerfile @@ -12,4 +12,4 @@ RUN apt-get update -y && \ WORKDIR /app -CMD /markus_venv/bin/python run.py +CMD ["/markus_venv/bin/python", "run.py"] diff --git a/client/.env b/client/.env index d997fcd3..a0e0af5b 100644 --- a/client/.env +++ b/client/.env @@ -3,4 +3,4 @@ FLASK_HOST=localhost FLASK_PORT=5000 ACCESS_LOG= ERROR_LOG= -SETTINGS_JOB_TIMEOUT=600 +SETTINGS_JOB_TIMEOUT=1200 diff --git a/client/autotest_client/__init__.py b/client/autotest_client/__init__.py index ae5c67e4..bf60acb7 100644 --- a/client/autotest_client/__init__.py +++ b/client/autotest_client/__init__.py @@ -24,7 +24,7 @@ ERROR_LOG = os.environ.get("ERROR_LOG") ACCESS_LOG = os.environ.get("ACCESS_LOG") -SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 600) +SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 1200) REDIS_URL = os.environ["REDIS_URL"] REDIS_CONNECTION = redis.Redis.from_url( diff --git a/client/autotest_client/form_management.py b/client/autotest_client/form_management.py index f3fef9f9..e6ed841a 100644 --- a/client/autotest_client/form_management.py +++ b/client/autotest_client/form_management.py @@ -135,6 +135,5 @@ def validate_against_schema(test_specs: Dict, schema: Dict, filenames: List[str] schema["definitions"]["files_list"]["enum"] = filenames # don't validate based on categories schema["definitions"]["test_data_categories"].pop("enum") - schema["definitions"]["test_data_categories"].pop("enumNames") error = _validate_with_defaults(schema, test_specs, best_only=True) return str(error) if error else None diff --git a/client/requirements.txt b/client/requirements.txt index a9bf1f4c..5f5c26ed 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -1,6 +1,6 @@ -flask==3.1.2 -python-dotenv==1.2.1 -rq==2.6.0 -redis==7.1.0 -jsonschema==4.25.1 -Werkzeug==3.1.3 +flask==3.1.3 +python-dotenv==1.2.2 +rq==2.7.0 +redis==7.4.0 +jsonschema==4.26.0 +Werkzeug==3.1.7 diff --git a/compose.yaml b/compose.yaml index 39e38ec8..d49eaddc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,7 +7,7 @@ services: UBUNTU_VERSION: '24.04' LOGIN_USER: 'docker' WORKSPACE: '/home/docker/.autotesting' - image: markus-autotest-server-dev:1.4.0 + image: markus-autotest-server-dev:1.5.0 volumes: - ./server:/app:cached - venv_server:/home/docker/markus_venv @@ -29,7 +29,7 @@ services: dockerfile: ./.dockerfiles/Dockerfile args: UBUNTU_VERSION: '24.04' - image: markus-autotest-client-dev:1.4.0 + image: markus-autotest-client-dev:1.5.0 container_name: 'autotest-client' volumes: - ./client:/app:cached diff --git a/server/.dockerfiles/Dockerfile b/server/.dockerfiles/Dockerfile index 60ca5a48..a1615cc5 100644 --- a/server/.dockerfiles/Dockerfile +++ b/server/.dockerfiles/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.7-labs ARG UBUNTU_VERSION=24.04 FROM ubuntu:$UBUNTU_VERSION AS base @@ -8,7 +9,9 @@ RUN userdel -r ubuntu ARG LOGIN_USER ARG WORKSPACE -RUN apt-get update -y && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \ DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python3.11 \ @@ -21,18 +24,7 @@ RUN apt-get update -y && \ postgresql-client \ libpq-dev \ sudo \ - git \ - libfontconfig1-dev \ - libcurl4-openssl-dev \ - libfreetype6-dev \ - libpng-dev \ - libtiff5-dev \ - libjpeg-dev \ - libharfbuzz-dev \ - libfribidi-dev \ - libxml2-dev \ - libnuma-dev \ - r-base + git RUN useradd -ms /bin/bash $LOGIN_USER && \ usermod -aG sudo $LOGIN_USER && \ @@ -43,17 +35,29 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \ done && \ chmod a+x /home/${LOGIN_USER} -COPY . /app +RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \ + mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv -RUN find /app/autotest_server/testers -name requirements.system -exec {} \; -RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site +# Copy requirements.system files for all testers. These are copied separately from other files +# to avoid Docker cache invalidation of the subsequent RUN command if any other files are changed +# in markus-autotesting/server. +COPY --parents \ + ./autotest_server/testers/java/requirements.system \ + ./autotest_server/testers/js/requirements.system \ + ./autotest_server/testers/haskell/requirements.system \ + ./autotest_server/testers/r/requirements.system \ + ./autotest_server/testers/racket/requirements.system \ + /app +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=cache,target=$WORKSPACE/.stack,sharing=locked \ + find /app/autotest_server/testers -name requirements.system -exec {} \; -RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \ - mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv +COPY . /app WORKDIR /home/${LOGIN_USER} USER ${LOGIN_USER} -CMD /app/.dockerfiles/cmd-dev.sh +CMD ["/app/.dockerfiles/cmd-dev.sh"] diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index 7af410aa..f2f93dae 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -4,8 +4,8 @@ import shutil import time import json -import subprocess import signal +import subprocess import socket import getpass import requests @@ -101,6 +101,31 @@ def _kill_user_processes(test_username: str) -> None: subprocess.run(kill_cmd, shell=True) +def _kill_pgid_children(proc: subprocess.Popen) -> None: + """ + Kill all processes in our process group except the current (worker) process. + Tests may spawn subprocesses (e.g. CSC209) that share our group; + proc.kill() alone would leave those running. + Falls back to proc.kill() when /proc is unavailable (non-Linux). + """ + worker_pid = os.getpid() + worker_pgid = os.getpgid(worker_pid) + try: + for entry in os.scandir("/proc"): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + if pid == worker_pid: + continue + try: + if os.getpgid(pid) == worker_pgid: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + pass + except FileNotFoundError: + proc.kill() + + def _create_test_script_command(tester_type: str) -> str: """ Return string representing a command line command to @@ -218,13 +243,18 @@ def _run_test_specs( out, err = "", "" timeout_expired = None timeout = test_data.get("timeout") + max_timeout = config.get("max_test_timeout") + if max_timeout is not None: + if timeout is None: + timeout = max_timeout + else: + timeout = min(timeout, max_timeout) try: env = settings.get("_env", {}) env_vars = {**os.environ, **_get_env_vars(test_username), **env} env_vars = _update_env_vars(env_vars, test_env_vars) proc = subprocess.Popen( args, - start_new_session=True, cwd=tests_path, shell=True, stdout=subprocess.PIPE, @@ -238,14 +268,16 @@ def _run_test_specs( settings_json = json.dumps({**settings, "test_data": test_data}) out, err = proc.communicate(input=settings_json, timeout=timeout) except subprocess.TimeoutExpired: - if test_username == getpass.getuser(): - pgrp = os.getpgid(proc.pid) - os.killpg(pgrp, signal.SIGKILL) - else: + if test_username != getpass.getuser(): _kill_user_processes(test_username) + else: + _kill_pgid_children(proc) + proc.wait() out, err = proc.communicate() - if err == "Killed\n": # Default message from shell - test_group_name = test_data.get("extra_info", {}).get("name", "").strip() + test_group_name = test_data.get("extra_info", {}).get("name", "").strip() + if err == "Killed\n" or (not err and proc.returncode is not None and proc.returncode != 0): + # err can be "Killed\n" (shell default) or empty (SIGKILL/OOM silent crash). + # Check proc.returncode to reliably detect both cases. if test_group_name: err = f"Tests for {test_group_name} did not complete within time limit ({timeout}s)\n" else: diff --git a/server/autotest_server/schema_skeleton.json b/server/autotest_server/schema_skeleton.json index b1b76916..5385932c 100644 --- a/server/autotest_server/schema_skeleton.json +++ b/server/autotest_server/schema_skeleton.json @@ -6,8 +6,7 @@ }, "test_data_categories": { "type": "string", - "enum": [], - "enumNames": [] + "enum": [] }, "extra_group_data": {}, "installed_testers": { diff --git a/server/autotest_server/settings.yml b/server/autotest_server/settings.yml index 93502031..7684de1f 100644 --- a/server/autotest_server/settings.yml +++ b/server/autotest_server/settings.yml @@ -2,6 +2,10 @@ workspace: !ENV ${WORKSPACE} redis_url: !ENV ${REDIS_URL} supervisor_url: !ENV ${SUPERVISOR_URL} worker_log_dir: !ENV ${WORKER_LOG_DIR} +default_remote_url: https://polymouth.teach.cs.toronto.edu:443/chat +remote_url_whitelist: + - https://polymouth.teach.cs.toronto.edu:443/chat +max_test_timeout: 3600 workers: - user: !ENV ${USER} queues: diff --git a/server/autotest_server/settings_schema.json b/server/autotest_server/settings_schema.json index b14ab9b2..d0c633a5 100644 --- a/server/autotest_server/settings_schema.json +++ b/server/autotest_server/settings_schema.json @@ -19,9 +19,25 @@ "type": "string", "minLength": 1 }, + "default_remote_url": { + "type": "string", + "minLength": 1 + }, + "remote_url_whitelist": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, "rlimit_settings": { "type": "object" }, + "max_test_timeout": { + "type": "integer", + "minimum": 1 + }, "workers": { "type": "array", "minItems": 1, diff --git a/server/autotest_server/testers/__init__.py b/server/autotest_server/testers/__init__.py index 82826329..096e3c4e 100644 --- a/server/autotest_server/testers/__init__.py +++ b/server/autotest_server/testers/__init__.py @@ -1,15 +1,17 @@ -import os +from __future__ import annotations -_TESTERS = ("ai", "custom", "haskell", "java", "jupyter", "py", "pyta", "r", "racket") +import importlib +import os +_TESTERS = ("ai", "custom", "haskell", "java", "js", "jupyter", "py", "pyta", "r", "racket") -def install(testers=_TESTERS): - import importlib - settings = {} +def install(testers: list[str] = _TESTERS) -> tuple[dict, dict]: + installed_testers = [] for tester in testers: mod = importlib.import_module(f".{tester}.setup", package="autotest_server.testers") try: + print(f"[AUTOTESTER] calling autotest_server.testers.{tester}.setup.install()") mod.install() except Exception as e: msg = ( @@ -19,5 +21,25 @@ def install(testers=_TESTERS): " and then rerunning this function." ) raise Exception(msg) from e - settings[tester] = mod.settings() - return settings + installed_testers.append(tester) + return get_settings(installed_testers) + + +def get_settings(testers: list[str] = _TESTERS) -> tuple[dict, dict]: + """Return JSON schemas for the settings for the given testers. + + The return values are: + 1. A dictionary mapping tester name to JSON schema + 2. A dictionary of JSON schema definitions used by the tester schemas + """ + schemas = {} + definitions = {} + for tester in testers: + mod = importlib.import_module(f".{tester}.setup", package="autotest_server.testers") + tester_schema, tester_definitions = mod.settings() + if "title" in tester_schema and f"{tester_schema['title']}TesterSettings" in tester_definitions: + tester_definitions.pop(f"{tester_schema['title']}TesterSettings") + schemas[tester] = tester_schema + definitions.update(tester_definitions) + + return schemas, definitions diff --git a/server/autotest_server/testers/ai/ai_tester.py b/server/autotest_server/testers/ai/ai_tester.py index 8ecf6e04..e4a6b152 100644 --- a/server/autotest_server/testers/ai/ai_tester.py +++ b/server/autotest_server/testers/ai/ai_tester.py @@ -72,12 +72,12 @@ def call_ai_feedback(self) -> dict: output_mode = test_group.get("output") cmd = [sys.executable, "-m", "ai_feedback"] - # Temporarily disable non-local models + # Restrict to remote model only — prevent access to cloud AIs if config.get("model", "") != "remote": results[test_label] = { "title": test_label, "status": "error", - "message": f"Unsupported model type: \"{config.get('model', '')}\"", + "message": f"Unsupported model type: \"{config.get('model', '')}\". Only 'remote' model is allowed.", } return results @@ -88,7 +88,7 @@ def call_ai_feedback(self) -> dict: results[test_label] = { "title": test_label, "status": "error", - "message": f'Could not file submission file "{submission_file}"', + "message": f'Could not find submission file "{submission_file}"', } return results diff --git a/server/autotest_server/testers/ai/settings_schema.json b/server/autotest_server/testers/ai/settings_schema.json index 7e43b50a..6e4b29bc 100644 --- a/server/autotest_server/testers/ai/settings_schema.json +++ b/server/autotest_server/testers/ai/settings_schema.json @@ -76,7 +76,11 @@ "submission_image": { "type": "string" }, "submission_type": { "type": "string" }, "system_prompt": { "type": "string" }, - "test_output": { "type": "string" } + "test_output": { "type": "string" }, + "remote_url": { + "title": "Remote URL", + "type": "string" + } } }, "extra_info": { diff --git a/server/autotest_server/testers/ai/setup.py b/server/autotest_server/testers/ai/setup.py index c3fbeee0..cadc0aa2 100644 --- a/server/autotest_server/testers/ai/setup.py +++ b/server/autotest_server/testers/ai/setup.py @@ -48,7 +48,24 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: settings_ = json.load(f) - return settings_ + + # Late import: config reads settings.yml at module level, which fails in subprocess contexts (see 7430136) + from autotest_server.config import config + + try: + remote_url_prop = settings_["properties"]["test_data"]["items"]["properties"]["config"]["properties"][ + "remote_url" + ] + except KeyError as e: + raise RuntimeError(f"AI tester settings_schema.json missing expected 'remote_url' field: {e}") + whitelist = config.get("remote_url_whitelist", []) + if whitelist: + remote_url_prop["enum"] = whitelist + default_url = config.get("default_remote_url", "") + if default_url: + remote_url_prop["default"] = default_url + + return settings_, {} def install(): diff --git a/server/autotest_server/testers/custom/schema.py b/server/autotest_server/testers/custom/schema.py new file mode 100644 index 00000000..25ce8f1f --- /dev/null +++ b/server/autotest_server/testers/custom/schema.py @@ -0,0 +1,21 @@ +from __future__ import annotations +from typing import Annotated + +from msgspec import Meta +from markus_autotesting_core.types import AutotestFile, BaseTestData, BaseTesterSettings + + +class CustomTesterSettings(BaseTesterSettings, tag="custom"): + """The settings for the custom tester.""" + + test_data: Annotated[list[CustomTestData], Meta(title="Test Groups", min_length=1)] + + +class CustomTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the custom tester.""" + + script_files: Annotated[ + list[AutotestFile], + Meta(title="Test files", min_length=1, extra_json_schema={"uniqueItems": True}), + ] + """The file(s) that contain the tests to execute.""" diff --git a/server/autotest_server/testers/custom/settings_schema.json b/server/autotest_server/testers/custom/settings_schema.json deleted file mode 100644 index bec7ec1c..00000000 --- a/server/autotest_server/testers/custom/settings_schema.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "type": "object", - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "custom" - ] - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout" - ], - "properties": { - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/files_list" - }, - "uniqueItems": true - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} diff --git a/server/autotest_server/testers/custom/setup.py b/server/autotest_server/testers/custom/setup.py index 59c8ea18..7d25d238 100644 --- a/server/autotest_server/testers/custom/setup.py +++ b/server/autotest_server/testers/custom/setup.py @@ -1,5 +1,7 @@ import os -import json + +from ..schema import generate_schema +from .schema import CustomTesterSettings def create_environment(_settings, _env_dir, default_env_dir): @@ -11,5 +13,4 @@ def install(): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - return json.load(f) + return generate_schema(CustomTesterSettings) diff --git a/server/autotest_server/testers/haskell/config.py b/server/autotest_server/testers/haskell/config.py new file mode 100644 index 00000000..fe4a5803 --- /dev/null +++ b/server/autotest_server/testers/haskell/config.py @@ -0,0 +1,2 @@ +HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"] +STACK_RESOLVER = "lts-21.21" diff --git a/server/autotest_server/testers/haskell/haskell_tester.py b/server/autotest_server/testers/haskell/haskell_tester.py index 78010e53..48674fde 100644 --- a/server/autotest_server/testers/haskell/haskell_tester.py +++ b/server/autotest_server/testers/haskell/haskell_tester.py @@ -7,9 +7,6 @@ from ..tester import Tester, Test, TestError from ..specs import TestSpecs -home = os.getenv("HOME") -os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"] - class HaskellTest(Test): def __init__( diff --git a/server/autotest_server/testers/haskell/lib/Stats.hs b/server/autotest_server/testers/haskell/lib/Stats.hs index c96b7adb..77733b23 100644 --- a/server/autotest_server/testers/haskell/lib/Stats.hs +++ b/server/autotest_server/testers/haskell/lib/Stats.hs @@ -85,4 +85,10 @@ resultRow results = do \(show -> idx, (name, Result { resultDescription=dropWhileEnd isSpace -> description , resultShortDescription=result , resultTime=show -> time })) -> - [idx, name, time, result, description] + [ idx + -- Extract the property name from the TestName. Tasty uses '.' to join module and test name. + , reverse $ takeWhile (/= '.') $ reverse name + , time + , result + , description + ] diff --git a/server/autotest_server/testers/haskell/requirements.system b/server/autotest_server/testers/haskell/requirements.system index 7b493126..48176b85 100755 --- a/server/autotest_server/testers/haskell/requirements.system +++ b/server/autotest_server/testers/haskell/requirements.system @@ -1,15 +1,33 @@ #!/usr/bin/env bash +set -euxo pipefail -apt-get -y update - +# Install a system-wide ghc, which can be used as a default version in the Haskell tester. +# This should be synchronized with the LTS version and dependencies in config.py if ! dpkg -l ghc cabal-install &> /dev/null; then + apt-get -y update DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install fi -if [ ! -x "$HOME/.ghcup/bin/ghcup" ] && [ ! -x "/usr/local/bin/stack" ]; then - BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_INSTALL_NO_STACK=1 curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh - $HOME/.ghcup/bin/ghcup install stack recommended - if [ "$(id -u)" -eq 0 ]; then - cp $HOME/.ghcup/bin/stack /usr/local/bin/ - fi -fi +if [ ! -x "/usr/local/bin/stack" ]; then + # We use ghcup to install stack rather than relying on system packages. This enables newer versions of Stack to be installed. + # Install ghcup dependencies: https://www.haskell.org/ghcup/install/#linux-ubuntu + apt-get -y update + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' \ + build-essential \ + curl \ + libffi-dev \ + libffi8ubuntu1 \ + libgmp-dev \ + libgmp10 \ + libncurses-dev + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh + + # Create a dedicated, empty folder to satisfy ghcup's isolation requirements + mkdir -p /usr/local/bin/stack-isolated + + # Install stack cleanly into the isolated subfolder + ~/.ghcup/bin/ghcup install stack recommended --isolate /usr/local/bin/stack-isolated + + # Link the executable back into the global PATH + ln -sf /usr/local/bin/stack-isolated/stack /usr/local/bin/stack +fi \ No newline at end of file diff --git a/server/autotest_server/testers/haskell/schema.py b/server/autotest_server/testers/haskell/schema.py new file mode 100644 index 00000000..8e73feec --- /dev/null +++ b/server/autotest_server/testers/haskell/schema.py @@ -0,0 +1,27 @@ +from __future__ import annotations +from typing import Annotated + +from msgspec import Meta, Struct +from markus_autotesting_core.types import BaseTestData, BaseTesterSettings + +from .config import STACK_RESOLVER + + +class HaskellTesterSettings(BaseTesterSettings): + """The settings for the Haskell tester.""" + + env_data: Annotated[HaskellEnvData, Meta(title="Haskell environment")] + test_data: Annotated[list[HaskellTestData], Meta(title="Test Groups", min_length=1)] + + +class HaskellTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the Haskell tester.""" + + test_timeout: Annotated[int, Meta(title="Per-test timeout")] = 10 + test_cases: Annotated[int, Meta(title="Number of test cases (QuickCheck)")] = 100 + + +class HaskellEnvData(Struct, kw_only=True): + """Settings for the Haskell environment""" + + resolver_version: Annotated[str, Meta(title="Stackage LTS resolver version")] = STACK_RESOLVER diff --git a/server/autotest_server/testers/haskell/settings_schema.json b/server/autotest_server/testers/haskell/settings_schema.json deleted file mode 100644 index 5fd49f3d..00000000 --- a/server/autotest_server/testers/haskell/settings_schema.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "type": "object", - "required": [ - "env_data" - ], - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "haskell" - ] - }, - "env_data": { - "title": "Haskell environment", - "type": "object", - "required": [ - "resolver_version" - ], - "properties": { - "resolver_version": { - "title": "Resolver version", - "type": "string", - "default": null - } - } - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout", - "test_timeout", - "test_cases" - ], - "properties": { - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/files_list" - }, - "uniqueItems": true - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "test_timeout": { - "title": "Per-test timeout", - "type": "integer", - "default": 10 - }, - "test_cases": { - "title": "Number of test cases (QuickCheck)", - "type": "integer", - "default": 100 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} diff --git a/server/autotest_server/testers/haskell/setup.py b/server/autotest_server/testers/haskell/setup.py index 48e4d3f7..c3cb2104 100644 --- a/server/autotest_server/testers/haskell/setup.py +++ b/server/autotest_server/testers/haskell/setup.py @@ -1,12 +1,9 @@ import os -import json import subprocess -HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"] -STACK_RESOLVER = "lts-21.21" - -home = os.getenv("HOME") -os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"] +from .config import HASKELL_TEST_DEPS, STACK_RESOLVER +from ..schema import generate_schema +from .schema import HaskellTesterSettings def create_environment(_settings, _env_dir, default_env_dir): @@ -20,26 +17,33 @@ def create_environment(_settings, _env_dir, default_env_dir): def install(): try: - subprocess.run( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), - check=True, - capture_output=True, - text=True, - ) + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system") + print(f"[AUTOTESTER] Running {path}", flush=True) + subprocess.run(path, check=True) except subprocess.CalledProcessError as e: raise RuntimeError(f"Error executing Haskell requirements.system: {e}") resolver = STACK_RESOLVER - cmd = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS] + + cmd_update = ["stack", "update"] + print(f'[AUTOTESTER] Running {" ".join(cmd_update)}', flush=True) + try: + subprocess.run(cmd_update, check=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running {cmd_update}: {e}") + + cmd_build = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS] + print(f'[AUTOTESTER] Running {" ".join(cmd_build)}', flush=True) try: - subprocess.run(cmd, check=True, capture_output=True) + subprocess.run(cmd_build, check=True) except subprocess.CalledProcessError as e: - raise RuntimeError(f"Error running {cmd}: {e}") + raise RuntimeError(f"Error running {cmd_build}: {e}") try: + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh") + print(f"[AUTOTESTER] Running {path}", flush=True) subprocess.run( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh"), + path, check=True, shell=True, - capture_output=True, text=True, ) except subprocess.CalledProcessError as e: @@ -47,8 +51,4 @@ def install(): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - settings_ = json.load(f) - resolver_versions = settings_["properties"]["env_data"]["properties"]["resolver_version"] - resolver_versions["default"] = STACK_RESOLVER - return settings_ + return generate_schema(HaskellTesterSettings) diff --git a/server/autotest_server/testers/haskell/stack_permissions.sh b/server/autotest_server/testers/haskell/stack_permissions.sh index 2bb5d2f3..f24bd8bd 100755 --- a/server/autotest_server/testers/haskell/stack_permissions.sh +++ b/server/autotest_server/testers/haskell/stack_permissions.sh @@ -1,3 +1,6 @@ +#!/usr/bin/env bash +set -euxo pipefail + echo "allow-different-user: true" >> $STACK_ROOT/config.yaml echo "recommend-stack-upgrade: false" >> $STACK_ROOT/config.yaml chmod a+w $STACK_ROOT/stack.sqlite3.pantry-write-lock diff --git a/server/autotest_server/testers/java/requirements.system b/server/autotest_server/testers/java/requirements.system index bbe96712..27082c3a 100755 --- a/server/autotest_server/testers/java/requirements.system +++ b/server/autotest_server/testers/java/requirements.system @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euxo pipefail if ! dpkg -l openjdk-8-jdk wget &> /dev/null; then apt-get -y update diff --git a/server/autotest_server/testers/java/schema.py b/server/autotest_server/testers/java/schema.py new file mode 100644 index 00000000..6629a451 --- /dev/null +++ b/server/autotest_server/testers/java/schema.py @@ -0,0 +1,18 @@ +from __future__ import annotations +from typing import Annotated + +from msgspec import Meta +from markus_autotesting_core.types import BaseTestData, BaseTesterSettings + + +class JavaTesterSettings(BaseTesterSettings): + """The settings for the Java tester.""" + + test_data: Annotated[list[JavaTestData], Meta(title="Test Groups", min_length=1)] + + +class JavaTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the Java tester.""" + + classpath: Annotated[str, Meta(title="Java Class Path")] = "." + sources_path: Annotated[str, Meta(title="Java Sources (glob)")] = "" diff --git a/server/autotest_server/testers/java/settings_schema.json b/server/autotest_server/testers/java/settings_schema.json deleted file mode 100644 index 073a729a..00000000 --- a/server/autotest_server/testers/java/settings_schema.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "type": "object", - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "java" - ] - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout" - ], - "properties": { - "classpath": { - "title": "Java Class Path", - "type": "string" - }, - "sources_path": { - "title": "Java Sources (glob)", - "type": "string" - }, - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/files_list" - }, - "uniqueItems": true - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} diff --git a/server/autotest_server/testers/java/setup.py b/server/autotest_server/testers/java/setup.py index 26c8a018..dd3adc3a 100644 --- a/server/autotest_server/testers/java/setup.py +++ b/server/autotest_server/testers/java/setup.py @@ -1,8 +1,10 @@ import os -import json import subprocess import requests +from ..schema import generate_schema +from .schema import JavaTesterSettings + def create_environment(_settings, _env_dir, default_env_dir): return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")} @@ -10,7 +12,9 @@ def create_environment(_settings, _env_dir, default_env_dir): def install(): this_dir = os.path.dirname(os.path.realpath(__file__)) - subprocess.run(os.path.join(this_dir, "requirements.system"), check=True) + path = os.path.join(this_dir, "requirements.system") + print(f"[AUTOTESTER] Running {path}", flush=True) + subprocess.run(path, check=True) url = ( "https://repo1.maven.org/maven2/org/junit/platform/junit-platform-console-standalone/1.7.0/junit-platform" "-console-standalone-1.7.0.jar" @@ -21,5 +25,4 @@ def install(): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - return json.load(f) + return generate_schema(JavaTesterSettings) diff --git a/server/autotest_server/testers/js/__init__.py b/server/autotest_server/testers/js/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/testers/js/js_tester.py b/server/autotest_server/testers/js/js_tester.py new file mode 100644 index 00000000..3ce629e3 --- /dev/null +++ b/server/autotest_server/testers/js/js_tester.py @@ -0,0 +1,138 @@ +import subprocess +import json +import os + +from ..tester import Tester, Test, TestError +from ..specs import TestSpecs + + +class JsTest(Test): + def __init__(self, tester, result): + self.test_name_ = result.get("fullName", "unknown") + self.status = result.get("status") + self.message = "\n".join(result.get("failureMessages", [])) + super().__init__(tester) + + @property + def test_name(self): + """Return the full name of this test.""" + return self.test_name_ + + @Test.run_decorator + def run(self): + """Return the result of this test based on its Jest status.""" + if self.status == "passed": + return self.passed() + elif self.status == "failed": + return self.failed(self.message) + else: + return self.error(message=self.message or f"Unexpected status: {self.status}") + + +class JsTester(Tester): + + def __init__( + self, + specs: TestSpecs, + test_class=JsTest, + resource_settings: list[tuple[int, tuple[int, int]]] | None = None, + ) -> None: + """ + Initialize a JavaScript tester using the specifications in specs. + + This tester will create tests of type test_class. + """ + super().__init__(specs, test_class, resource_settings=resource_settings) + + def _run_pnpm_install(self, dir_path): + """ + Run pnpm install in dir_path to install dependencies from package.json. + """ + corepack_home = os.path.join(dir_path, ".corepack") + xdg_cache_home = os.path.join(dir_path, ".cache") + os.makedirs(corepack_home, exist_ok=True) + os.makedirs(xdg_cache_home, exist_ok=True) + env = {**os.environ, "COREPACK_HOME": corepack_home, "XDG_CACHE_HOME": xdg_cache_home} + result = subprocess.run( + ["pnpm", "install"], + capture_output=True, + text=True, + cwd=dir_path, + env=env, + ) + return result + + def _run_jest(self, dir_path, timeout, test_files=None): + """ + Run Jest in dir_path and return its stdout and return code. + + --json: output results as JSON to stdout + --forceExit: prevents jest from hanging if tests leave open connections + --runInBand: run all tests serially in the current process + """ + cmd = ["pnpm", "exec", "jest", "--rootDir", dir_path, "--json", "--forceExit", "--runInBand"] + if test_files: + cmd.extend(test_files) + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=dir_path, + timeout=timeout, + ) + return result.stdout, result.returncode + + def _parse_jest_output(self, raw_json): + """ + Parse Jest's JSON output and return a list of individual test results. + + Returns a tuple (results, error) where error is set if JSON parsing fails. + """ + try: + data = json.loads(raw_json) + except json.JSONDecodeError as e: + return None, e + + results = [] + for test_suite in data.get("testResults", []): + for test in test_suite.get("assertionResults", []): + results.append(test) + + return results, None + + @Tester.run_decorator + def run(self): + """ + Run pnpm install and then Jest, parsing the results and printing each test outcome. + """ + dir_path = os.getcwd() + test_data = self.specs.get("test_data", default={}) or {} + + timeout = test_data.get("timeout", 30) + if os.path.isfile(os.path.join(dir_path, "package.json")): + pnpm_result = self._run_pnpm_install(dir_path) + if pnpm_result.returncode != 0: + err = pnpm_result.stderr or pnpm_result.stdout or "(no output)" + raise TestError(f"pnpm install failed:\n{err}") + + script_files = test_data.get("script_files", []) + try: + jest_json_output, jest_rc = self._run_jest(dir_path, timeout, test_files=script_files) + except subprocess.TimeoutExpired: + raise TestError("Jest timed out") + + if jest_rc != 0: + err = jest_json_output + raise TestError(f"Jest failed (exit code {jest_rc}).\n{err}") + if not jest_json_output: + raise TestError("Jest produced no output") + + test_results, err = self._parse_jest_output(jest_json_output) + if err: + raise TestError(str(err)) + + for result in test_results: + if result.get("status") == "pending": + continue + test = self.test_class(self, result) + print(test.run(), flush=True) diff --git a/server/autotest_server/testers/js/requirements.system b/server/autotest_server/testers/js/requirements.system new file mode 100755 index 00000000..f717c8f0 --- /dev/null +++ b/server/autotest_server/testers/js/requirements.system @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euxo pipefail + +if ! command -v node &> /dev/null; then + apt-get -y update + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' curl ca-certificates + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' nodejs +fi + +if ! command -v pnpm &> /dev/null; then + corepack enable + corepack prepare pnpm@latest --activate +fi + +if ! command -v jest &> /dev/null; then + export PNPM_HOME="/usr/local/share/pnpm" + export PATH="${PNPM_HOME}:${PATH}" + pnpm config set global-bin-dir /usr/local/bin + pnpm config set global-dir /usr/local/share/pnpm-global + pnpm add -g jest +fi diff --git a/server/autotest_server/testers/js/schema.py b/server/autotest_server/testers/js/schema.py new file mode 100644 index 00000000..9c0bbfab --- /dev/null +++ b/server/autotest_server/testers/js/schema.py @@ -0,0 +1,16 @@ +from __future__ import annotations +from typing import Annotated +from msgspec import Meta +from markus_autotesting_core.types import BaseTestData, BaseTesterSettings + + +class JsTesterSettings(BaseTesterSettings): + """The settings for the JavaScript tester.""" + + test_data: Annotated[list[JsTestData], Meta(title="Test Groups", min_length=1)] + + +class JsTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the JavaScript tester.""" + + pass diff --git a/server/autotest_server/testers/js/setup.py b/server/autotest_server/testers/js/setup.py new file mode 100644 index 00000000..f1819e29 --- /dev/null +++ b/server/autotest_server/testers/js/setup.py @@ -0,0 +1,19 @@ +import os +import subprocess +from ..schema import generate_schema +from .schema import JsTesterSettings + + +def create_environment(_settings, _env_dir, default_env_dir): + return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")} + + +def install(): + """Run the requirements.system shell script to install Node.js, pnpm, and Jest.""" + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system") + print(f"[AUTOTESTER] Running {path}", flush=True) + subprocess.run(path, check=True) + + +def settings(): + return generate_schema(JsTesterSettings) diff --git a/server/autotest_server/testers/jupyter/jupyter_tester.py b/server/autotest_server/testers/jupyter/jupyter_tester.py index f38a4adc..b3f8dd77 100644 --- a/server/autotest_server/testers/jupyter/jupyter_tester.py +++ b/server/autotest_server/testers/jupyter/jupyter_tester.py @@ -73,7 +73,10 @@ def _run_jupyter_tests(test_file: str) -> List[Dict]: try: sys.stdout = null_out plugin = JupyterPlugin() - pytest.main([test_file], plugins=["notebook_helper.pytest.notebook_collector_plugin", plugin]) + pytest.main( + [test_file, "-p", "no:cacheprovider"], + plugins=["notebook_helper.pytest.notebook_collector_plugin", plugin], + ) results.extend(plugin.results.values()) finally: sys.stdout = sys.__stdout__ diff --git a/server/autotest_server/testers/jupyter/schema.py b/server/autotest_server/testers/jupyter/schema.py new file mode 100644 index 00000000..968269e3 --- /dev/null +++ b/server/autotest_server/testers/jupyter/schema.py @@ -0,0 +1,40 @@ +from __future__ import annotations +from enum import Enum +import shutil +from typing import Annotated + +from msgspec import Meta, Struct +from markus_autotesting_core.types import AutotestFile, BaseTestData, BaseTesterSettings + +PYTHON_VERSIONS = [f"3.{x}" for x in range(11, 14) if shutil.which(f"python3.{x}")] +PythonVersion = Enum("PythonVersion", {v.replace(".", "_"): v for v in PYTHON_VERSIONS}) + + +class JupyterTesterSettings(BaseTesterSettings): + """The settings for the Jupyter tester.""" + + env_data: Annotated[JupyterEnvData, Meta(title="Python environment")] + test_data: Annotated[list[JupyterTestData], Meta(title="Test Groups", min_length=1)] + + +class JupyterTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the Jupyter tester.""" + + script_files: Annotated[list[JupyterScriptFile], Meta(title="Test files", min_length=1)] + """The file(s) that contain the tests to execute.""" + + +class JupyterScriptFile(Struct, kw_only=True): + """The configuration for a single Jupyter test file.""" + + test_file: Annotated[AutotestFile, Meta(title="Test file")] + student_file: Annotated[str, Meta(title="Student file")] + test_merge: Annotated[bool, Meta(title="Test that files can be merged")] = False + + +class JupyterEnvData(Struct, kw_only=True): + """The settings for the Python environment.""" + + python_version: Annotated[PythonVersion, Meta(title="Python version")] = PYTHON_VERSIONS[-1] + pip_requirements: Annotated[str, Meta(title="Package requirements")] = "" + pip_requirements_file: Annotated[str, Meta(title="Package requirements file")] = "" diff --git a/server/autotest_server/testers/jupyter/settings_schema.json b/server/autotest_server/testers/jupyter/settings_schema.json deleted file mode 100644 index f425b1bc..00000000 --- a/server/autotest_server/testers/jupyter/settings_schema.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "type": "object", - "required": [ - "env_data" - ], - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "jupyter" - ] - }, - "env_data": { - "title": "Python environment", - "type": "object", - "required": [ - "python_version" - ], - "properties": { - "python_version": { - "title": "Python version", - "type": "string", - "enum": [] - }, - "pip_requirements": { - "title": "Package requirements", - "type": "string" - }, - "pip_requirements_file": { - "title": "Package requirements file", - "type": "string" - } - } - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout" - ], - "properties": { - "script_files": { - "items": { - "type": "object", - "required": ["test_file", "student_file", "test_merge"], - "properties": { - "test_file": { - "title": "Test file", - "$ref": "#/definitions/files_list" - }, - "student_file": { - "title": "Student file", - "type": "string" - }, - "test_merge": { - "title": "Test that files can be merged", - "type": "boolean", - "default": false - } - } - }, - "minItems": 1, - "title": "Test files", - "type": "array", - "uniqueItems": true, - "default": [] - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} \ No newline at end of file diff --git a/server/autotest_server/testers/jupyter/setup.py b/server/autotest_server/testers/jupyter/setup.py index e48918a5..f4cb96b2 100644 --- a/server/autotest_server/testers/jupyter/setup.py +++ b/server/autotest_server/testers/jupyter/setup.py @@ -1,8 +1,9 @@ import os -import shutil -import json import subprocess +from ..schema import generate_schema +from .schema import JupyterTesterSettings + def create_environment(settings_, env_dir, _default_env_dir): env_data = settings_.get("env_data", {}) @@ -22,13 +23,7 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - settings_ = json.load(f) - py_versions = [f"3.{x}" for x in range(11, 14) if shutil.which(f"python3.{x}")] - python_versions = settings_["properties"]["env_data"]["properties"]["python_version"] - python_versions["enum"] = py_versions - python_versions["default"] = py_versions[-1] - return settings_ + return generate_schema(JupyterTesterSettings) def install(): diff --git a/server/autotest_server/testers/py/lib/c_helper.py b/server/autotest_server/testers/py/lib/c_helper.py index 36691949..a3e41735 100644 --- a/server/autotest_server/testers/py/lib/c_helper.py +++ b/server/autotest_server/testers/py/lib/c_helper.py @@ -9,7 +9,6 @@ from typing import Optional, List import unittest - DEFAULT_LTRACE_LOG_FILE = "ltrace_log.txt" DEFAULT_GCC_FLAGS = ["-std=gnu99", "-Wall", "-g"] DEFAULT_LTRACE_FLAGS = ["-f", "-n", "2", "-o", DEFAULT_LTRACE_LOG_FILE] diff --git a/server/autotest_server/testers/py/py_tester.py b/server/autotest_server/testers/py/py_tester.py index 61ae13c7..5f0e468b 100644 --- a/server/autotest_server/testers/py/py_tester.py +++ b/server/autotest_server/testers/py/py_tester.py @@ -251,11 +251,8 @@ def run(self) -> str: """ Return a json string containing all test result information. """ - if self.points_earned is not None and 0 < self.points_earned < self.points_total: - return self.partially_passed(points_earned=self.points_earned, message=self.message) - elif self.points_earned is not None and self.points_earned > self.points_total: - bonus = self.points_earned - self.points_total - return self.passed_with_bonus(points_bonus=bonus, message=self.message) + if self.points_earned is not None: + return self.done(points_earned=self.points_earned, message=self.message) elif self.status == "success": return self.passed(message=self.message) elif self.status == "failure": @@ -301,7 +298,7 @@ def _run_unittest_tests(self, test_file: str) -> List[Dict]: test_suite = self._load_unittest_tests(test_file) with open(os.devnull, "w") as nullstream: test_runner = unittest.TextTestRunner( - verbosity=self.specs["test_data", "output_verbosity"], + verbosity=self.specs["test_data", "output_verbosity"] or "2", stream=nullstream, resultclass=TextTestResults, ) @@ -317,9 +314,9 @@ def _run_pytest_tests(self, test_file: str) -> List[Dict]: with open(os.devnull, "w") as null_out: try: sys.stdout = null_out - verbosity = self.specs["test_data", "output_verbosity"] + verbosity = self.specs["test_data", "output_verbosity"] or "short" plugin = PytestPlugin() - pytest.main([test_file, f"--tb={verbosity}"], plugins=[plugin]) + pytest.main([test_file, f"--tb={verbosity}", "-p", "no:cacheprovider"], plugins=[plugin]) results.extend(plugin.results.values()) self.annotations = plugin.annotations self.overall_comments = plugin.overall_comments diff --git a/server/autotest_server/testers/py/schema.py b/server/autotest_server/testers/py/schema.py new file mode 100644 index 00000000..7304f18b --- /dev/null +++ b/server/autotest_server/testers/py/schema.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from enum import Enum +import shutil +from typing import Annotated, Literal + +from msgspec import Meta, Struct +from markus_autotesting_core.types import BaseTestData, BaseTesterSettings + +PYTHON_VERSIONS = [f"3.{x}" for x in range(11, 14) if shutil.which(f"python3.{x}")] +PythonVersion = Enum("PythonVersion", {v.replace(".", "_"): v for v in PYTHON_VERSIONS}) + + +class PyTesterSettings(BaseTesterSettings): + """The settings for the Python tester.""" + + env_data: Annotated[PythonEnvData, Meta(title="Python environment")] + test_data: Annotated[list[PyTestData], Meta(title="Test Groups", min_length=1)] + + +class PythonEnvData(Struct, kw_only=True): + """The settings for the Python environment.""" + + python_version: Annotated[PythonVersion, Meta(title="Python version")] = PYTHON_VERSIONS[-1] + pip_requirements: Annotated[str, Meta(title="Package requirements")] = "" + pip_requirements_file: Annotated[str, Meta(title="Package requirements file")] = "" + + +class PyTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the Python tester.""" + + tester: Annotated[Literal["unittest", "pytest"], Meta(title="Test runner")] = "pytest" + output_verbosity: Annotated[ + Literal["", "0", "1", "2", "short", "auto", "long", "no", "line", "native"], Meta(title="Output verbosity") + ] = "" diff --git a/server/autotest_server/testers/py/settings_schema.json b/server/autotest_server/testers/py/settings_schema.json deleted file mode 100644 index b5e1f381..00000000 --- a/server/autotest_server/testers/py/settings_schema.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "type": "object", - "required": [ - "env_data" - ], - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "py" - ] - }, - "env_data": { - "title": "Python environment", - "type": "object", - "required": [ - "python_version" - ], - "properties": { - "python_version": { - "title": "Python version", - "type": "string", - "enum": [] - }, - "pip_requirements": { - "title": "Package requirements", - "type": "string" - }, - "pip_requirements_file": { - "title": "Package requirements file", - "type": "string" - } - } - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout", - "tester", - "output_verbosity" - ], - "properties": { - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/files_list" - }, - "uniqueItems": true - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "tester": { - "title": "Test runner", - "type": "string", - "enum": [ - "pytest", - "unittest" - ], - "default": "pytest" - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - }, - "dependencies": { - "tester": { - "oneOf": [ - { - "properties": { - "tester": { - "type": "string", - "enum": [ - "pytest" - ] - }, - "output_verbosity": { - "title": "Pytest output verbosity", - "type": "string", - "enum": [ - "short", - "auto", - "long", - "no", - "line", - "native" - ], - "default": "short" - } - } - }, - { - "properties": { - "tester": { - "type": "string", - "enum": [ - "unittest" - ] - }, - "output_verbosity": { - "title": "Unittest output verbosity", - "type": "integer", - "enum": [ - 2, - 1, - 0 - ], - "default": 2 - } - } - } - ] - } - } - } - } - } -} \ No newline at end of file diff --git a/server/autotest_server/testers/py/setup.py b/server/autotest_server/testers/py/setup.py index e48918a5..8de0b2ff 100644 --- a/server/autotest_server/testers/py/setup.py +++ b/server/autotest_server/testers/py/setup.py @@ -1,8 +1,9 @@ import os -import shutil -import json import subprocess +from ..schema import generate_schema +from .schema import PyTesterSettings + def create_environment(settings_, env_dir, _default_env_dir): env_data = settings_.get("env_data", {}) @@ -22,13 +23,47 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - settings_ = json.load(f) - py_versions = [f"3.{x}" for x in range(11, 14) if shutil.which(f"python3.{x}")] - python_versions = settings_["properties"]["env_data"]["properties"]["python_version"] - python_versions["enum"] = py_versions - python_versions["default"] = py_versions[-1] - return settings_ + json_schema, components = generate_schema(PyTesterSettings) + + # Modify output_verbosity enum manually. msgspec does not support JSON schema generation for + # Literal type annotations that contain multiple types. + components["PyTestData"]["properties"]["output_verbosity"]["enum"] = [ + "", + 0, + 1, + 2, + "auto", + "line", + "long", + "native", + "no", + "short", + ] + + # Inject dependencies for output_verbosity for JSON Schema form + json_schema["properties"]["test_data"]["items"]["dependencies"] = { + "tester": { + "oneOf": [ + { + "properties": { + "tester": {"enum": ["pytest"]}, + "output_verbosity": { + "enum": ["short", "auto", "long", "no", "line", "native"], + "default": "short", + }, + } + }, + { + "properties": { + "tester": {"enum": ["unittest"]}, + "output_verbosity": {"enum": [0, 1, 2], "default": 2}, + } + }, + ] + } + } + + return json_schema, components def install(): diff --git a/server/autotest_server/testers/pyta/setup.py b/server/autotest_server/testers/pyta/setup.py index 431affca..de06ca27 100644 --- a/server/autotest_server/testers/pyta/setup.py +++ b/server/autotest_server/testers/pyta/setup.py @@ -36,7 +36,7 @@ def settings(): python_versions["default"] = py_versions[-1] pyta_version = settings_["properties"]["env_data"]["properties"]["pyta_version"] pyta_version["default"] = PYTA_VERSION - return settings_ + return settings_, {} def install(): diff --git a/server/autotest_server/testers/r/requirements.system b/server/autotest_server/testers/r/requirements.system index 4e65d8aa..8ffc5ea3 100755 --- a/server/autotest_server/testers/r/requirements.system +++ b/server/autotest_server/testers/r/requirements.system @@ -1,6 +1,29 @@ #!/usr/bin/env bash +set -euxo pipefail if ! dpkg -l r-base &> /dev/null; then apt-get -y update DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' r-base + + # Set global R timezone + echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site + + # Install system requirements for tidyverse. Obtained by running `pak::pkg_system_requirements("tidyverse")`. + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' \ + libx11-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + make \ + zlib1g-dev \ + pandoc \ + libfreetype6-dev \ + libjpeg-dev \ + libpng-dev \ + libtiff-dev \ + libwebp-dev \ + libicu-dev \ + libfontconfig1-dev \ + libfribidi-dev \ + libharfbuzz-dev \ + libxml2-dev fi diff --git a/server/autotest_server/testers/r/schema.py b/server/autotest_server/testers/r/schema.py new file mode 100644 index 00000000..2d9e2685 --- /dev/null +++ b/server/autotest_server/testers/r/schema.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from typing import Annotated + +from msgspec import field, Meta, Struct +from markus_autotesting_core.types import BaseTestData, BaseTesterSettings + + +class RTesterSettings(BaseTesterSettings): + """The settings for the R tester.""" + + env_data: Annotated[REnvData, Meta(title="R environment")] + test_data: Annotated[list[RTestData], Meta(title="Test Groups", min_length=1)] + + +class REnvData(Struct, kw_only=True): + """Settings for the R environment""" + + renv_lock: Annotated[bool, Meta(title="Use renv to set up environment")] = field(default=False, name="renv.lock") + requirements: Annotated[str, Meta(title="R package requirements")] = "" + + +class RTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the R tester.""" + + pass diff --git a/server/autotest_server/testers/r/settings_schema.json b/server/autotest_server/testers/r/settings_schema.json deleted file mode 100644 index 9b9d786d..00000000 --- a/server/autotest_server/testers/r/settings_schema.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "type": "object", - "required": [ - "env_data" - ], - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "r" - ] - }, - "env_data": { - "title": "R environment", - "type": "object", - "properties": { - "renv.lock": { - "title": "Use renv to set up environment", - "type": "boolean", - "default": false - }, - "requirements": { - "title": "Package requirements", - "type": "string" - } - } - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout" - ], - "properties": { - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/files_list" - }, - "uniqueItems": true - }, - "category": { - "title": "Category", - "type": "array", - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} diff --git a/server/autotest_server/testers/r/setup.py b/server/autotest_server/testers/r/setup.py index 6d418582..28af63d9 100644 --- a/server/autotest_server/testers/r/setup.py +++ b/server/autotest_server/testers/r/setup.py @@ -1,7 +1,9 @@ import os -import json import subprocess +from ..schema import generate_schema +from .schema import RTesterSettings + def create_environment(settings_, env_dir, default_env_dir): env_data = settings_.get("env_data", {}) @@ -34,9 +36,10 @@ def create_environment(settings_, env_dir, default_env_dir): def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - return json.load(f) + return generate_schema(RTesterSettings) def install(): - subprocess.run(os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), check=True) + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system") + print(f"[AUTOTESTER] Running {path}", flush=True) + subprocess.run(path, check=True) diff --git a/server/autotest_server/testers/racket/requirements.system b/server/autotest_server/testers/racket/requirements.system index dfd27b29..4dc70c06 100755 --- a/server/autotest_server/testers/racket/requirements.system +++ b/server/autotest_server/testers/racket/requirements.system @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euxo pipefail if ! dpkg -l racket &> /dev/null; then apt-get -y update diff --git a/server/autotest_server/testers/racket/schema.py b/server/autotest_server/testers/racket/schema.py new file mode 100644 index 00000000..47ac45f2 --- /dev/null +++ b/server/autotest_server/testers/racket/schema.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from typing import Annotated + +from msgspec import Meta, Struct +from markus_autotesting_core.types import AutotestFile, BaseTestData, BaseTesterSettings + + +class RacketTesterSettings(BaseTesterSettings): + """The settings for the Racket tester.""" + + test_data: Annotated[list[RacketTestData], Meta(title="Test Groups", min_length=1)] + + +class RacketTestData(BaseTestData, kw_only=True): + """The `test_data` specification for the Racket tester.""" + + script_files: Annotated[list[_RacketScriptFile], Meta(title="Test files", min_length=1)] + """The file(s) that contain the tests to execute.""" + + +class _RacketScriptFile(Struct, kw_only=True): + """The configuration for a single Racket test file.""" + + script_file: Annotated[AutotestFile, Meta(title="Test file")] + test_suite_name: Annotated[str, Meta(title="Test suite name")] = "all-tests" diff --git a/server/autotest_server/testers/racket/settings_schema.json b/server/autotest_server/testers/racket/settings_schema.json deleted file mode 100644 index eaa9b94a..00000000 --- a/server/autotest_server/testers/racket/settings_schema.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "type": "object", - "properties": { - "tester_type": { - "type": "string", - "enum": [ - "racket" - ] - }, - "test_data": { - "title": "Test Groups", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "script_files", - "timeout" - ], - "properties": { - "script_files": { - "title": "Test files", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "properties": { - "script_file": { - "title": "Test file", - "$ref": "#/definitions/files_list" - }, - "test_suite_name": { - "title": "Test suite name", - "type": "string", - "default": "all-tests" - } - } - } - }, - "category": { - "title": "Category", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/test_data_categories" - }, - "uniqueItems": true - }, - "timeout": { - "title": "Timeout", - "type": "integer", - "default": 30 - }, - "feedback_file_names": { - "title": "Feedback files", - "type": "array", - "items": { - "type": "string" - } - }, - "extra_info": { - "$ref": "#/definitions/extra_group_data" - } - } - } - } - } -} diff --git a/server/autotest_server/testers/racket/setup.py b/server/autotest_server/testers/racket/setup.py index a80bcc6a..f3f9b1ec 100644 --- a/server/autotest_server/testers/racket/setup.py +++ b/server/autotest_server/testers/racket/setup.py @@ -1,16 +1,19 @@ import os -import json import subprocess +from ..schema import generate_schema +from .schema import RacketTesterSettings + def create_environment(_settings, _env_dir, default_env_dir): return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")} def settings(): - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: - return json.load(f) + return generate_schema(RacketTesterSettings) def install(): - subprocess.run(os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), check=True) + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system") + print(f"[AUTOTESTER] Running {path}", flush=True) + subprocess.run(path, check=True) diff --git a/server/autotest_server/testers/schema.py b/server/autotest_server/testers/schema.py new file mode 100644 index 00000000..82d987df --- /dev/null +++ b/server/autotest_server/testers/schema.py @@ -0,0 +1,25 @@ +"""Helper functions for defining tester schemas.""" + +from __future__ import annotations + +from markus_autotesting_core.types import BaseTesterSettings +import msgspec + + +def generate_schema(tester_class: type[BaseTesterSettings]) -> tuple[dict, list]: + """Generate a schema for a given tester class. This handles common post-processing for all tester classes. + + Returns a schema and list of definitions used by the schema. + """ + _, components = msgspec.json.schema_components([tester_class]) + tester_component = components[tester_class.__name__] + tester_name = tester_component["title"].removesuffix("TesterSettings") + + # Modify tester title + tester_component["title"] = tester_name + tester_component["properties"]["tester_type"]["default"] = tester_name.removesuffix("TesterSettings").lower() + + # Remove private schema properties + del tester_component["properties"]["_env"] + + return tester_component, components diff --git a/server/autotest_server/tests/fixtures/specs/custom/simple.json b/server/autotest_server/tests/fixtures/specs/custom/simple.json new file mode 100644 index 00000000..3637d593 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/custom/simple.json @@ -0,0 +1,19 @@ +{ + "tester_type": "custom", + "test_data": [ + { + "script_files": [ + "autotest_01.sh" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Custom Test Group" + } + } + ] +} diff --git a/server/autotest_server/tests/fixtures/specs/haskell/simple.json b/server/autotest_server/tests/fixtures/specs/haskell/simple.json new file mode 100644 index 00000000..fcc61763 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/haskell/simple.json @@ -0,0 +1,22 @@ +{ + "tester_type": "haskell", + "test_data": [ + { + "script_files": [ + "Test.hs" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "test_timeout": 10, + "test_cases": 100, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Haskell Test Group" + } + } + ], + "env_data": {} +} diff --git a/server/autotest_server/tests/fixtures/specs/java/simple.json b/server/autotest_server/tests/fixtures/specs/java/simple.json new file mode 100644 index 00000000..922bc5d9 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/java/simple.json @@ -0,0 +1,33 @@ +{ + "tester_type": "java", + "test_data": [ + { + "script_files": [ + "Test1.java" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Java Test Group 1" + } + }, + { + "script_files": [ + "Test2.java" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Java Test Group 2" + } + } + ] +} \ No newline at end of file diff --git a/server/autotest_server/tests/fixtures/specs/js/simple.json b/server/autotest_server/tests/fixtures/specs/js/simple.json new file mode 100644 index 00000000..298153e3 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/js/simple.json @@ -0,0 +1,19 @@ +{ + "tester_type": "js", + "test_data": [ + { + "script_files": [ + "test.js" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "JS Test Group 1" + } + } + ] +} diff --git a/server/autotest_server/tests/fixtures/specs/jupyter/simple.json b/server/autotest_server/tests/fixtures/specs/jupyter/simple.json new file mode 100644 index 00000000..bc171211 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/jupyter/simple.json @@ -0,0 +1,27 @@ +{ + "tester_type": "jupyter", + "env_data": { + "python_version": "3.13", + "pip_requirements": "matplotlib numpy" + }, + "test_data": [ + { + "script_files": [ + { + "student_file": "submission.ipynb", + "test_file": "test.ipynb", + "test_merge": true + } + ], + "timeout": 30, + "category": [ + "instructor" + ], + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Jupyter Test Group" + } + } + ] +} diff --git a/server/autotest_server/tests/fixtures/specs/py/simple.json b/server/autotest_server/tests/fixtures/specs/py/simple.json new file mode 100644 index 00000000..95603be0 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/py/simple.json @@ -0,0 +1,41 @@ +{ + "tester_type": "py", + "env_data": { + "python_version": "3.13", + "pip_requirements": "hypothesis pytest-timeout" + }, + "test_data": [ + { + "script_files": [ + "test.py" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "tester": "unittest", + "output_verbosity": 2, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Python Test Group 1" + } + }, + { + "script_files": [ + "test2.py" + ], + "category": [ + "instructor" + ], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Python Test Group 2" + } + } + ] +} diff --git a/server/autotest_server/tests/fixtures/specs/r/simple.json b/server/autotest_server/tests/fixtures/specs/r/simple.json new file mode 100644 index 00000000..a538cadb --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/r/simple.json @@ -0,0 +1,35 @@ +{ + "env_data": { + "requirements": "knitr" + }, + "test_data": [ + { + "category": [ + "instructor" + ], + "extra_info": { + "criterion": "criterion", + "name": "R Test Group", + "display_output": "instructors" + }, + "script_files": [ + "test.R" + ], + "timeout": 30 + }, + { + "category": [ + "instructor" + ], + "extra_info": { + "name": "R Test Group for Rmarkdown", + "display_output": "instructors" + }, + "script_files": [ + "test_rmd.R" + ], + "timeout": 30 + } + ], + "tester_type": "r" +} diff --git a/server/autotest_server/tests/fixtures/specs/racket/simple.json b/server/autotest_server/tests/fixtures/specs/racket/simple.json new file mode 100644 index 00000000..611f67c3 --- /dev/null +++ b/server/autotest_server/tests/fixtures/specs/racket/simple.json @@ -0,0 +1,22 @@ +{ + "tester_type": "racket", + "test_data": [ + { + "script_files": [ + { + "test_suite_name": "all-tests", + "script_file": "test.rkt" + } + ], + "category": [ + "instructor" + ], + "timeout": 30, + "extra_info": { + "criterion": "criterion", + "display_output": "instructors", + "name": "Racket Test Group" + } + } + ] +} \ No newline at end of file diff --git a/server/autotest_server/tests/test_timeout.py b/server/autotest_server/tests/test_timeout.py new file mode 100644 index 00000000..6ad5579b --- /dev/null +++ b/server/autotest_server/tests/test_timeout.py @@ -0,0 +1,271 @@ +import signal +import subprocess +import unittest +from unittest.mock import patch, MagicMock, ANY + +import autotest_server + +_UNSET = object() + + +class TestMaxTestTimeout(unittest.TestCase): + """Tests for max_test_timeout configuration enforcement in _run_test_specs.""" + + @staticmethod + def _make_settings(timeout=_UNSET): + test_data = {"category": ["unit"], "extra_info": {"name": "test group"}} + if timeout is not _UNSET: + test_data["timeout"] = timeout + return {"testers": [{"tester_type": "py", "test_data": [test_data]}]} + + def _run_and_get_proc(self, test_settings, max_test_timeout=_UNSET): + """Run _run_test_specs with mocked dependencies, return mock proc.""" + mock_proc = MagicMock() + mock_proc.communicate.return_value = ("{}", "") + + test_config = {} + if max_test_timeout is not _UNSET: + test_config["max_test_timeout"] = max_test_timeout + + with patch("autotest_server._create_test_script_command", return_value="echo test"), patch( + "autotest_server._get_env_vars", return_value={} + ), patch("autotest_server._update_env_vars", side_effect=lambda b, t: {**b, **t}), patch( + "autotest_server.subprocess.Popen", return_value=mock_proc + ), patch( + "autotest_server._get_feedback", return_value=([], []) + ), patch.object( + autotest_server, "config", test_config + ): + autotest_server._run_test_specs( + cmd="echo {}", + test_settings=test_settings, + categories=["unit"], + tests_path="/tmp/test", + test_username="testuser", + test_id=1, + test_env_vars={}, + ) + return mock_proc + + def test_timeout_capped_when_exceeds_max(self): + mock_proc = self._run_and_get_proc(self._make_settings(timeout=600), max_test_timeout=300) + mock_proc.communicate.assert_called_once_with(input=ANY, timeout=300) + + def test_timeout_unchanged_when_below_max(self): + mock_proc = self._run_and_get_proc(self._make_settings(timeout=60), max_test_timeout=300) + mock_proc.communicate.assert_called_once_with(input=ANY, timeout=60) + + def test_timeout_defaults_to_max_when_unset(self): + mock_proc = self._run_and_get_proc(self._make_settings(), max_test_timeout=300) + mock_proc.communicate.assert_called_once_with(input=ANY, timeout=300) + + def test_timeout_unchanged_when_max_not_configured(self): + mock_proc = self._run_and_get_proc(self._make_settings(timeout=600)) + mock_proc.communicate.assert_called_once_with(input=ANY, timeout=600) + + def test_timeout_none_passes_through_when_max_not_configured(self): + mock_proc = self._run_and_get_proc(self._make_settings()) + mock_proc.communicate.assert_called_once_with(input=ANY, timeout=None) + + +class TestTimeoutKillHandler(unittest.TestCase): + """Tests for timeout kill handler after removing start_new_session.""" + + @staticmethod + def _make_settings(): + return { + "testers": [ + { + "tester_type": "py", + "test_data": [{"category": ["unit"], "timeout": 30, "extra_info": {"name": "test group"}}], + } + ] + } + + def _run_with_timeout(self, test_username, current_user): + """Run _run_test_specs where proc.communicate raises TimeoutExpired. + + Returns (results, mock_proc, mock_kill_user). + """ + mock_proc = MagicMock() + mock_proc.communicate.side_effect = [ + subprocess.TimeoutExpired(cmd="test", timeout=30), + ("", "Killed\n"), + ] + + with patch("autotest_server._create_test_script_command", return_value="echo test"), patch( + "autotest_server._get_env_vars", return_value={} + ), patch("autotest_server._update_env_vars", side_effect=lambda b, t: {**b, **t}), patch( + "autotest_server.subprocess.Popen", return_value=mock_proc + ), patch( + "autotest_server._get_feedback", return_value=([], []) + ), patch( + "autotest_server.getpass.getuser", return_value=current_user + ), patch( + "autotest_server._kill_user_processes" + ) as mock_kill_user, patch.object( + autotest_server, "config", {"max_test_timeout": 30} + ): + results = autotest_server._run_test_specs( + cmd="echo {}", + test_settings=self._make_settings(), + categories=["unit"], + tests_path="/tmp/test", + test_username=test_username, + test_id=1, + test_env_vars={}, + ) + return results, mock_proc, mock_kill_user + + def test_kills_user_processes_for_different_user(self): + """Production path: test_username != current user. + On tomlin, workers run as 'autotest' and tests run as 'autotst0'. + """ + results, mock_proc, mock_kill_user = self._run_with_timeout("autotst0", "autotest") + mock_kill_user.assert_called_once_with("autotst0") + mock_proc.kill.assert_not_called() + self.assertIn("did not complete within time limit", results[0]["stderr"]) + + def test_kills_pgid_children_for_same_user(self): + """Dev/local path: kills all processes in our pgid except the worker.""" + worker_pid = 100 + child_pid = 200 + grandchild_pid = 201 + unrelated_pid = 300 + worker_pgid = 50 + + mock_proc = MagicMock() + mock_proc.communicate.side_effect = [ + subprocess.TimeoutExpired(cmd="test", timeout=30), + ("", "Killed\n"), + ] + + def make_entry(name): + e = MagicMock() + e.name = name + return e + + mock_entries = [ + make_entry(str(child_pid)), + make_entry(str(grandchild_pid)), + make_entry(str(unrelated_pid)), + make_entry(str(worker_pid)), + make_entry("self"), + ] + + def mock_getpgid(pid): + if pid in (worker_pid, child_pid, grandchild_pid): + return worker_pgid + return 999 # different group + + with patch("autotest_server._create_test_script_command", return_value="echo test"), patch( + "autotest_server._get_env_vars", return_value={} + ), patch("autotest_server._update_env_vars", side_effect=lambda b, t: {**b, **t}), patch( + "autotest_server.subprocess.Popen", return_value=mock_proc + ), patch( + "autotest_server._get_feedback", return_value=([], []) + ), patch( + "autotest_server.getpass.getuser", return_value="autotest" + ), patch( + "autotest_server._kill_user_processes" + ) as mock_kill_user, patch( + "autotest_server.os.getpid", return_value=worker_pid + ), patch( + "autotest_server.os.getpgid", side_effect=mock_getpgid + ), patch( + "autotest_server.os.scandir", return_value=mock_entries + ), patch( + "autotest_server.os.kill" + ) as mock_kill, patch.object( + autotest_server, "config", {"max_test_timeout": 30} + ): + results = autotest_server._run_test_specs( + cmd="echo {}", + test_settings=self._make_settings(), + categories=["unit"], + tests_path="/tmp/test", + test_username="autotest", + test_id=1, + test_env_vars={}, + ) + + # Should kill child and grandchild, not worker or unrelated + mock_kill.assert_any_call(child_pid, signal.SIGKILL) + mock_kill.assert_any_call(grandchild_pid, signal.SIGKILL) + self.assertEqual(mock_kill.call_count, 2) + mock_proc.wait.assert_called_once() + mock_kill_user.assert_not_called() + self.assertIn("did not complete within time limit", results[0]["stderr"]) + + def test_fallback_to_proc_kill_without_proc_fs(self): + """Non-Linux fallback: /proc missing, falls back to proc.kill().""" + mock_proc = MagicMock() + mock_proc.communicate.side_effect = [ + subprocess.TimeoutExpired(cmd="test", timeout=30), + ("", "Killed\n"), + ] + + with patch("autotest_server._create_test_script_command", return_value="echo test"), patch( + "autotest_server._get_env_vars", return_value={} + ), patch("autotest_server._update_env_vars", side_effect=lambda b, t: {**b, **t}), patch( + "autotest_server.subprocess.Popen", return_value=mock_proc + ), patch( + "autotest_server._get_feedback", return_value=([], []) + ), patch( + "autotest_server.getpass.getuser", return_value="autotest" + ), patch( + "autotest_server._kill_user_processes" + ) as mock_kill_user, patch( + "autotest_server.os.scandir", side_effect=FileNotFoundError + ), patch.object( + autotest_server, "config", {"max_test_timeout": 30} + ): + results = autotest_server._run_test_specs( + cmd="echo {}", + test_settings=self._make_settings(), + categories=["unit"], + tests_path="/tmp/test", + test_username="autotest", + test_id=1, + test_env_vars={}, + ) + + mock_proc.kill.assert_called_once() + mock_proc.wait.assert_called_once() + mock_kill_user.assert_not_called() + self.assertIn("did not complete within time limit", results[0]["stderr"]) + + def test_detects_silent_crash_via_returncode(self): + """SIGKILL/OOM kill: err is empty but proc.returncode is non-zero.""" + mock_proc = MagicMock() + mock_proc.communicate.side_effect = [ + subprocess.TimeoutExpired(cmd="test", timeout=30), + ("", ""), + ] + mock_proc.returncode = -9 # SIGKILL + + with patch("autotest_server._create_test_script_command", return_value="echo test"), patch( + "autotest_server._get_env_vars", return_value={} + ), patch("autotest_server._update_env_vars", side_effect=lambda b, t: {**b, **t}), patch( + "autotest_server.subprocess.Popen", return_value=mock_proc + ), patch( + "autotest_server._get_feedback", return_value=([], []) + ), patch( + "autotest_server.getpass.getuser", return_value="autotest" + ), patch( + "autotest_server._kill_user_processes" + ), patch( + "autotest_server.os.scandir", side_effect=FileNotFoundError + ), patch.object( + autotest_server, "config", {"max_test_timeout": 30} + ): + results = autotest_server._run_test_specs( + cmd="echo {}", + test_settings=self._make_settings(), + categories=["unit"], + tests_path="/tmp/test", + test_username="autotest", + test_id=1, + test_env_vars={}, + ) + self.assertIn("did not complete within time limit", results[0]["stderr"]) diff --git a/server/autotest_server/tests/testers/ai/test_ai_tester.py b/server/autotest_server/tests/testers/ai/test_ai_tester.py index fbd5a84c..a4156298 100644 --- a/server/autotest_server/tests/testers/ai/test_ai_tester.py +++ b/server/autotest_server/tests/testers/ai/test_ai_tester.py @@ -7,12 +7,12 @@ from ....testers.ai.ai_tester import AiTester, AiTest from ....testers.specs import TestSpecs +DEFAULT_REMOTE_URL = "https://polymouth.teach.cs.toronto.edu:443/chat" +FIXTURES_DIR = str(Path(__file__).resolve().parent / "fixtures") + @pytest.fixture(autouse=True, scope="session") -def set_required_env(tmp_path_factory): - root = tmp_path_factory.mktemp("autotest") - logs = root / "logs" - logs.mkdir(exist_ok=True) +def set_required_env(): os.environ.setdefault("WORKSPACE", "./") os.environ.setdefault("WORKER_LOG_DIR", "./") os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0") @@ -20,88 +20,130 @@ def set_required_env(tmp_path_factory): os.makedirs(os.environ["WORKER_LOG_DIR"], exist_ok=True) -def create_ai_tester(): - # test_data is an ARRAY; output must be one of the enum values - parent_dir = str(Path(__file__).resolve().parent) - spec = { +def _make_spec(output="overall_comment", config_overrides=None): + """Build a test spec with sensible defaults. Override only what varies.""" + config = { + "model": "remote", + "prompt": "code_table", + "scope": "code", + "submission": FIXTURES_DIR + "/sample_submission.py", + "submission_type": "python", + "remote_url": DEFAULT_REMOTE_URL, + } + if config_overrides: + config.update(config_overrides) + return { "tester_type": "ai", "env_data": {"ai_feedback_version": "main"}, "test_data": { "category": ["instructor"], - "config": { - "model": "remote", - "prompt": "code_table", - "scope": "code", - "submission": parent_dir + "/fixtures/sample_submission.py", - "submission_type": "python", - }, + "config": config, "extra_info": { "name": "AI FEEDBACK COMMENTS", "display_output": "instructors", "test_group_id": 17, "criterion": None, }, - "output": "overall_comment", + "output": output, "timeout": 30, "test_label": "Test A", }, "_env": {"PYTHON": "/home/docker/.autotesting/scripts/128/ai_1/bin/python3"}, } - raw_spec = json.dumps(spec) - return AiTester(specs=TestSpecs.from_json(raw_spec)) -def test_ai_test_success_runs_properly(): - result = {"title": "Test A", "message": "Looks good", "status": "success"} - test = AiTest(tester=create_ai_tester(), result=result) - output = test.run() - assert '"status": "pass"' in output - assert "Looks good" in output +def _make_tester(**kwargs): + return AiTester(specs=TestSpecs.from_json(json.dumps(_make_spec(**kwargs)))) -def test_ai_test_error_runs_properly(): - result = {"title": "Test A", "message": "Syntax error", "status": "error"} - test = AiTest(tester=create_ai_tester(), result=result) - output = test.run() - assert '"status": "error"' in output - assert "Syntax error" in output - - -def test_call_ai_feedback_success(monkeypatch): - tester = create_ai_tester() +def _mock_subprocess(monkeypatch, *, stdout="OK", stderr=""): mocked = subprocess.CompletedProcess( - args=["python", "-m", "ai_feedback"], returncode=0, stdout="Great job!", stderr="" + args=["python", "-m", "ai_feedback"], returncode=0, stdout=stdout, stderr=stderr ) monkeypatch.setattr(subprocess, "run", lambda *a, **kw: mocked) + + +@pytest.mark.parametrize( + "status,expected_status,message", + [ + ("success", "pass", "Looks good"), + ("error", "error", "Syntax error"), + ], +) +def test_ai_test_result_formatting(status, expected_status, message): + result = {"title": "Test A", "message": message, "status": status} + test = AiTest(tester=_make_tester(), result=result) + output = test.run() + assert f'"status": "{expected_status}"' in output + assert message in output + + +def test_overall_comment_output(monkeypatch): + tester = _make_tester() + _mock_subprocess(monkeypatch, stdout="Great job!") results = tester.call_ai_feedback() - print(results) - assert "Test A" in results assert results["Test A"]["status"] == "success" assert tester.overall_comments == ["Great job!"] assert tester.annotations == [] -def test_call_ai_feedback_error(monkeypatch): - tester = create_ai_tester() +def test_annotations_output(monkeypatch): + annotation_data = {"annotations": [{"line": 1, "text": "Good variable name"}]} + tester = _make_tester(output="annotations") + _mock_subprocess(monkeypatch, stdout=json.dumps(annotation_data)) + results = tester.call_ai_feedback() + assert results["Test A"]["status"] == "success" + assert len(tester.annotations) == 1 + assert tester.annotations[0]["text"] == "Good variable name" + + +def test_message_output(monkeypatch): + tester = _make_tester(output="message") + _mock_subprocess(monkeypatch, stdout="Score: 8/10") + results = tester.call_ai_feedback() + assert results["Test A"]["status"] == "success" + assert results["Test A"]["message"] == "Score: 8/10" + + +def test_subprocess_error(monkeypatch): + tester = _make_tester() def raise_err(*args, **kwargs): - raise subprocess.CalledProcessError(1, args, stderr="Runtime error") + raise subprocess.CalledProcessError(1, "cmd", stderr="Runtime error") monkeypatch.setattr(subprocess, "run", raise_err) results = tester.call_ai_feedback() assert results["Test A"]["status"] == "error" assert "Runtime error" in results["Test A"]["message"] - assert tester.overall_comments == [] - assert tester.annotations == [] -def test_run_prints_test_results(monkeypatch, capsys): - tester = create_ai_tester() - monkeypatch.setattr( - subprocess, - "run", - lambda *a, **kw: subprocess.CompletedProcess(args=a, returncode=0, stdout="Nice work", stderr=""), - ) +def test_rejects_non_remote_model(): + tester = _make_tester(config_overrides={"model": "openai"}) + results = tester.call_ai_feedback() + assert results["Test A"]["status"] == "error" + assert "Unsupported model type" in results["Test A"]["message"] + assert "openai" in results["Test A"]["message"] + + +def test_missing_submission_file(): + tester = _make_tester(config_overrides={"submission": FIXTURES_DIR + "/nonexistent.py"}) + results = tester.call_ai_feedback() + assert results["Test A"]["status"] == "error" + assert "Could not find submission file" in results["Test A"]["message"] + + +def test_opt_out_skips_feedback(tmp_path): + opt_out_file = tmp_path / "opt_out.py" + opt_out_file.write_text("# NO_EXTERNAL_AI_FEEDBACK\ndef foo(): pass\n") + tester = _make_tester(config_overrides={"submission": str(opt_out_file)}) + results = tester.call_ai_feedback() + assert results["Test A"]["status"] == "success" + assert "NO_EXTERNAL_AI_FEEDBACK" in results["Test A"]["message"] + + +def test_run_prints_output(monkeypatch, capsys): + tester = _make_tester() + _mock_subprocess(monkeypatch, stdout="Nice work") monkeypatch.setattr(AiTest, "run", lambda self: '{"status": "success", "message": "Test Passed"}') tester.run() captured = capsys.readouterr() diff --git a/server/autotest_server/tests/testers/js/__init__.py b/server/autotest_server/tests/testers/js/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/tests/testers/js/test_js_tester.py b/server/autotest_server/tests/testers/js/test_js_tester.py new file mode 100644 index 00000000..353a0326 --- /dev/null +++ b/server/autotest_server/tests/testers/js/test_js_tester.py @@ -0,0 +1,142 @@ +import json +import subprocess +from unittest.mock import MagicMock + +from ....testers.specs import TestSpecs +from ....testers.js.js_tester import JsTester, JsTest + + +def make_tester(): + return JsTester(specs=TestSpecs.from_json('{"test_data": {"script_files": ["test.js"]}}')) + + +def make_jest_output(*test_dicts): + """Build a minimal Jest JSON output containing the given assertion results.""" + return json.dumps({"testResults": [{"assertionResults": list(test_dicts)}]}) + + +def test_parse_jest_output_returns_assertion_results(): + """Test that_parse_jest_output parses Jest JSON and returns assertionResults.""" + jest_json = { + "testResults": [ + { + "assertionResults": [ + { + "fullName": "js test 1", + "status": "passed", + "failureMessages": [], + }, + { + "fullName": "js test 2", + "status": "failed", + "failureMessages": ["Expected 1, got 2"], + }, + ] + } + ] + } + tester = make_tester() + results, err = tester._parse_jest_output(json.dumps(jest_json)) + assert err is None + assert len(results) == 2 + assert results[0]["fullName"] == "js test 1" + assert results[0]["status"] == "passed" + assert results[1]["fullName"] == "js test 2" + assert results[1]["status"] == "failed" + assert results[1]["failureMessages"] == ["Expected 1, got 2"] + + +def test_js_test_passed_output_format(): + """JsTest with status passed produces framework JSON with status pass.""" + specs = TestSpecs.from_json('{"test_data": {"script_files": ["test.js"]}, "points": {}}') + tester = JsTester(specs=specs) + result = { + "fullName": "my test", + "status": "passed", + "failureMessages": [], + } + test = JsTest(tester, result) + out = test.run() + data = json.loads(out) + assert data["name"] == "my test" + assert data["status"] == "pass" + assert data["marks_earned"] == data["marks_total"] + assert data["marks_total"] == 1 + + +def test_js_test_failed_output_format(): + """JsTest with status failed produces framework JSON with status fail.""" + specs = TestSpecs.from_json('{"test_data": {"script_files": ["test.js"]}, "points": {}}') + tester = JsTester(specs=specs) + result = { + "fullName": "failing test", + "status": "failed", + "failureMessages": ["AssertionError: expected true"], + } + test = JsTest(tester, result) + out = test.run() + data = json.loads(out) + assert data["name"] == "failing test" + assert data["status"] == "fail" + assert data["marks_earned"] == 0 + assert "expected true" in data["output"] + + +def test_parse_jest_output_invalid_json(): + """_parse_jest_output returns an error when given invalid JSON.""" + tester = make_tester() + results, err = tester._parse_jest_output("not valid json") + assert results is None + assert err is not None + + +def test_pending_tests_are_skipped(monkeypatch, capsys): + """JsTester.run() silently skips tests with status pending.""" + mock_pnpm = MagicMock() + mock_pnpm.returncode = 0 + + jest_output = make_jest_output( + {"fullName": "skipped test", "status": "pending", "failureMessages": []}, + ) + mock_jest = MagicMock() + mock_jest.stdout = jest_output + mock_jest.returncode = 0 + + def fake_run(cmd, **kwargs): + if cmd[:2] == ["pnpm", "install"]: + return mock_pnpm + return mock_jest + + monkeypatch.setattr(subprocess, "run", fake_run) + tester = make_tester() + tester.run() + captured = capsys.readouterr() + assert captured.out == "" + + +def test_run_prints_passed_and_failed_results(monkeypatch, capsys): + """JsTester.run() prints output for each non-pending test.""" + mock_pnpm = MagicMock() + mock_pnpm.returncode = 0 + + jest_output = make_jest_output( + {"fullName": "passing test", "status": "passed", "failureMessages": []}, + {"fullName": "failing test", "status": "failed", "failureMessages": ["oops"]}, + ) + mock_jest = MagicMock() + mock_jest.stdout = jest_output + mock_jest.returncode = 0 + + def fake_run(cmd, **kwargs): + if cmd[:2] == ["pnpm", "install"]: + return mock_pnpm + return mock_jest + + monkeypatch.setattr(subprocess, "run", fake_run) + tester = make_tester() + tester.run() + captured = capsys.readouterr() + lines = [line for line in captured.out.strip().splitlines() if line] + assert len(lines) == 2 + assert json.loads(lines[0])["status"] == "pass" + assert json.loads(lines[1])["status"] == "fail" diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py new file mode 100644 index 00000000..1d25e812 --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_marks_earned.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.mark.parametrize("marks_earned", [0, 1, 2]) +def test_partial_marks(request, marks_earned: int) -> None: + request.node.add_marker(pytest.mark.markus_marks_total(2)) + request.node.add_marker(pytest.mark.markus_marks_earned(marks_earned)) + assert False, f"Should be {marks_earned}/2" diff --git a/server/autotest_server/tests/testers/py/test_py_tester.py b/server/autotest_server/tests/testers/py/test_py_tester.py index 43a53b93..73deab71 100644 --- a/server/autotest_server/tests/testers/py/test_py_tester.py +++ b/server/autotest_server/tests/testers/py/test_py_tester.py @@ -1,14 +1,13 @@ from ....testers.specs import TestSpecs -from ....testers.py.py_tester import PyTester +from ....testers.py.py_tester import PyTester, PyTest +import json import re def test_success(request, monkeypatch) -> None: """Test that when a test succeeds, it is added to the results.""" monkeypatch.chdir(request.fspath.dirname) - tester = PyTester( - specs=TestSpecs.from_json( - """ + tester = PyTester(specs=TestSpecs.from_json(""" { "test_data": { "script_files": ["fixtures/sample_tests_success.py"], @@ -22,9 +21,7 @@ def test_success(request, monkeypatch) -> None: } } } - """ - ) - ) + """)) results = tester.run_python_tests() assert len(results) == 1 assert "fixtures/sample_tests_success.py" in results @@ -41,9 +38,7 @@ def test_success(request, monkeypatch) -> None: def test_skip(request, monkeypatch) -> None: """Test that when a test is skipped, it is omitted from the results.""" monkeypatch.chdir(request.fspath.dirname) - tester = PyTester( - specs=TestSpecs.from_json( - """ + tester = PyTester(specs=TestSpecs.from_json(""" { "test_data": { "script_files": ["fixtures/sample_tests_skip.py"], @@ -57,8 +52,52 @@ def test_skip(request, monkeypatch) -> None: } } } - """ - ) - ) + """)) results = tester.run_python_tests() assert results == {"fixtures/sample_tests_skip.py": []} + + +def test_marks_earned_respected_when_equal_to_total(request, monkeypatch) -> None: + """Test that markus_marks_earned is respected even when earned == total and test fails (TICKET-602).""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester(specs=TestSpecs.from_json(""" + { + "test_data": { + "script_files": ["fixtures/sample_tests_marks_earned.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """)) + results = tester.run_python_tests() + test_results = results["fixtures/sample_tests_marks_earned.py"] + assert len(test_results) == 3 + + # Build PyTest instances and run them to get final marks + outputs = [] + for res in test_results: + test = PyTest(tester, "fixtures/sample_tests_marks_earned.py", res) + outputs.append(json.loads(test.run())) + + assert len(outputs) == 3 + + # Sort by marks_earned so assertions are deterministic + outputs.sort(key=lambda o: o["marks_earned"]) + + # marks_earned=0: should get 0 marks (TICKET-603) + assert outputs[0]["marks_earned"] == 0 + assert outputs[0]["marks_total"] == 2 + + # marks_earned=1: should get 1 mark (partial) + assert outputs[1]["marks_earned"] == 1 + assert outputs[1]["marks_total"] == 2 + + # marks_earned=2: should get 2 marks, not 0 (TICKET-602) + assert outputs[2]["marks_earned"] == 2 + assert outputs[2]["marks_total"] == 2 diff --git a/server/autotest_server/tests/testers/r/test_r_tester.py b/server/autotest_server/tests/testers/r/test_r_tester.py index bec5e022..687a0815 100644 --- a/server/autotest_server/tests/testers/r/test_r_tester.py +++ b/server/autotest_server/tests/testers/r/test_r_tester.py @@ -26,9 +26,7 @@ def test_success_with_context(request, monkeypatch): mock_process.stderr = "" with patch("subprocess.run", return_value=mock_process): - tester = RTester( - specs=TestSpecs.from_json( - """ + tester = RTester(specs=TestSpecs.from_json(""" { "test_data": { "script_files": ["fixtures/sample_tests_success.R"], @@ -40,9 +38,7 @@ def test_success_with_context(request, monkeypatch): } } } - """ - ) - ) + """)) results = tester.run_r_tests() diff --git a/server/autotest_server/tests/testers/test_schema.py b/server/autotest_server/tests/testers/test_schema.py new file mode 100644 index 00000000..9d666497 --- /dev/null +++ b/server/autotest_server/tests/testers/test_schema.py @@ -0,0 +1,70 @@ +import json +import os +import jsonschema +import pytest + + +from autotest_server.testers import get_settings + + +def create_refs(files_list: list[str]): + return { + "files_list": {"type": "string", "enum": files_list}, + "extra_group_data": { + "type": "object", + "properties": { + "name": {"type": "string", "title": "Test Group name", "default": "Test Group"}, + "display_output": { + "type": "string", + "oneOf": [ + {"const": "instructors", "title": "Never display test output to students"}, + { + "const": "instructors_and_student_tests", + "title": "Only display test output to students for student-run tests", + }, + {"const": "instructors_and_students", "title": "Always display test output to students"}, + ], + "default": "instructors", + "title": "Display test output to students?", + }, + }, + "required": ["display_output"], + }, + "test_data_categories": {"type": "string", "enum": ["instructor", "student"]}, + "criterion": { + "type": ["string", "null"], + "title": "Criterion", + "oneOf": [{"const": None, "title": "Not applicable"}, {"const": "criterion", "title": "criterion"}], + "default": None, + }, + } + + +@pytest.mark.parametrize( + "tester,files_list", + [ + ("custom", ["autotest_01.sh"]), + ("haskell", ["Test.hs"]), + ("java", ["Test1.java", "Test2.java"]), + ("js", ["test.js"]), + ("jupyter", ["test.ipynb"]), + ("py", ["test.py", "test2.py"]), + ("r", ["test.R", "test_rmd.R"]), + ("racket", ["test.rkt"]), + ], +) +def test_valid_simple_schema(tester, files_list): + schemas, definitions = get_settings() + schema = schemas[tester] + if "definitions" not in schema: + schema["definitions"] = {} + schema["definitions"].update(create_refs(files_list=files_list)) + schema["$defs"] = definitions + + # Override the installed Python versions + definitions["PythonVersion"]["enum"] = ["3.13"] + + with open(os.path.join(os.path.dirname(__file__), "..", "fixtures", "specs", tester, "simple.json")) as f: + instance = json.load(f) + + jsonschema.validate(instance, schema) diff --git a/server/install.py b/server/install.py index 9d3fb706..4b3e9e86 100644 --- a/server/install.py +++ b/server/install.py @@ -76,13 +76,15 @@ def create_worker_log_dir(): def install_all_testers(): - settings = install_testers() + settings, definitions = install_testers() skeleton_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "autotest_server", "schema_skeleton.json") with open(skeleton_file) as f: skeleton = json.load(f) - skeleton["definitions"]["installed_testers"]["enum"] = list(settings.keys()) - skeleton["definitions"]["tester_schemas"]["oneOf"] = list(settings.values()) - REDIS_CONNECTION.set("autotest:schema", json.dumps(skeleton)) + + skeleton["definitions"]["installed_testers"]["enum"] = list(settings.keys()) + skeleton["definitions"]["tester_schemas"]["oneOf"] = list(settings.values()) + skeleton["$defs"] = definitions + REDIS_CONNECTION.set("autotest:schema", json.dumps(skeleton)) def install(): diff --git a/server/requirements.txt b/server/requirements.txt index c5c98d5d..4ad596a8 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,7 +1,9 @@ -rq==2.6.0 -redis==7.1.0 +rq==2.7.0 +redis==7.4.0 pyyaml==6.0.3 -jsonschema==4.25.1 -requests==2.32.5 +jsonschema==4.26.0 +markus-autotesting-core==0.1.0 +msgspec==0.20.0 +requests==2.33.1 psycopg2-binary==2.9.11 supervisor==4.3.0