Release 2.10.0

.gitignore

@@ -8,6 +8,8 @@ __pycache__
 .eggs
 venv
 venv2
+.venv
 build
 docker-compose.override.yml
 /workspace
+.vscode

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.11.0
+    rev: 26.3.1
     hooks:
     - id: black
   - repo: https://github.com/pycqa/flake8

Changelog.md

@@ -1,6 +1,21 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v2.10.0]
+- Fixed Haskell test results to only include the function name (#687)
+- Improved robustness of tester installation scripts and Docker configuration (#688)
+- Moved tidyverse installation steps from server Dockerfile into R tester requirements.system (#688)
+- Fixed Haskell tester installation using ghcup to install stack system-wide (#688)
+- Updated tester schema generation to use msgspec datatypes (#689)
+- Specify dockerfile frontend version (#691)
+- Add a new Javascript (Jest) tester to support Javascript autotesting (#698)
+- Add remote URL whitelist for AI tester to restrict allowed endpoints (#693)
+- Increased default settings job timeout from 600s to 1200s (#707)
+- Disable pytest cacheprovider to avoid creating .pytest_cache in isolated runs (#709)
+- Fixed Python tester to correctly report marks when `markus_marks_earned` equals total or zero (#716)
+- Fix worker orphan processes and add server-side max test timeout cap (#710)
+- Resolve ghcup isolation installation failure in Haskell autotester (#725)
+
 ## [v2.9.0]
 - Install stack with GHCup (#626)
 - Fixed AI tester to report error when the specified `submission` file is not found (#663)
@@ -39,13 +54,13 @@ All notable changes to this project will be documented here.
 - Update R tester to allow a renv.lock file (#581)
 - Improve display of Python package installation errors when creating environment (#585)
 - Update "setting up test environment" message with http response of status code 503 (#589)
-- Change rlimit resource settings to apply each worker individually (#587) 
+- Change rlimit resource settings to apply each worker individually (#587)
 - Drop support for Python 3.8 (#590)
 - Use Python 3.13 in development (#590)
 - Update Docker configuration to install dependencies in a separate service (#590)
 - Improve error reporting with handled assertion errors (#591)
 - Add custom pytest markers to Python tester to record MarkUs metadata (#592)
-- Stop the autotester from running tests if there are errors in test settings (#593) 
+- Stop the autotester from running tests if there are errors in test settings (#593)
 - Implement Redis backoff strategy (#594)
 
 ## [v2.6.0]
@@ -119,7 +134,7 @@ All notable changes to this project will be documented here.
 - Add ability to clean up test scripts that haven't been used for X days (#379)
 
 ## [v2.1.2]
-- Support dependencies on specific package versions and non-CRAN sources for R tester (#323) 
+- Support dependencies on specific package versions and non-CRAN sources for R tester (#323)
 
 ## [v2.1.1]
 - Remove the requirement for clients to send unique user name (#318)
@@ -140,7 +155,7 @@ All notable changes to this project will be documented here.
 - Add Jupyter tester (#284)
 
 ## [v1.10.3]
-- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) 
+- Fix bug where zip archive was unpacked two levels deep instead of just one (#271)
 - Pass PGHOST and PGINFO environment variables to tests (#272)
 - Update to new version of markus-api that supports uploading binary files (#273)
 - Fix bug where environment variables were not string types (#274)

README.md

@@ -128,6 +128,8 @@ The autotester currently supports testers for the following languages and testin
     - [`tasty-quickcheck`](https://hackage.haskell.org/package/tasty-quickcheck)
 - `java`
     - [JUnit](https://junit.org/junit4/)
+- `js` (JavaScript)
+    - [Jest](https://jestjs.io/)
 - `py` (python3)
     - [Unittest](https://docs.python.org/3/library/unittest.html)
     - [Pytest](https://docs.pytest.org/en/latest/)
@@ -155,6 +157,8 @@ Installing each tester will also install the following additional packages (syst
     - tasty-quickcheck (cabal package)
 - `java`
     - openjdk-8-jdk
+- `js` (JavaScript)
+    - none
 - `py` (python3)
     - none
 - `pyta`
@@ -202,6 +206,10 @@ supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:900
 
 worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs.
 
+max_test_timeout: # maximum number of seconds a single test is allowed to run before being killed.
+                  # When set, any per-test timeout exceeding this value is capped to it, and tests
+                  # with no timeout default to this value. default is: 3600
+
 rlimit_settings: # RLIMIT settings (see details below)
   nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300
     - 300

client/.dockerfiles/Dockerfile

@@ -12,4 +12,4 @@ RUN apt-get update -y && \
 
 WORKDIR /app
 
-CMD /markus_venv/bin/python run.py
+CMD ["/markus_venv/bin/python", "run.py"]

client/.env

@@ -3,4 +3,4 @@ FLASK_HOST=localhost
 FLASK_PORT=5000
 ACCESS_LOG=
 ERROR_LOG=
-SETTINGS_JOB_TIMEOUT=600
+SETTINGS_JOB_TIMEOUT=1200

client/autotest_client/__init__.py

@@ -24,7 +24,7 @@
 
 ERROR_LOG = os.environ.get("ERROR_LOG")
 ACCESS_LOG = os.environ.get("ACCESS_LOG")
-SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 600)
+SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 1200)
 REDIS_URL = os.environ["REDIS_URL"]
 
 REDIS_CONNECTION = redis.Redis.from_url(

client/autotest_client/form_management.py

@@ -135,6 +135,5 @@ def validate_against_schema(test_specs: Dict, schema: Dict, filenames: List[str]
     schema["definitions"]["files_list"]["enum"] = filenames
     # don't validate based on categories
     schema["definitions"]["test_data_categories"].pop("enum")
-    schema["definitions"]["test_data_categories"].pop("enumNames")
     error = _validate_with_defaults(schema, test_specs, best_only=True)
     return str(error) if error else None

client/requirements.txt

@@ -1,6 +1,6 @@
-flask==3.1.2
-python-dotenv==1.2.1
-rq==2.6.0
-redis==7.1.0
-jsonschema==4.25.1
-Werkzeug==3.1.3
+flask==3.1.3
+python-dotenv==1.2.2
+rq==2.7.0
+redis==7.4.0
+jsonschema==4.26.0
+Werkzeug==3.1.7

compose.yaml

@@ -7,7 +7,7 @@ services:
         UBUNTU_VERSION: '24.04'
         LOGIN_USER: 'docker'
         WORKSPACE: '/home/docker/.autotesting'
-    image: markus-autotest-server-dev:1.4.0
+    image: markus-autotest-server-dev:1.5.0
     volumes:
       - ./server:/app:cached
       - venv_server:/home/docker/markus_venv
@@ -29,7 +29,7 @@ services:
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '24.04'
-    image: markus-autotest-client-dev:1.4.0
+    image: markus-autotest-client-dev:1.5.0
     container_name: 'autotest-client'
     volumes:
       - ./client:/app:cached

server/.dockerfiles/Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.7-labs
 ARG UBUNTU_VERSION=24.04
 
 FROM ubuntu:$UBUNTU_VERSION AS base
@@ -8,7 +9,9 @@ RUN userdel -r ubuntu
 ARG LOGIN_USER
 ARG WORKSPACE
 
-RUN apt-get update -y && \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \
     DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install python3.11 \
@@ -21,18 +24,7 @@ RUN apt-get update -y && \
                        postgresql-client \
                        libpq-dev \
                        sudo \
-                       git \
-                       libfontconfig1-dev \
-                       libcurl4-openssl-dev \
-                       libfreetype6-dev \
-                       libpng-dev \
-                       libtiff5-dev \
-                       libjpeg-dev \
-                       libharfbuzz-dev \
-                       libfribidi-dev \
-                       libxml2-dev \
-                       libnuma-dev \
-                       r-base
+                       git
 
 RUN useradd -ms /bin/bash $LOGIN_USER && \
     usermod -aG sudo $LOGIN_USER && \
@@ -43,17 +35,29 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \
     done && \
     chmod a+x /home/${LOGIN_USER}
 
-COPY . /app
+RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \
+    mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
 
-RUN find /app/autotest_server/testers -name requirements.system -exec {} \;
 
-RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site
+# Copy requirements.system files for all testers. These are copied separately from other files
+# to avoid Docker cache invalidation of the subsequent RUN command if any other files are changed
+# in markus-autotesting/server.
+COPY --parents \
+    ./autotest_server/testers/java/requirements.system \
+    ./autotest_server/testers/js/requirements.system \
+    ./autotest_server/testers/haskell/requirements.system \
+    ./autotest_server/testers/r/requirements.system \
+    ./autotest_server/testers/racket/requirements.system \
+    /app
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=cache,target=$WORKSPACE/.stack,sharing=locked \
+    find /app/autotest_server/testers -name requirements.system -exec {} \;
 
-RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \
-    mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
+COPY . /app
 
 WORKDIR /home/${LOGIN_USER}
 
 USER ${LOGIN_USER}
 
-CMD /app/.dockerfiles/cmd-dev.sh
+CMD ["/app/.dockerfiles/cmd-dev.sh"]

server/autotest_server/__init__.py

@@ -4,8 +4,8 @@
 import shutil
 import time
 import json
-import subprocess
 import signal
+import subprocess
 import socket
 import getpass
 import requests
@@ -101,6 +101,31 @@ def _kill_user_processes(test_username: str) -> None:
     subprocess.run(kill_cmd, shell=True)
 
 
+def _kill_pgid_children(proc: subprocess.Popen) -> None:
+    """
+    Kill all processes in our process group except the current (worker) process.
+    Tests may spawn subprocesses (e.g. CSC209) that share our group;
+    proc.kill() alone would leave those running.
+    Falls back to proc.kill() when /proc is unavailable (non-Linux).
+    """
+    worker_pid = os.getpid()
+    worker_pgid = os.getpgid(worker_pid)
+    try:
+        for entry in os.scandir("/proc"):
+            if not entry.name.isdigit():
+                continue
+            pid = int(entry.name)
+            if pid == worker_pid:
+                continue
+            try:
+                if os.getpgid(pid) == worker_pgid:
+                    os.kill(pid, signal.SIGKILL)
+            except (ProcessLookupError, PermissionError, OSError):
+                pass
+    except FileNotFoundError:
+        proc.kill()
+
+
 def _create_test_script_command(tester_type: str) -> str:
     """
     Return string representing a command line command to
@@ -218,13 +243,18 @@ def _run_test_specs(
                 out, err = "", ""
                 timeout_expired = None
                 timeout = test_data.get("timeout")
+                max_timeout = config.get("max_test_timeout")
+                if max_timeout is not None:
+                    if timeout is None:
+                        timeout = max_timeout
+                    else:
+                        timeout = min(timeout, max_timeout)
                 try:
                     env = settings.get("_env", {})
                     env_vars = {**os.environ, **_get_env_vars(test_username), **env}
                     env_vars = _update_env_vars(env_vars, test_env_vars)
                     proc = subprocess.Popen(
                         args,
-                        start_new_session=True,
                         cwd=tests_path,
                         shell=True,
                         stdout=subprocess.PIPE,
@@ -238,14 +268,16 @@ def _run_test_specs(
                         settings_json = json.dumps({**settings, "test_data": test_data})
                         out, err = proc.communicate(input=settings_json, timeout=timeout)
                     except subprocess.TimeoutExpired:
-                        if test_username == getpass.getuser():
-                            pgrp = os.getpgid(proc.pid)
-                            os.killpg(pgrp, signal.SIGKILL)
-                        else:
+                        if test_username != getpass.getuser():
                             _kill_user_processes(test_username)
+                        else:
+                            _kill_pgid_children(proc)
+                            proc.wait()
                         out, err = proc.communicate()
-                        if err == "Killed\n":  # Default message from shell
-                            test_group_name = test_data.get("extra_info", {}).get("name", "").strip()
+                        test_group_name = test_data.get("extra_info", {}).get("name", "").strip()
+                        if err == "Killed\n" or (not err and proc.returncode is not None and proc.returncode != 0):
+                            # err can be "Killed\n" (shell default) or empty (SIGKILL/OOM silent crash).
+                            # Check proc.returncode to reliably detect both cases.
                             if test_group_name:
                                 err = f"Tests for {test_group_name} did not complete within time limit ({timeout}s)\n"
                             else:

server/autotest_server/schema_skeleton.json

@@ -6,8 +6,7 @@
     },
     "test_data_categories": {
       "type": "string",
-      "enum": [],
-      "enumNames": []
+      "enum": []
     },
     "extra_group_data": {},
     "installed_testers": {

server/autotest_server/settings.yml

@@ -2,6 +2,10 @@ workspace: !ENV ${WORKSPACE}
 redis_url: !ENV ${REDIS_URL}
 supervisor_url: !ENV ${SUPERVISOR_URL}
 worker_log_dir: !ENV ${WORKER_LOG_DIR}
+default_remote_url: https://polymouth.teach.cs.toronto.edu:443/chat
+remote_url_whitelist:
+  - https://polymouth.teach.cs.toronto.edu:443/chat
+max_test_timeout: 3600
 workers:
   - user: !ENV ${USER}
     queues:

server/autotest_server/settings_schema.json

@@ -19,9 +19,25 @@
       "type": "string",
       "minLength": 1
     },
+    "default_remote_url": {
+      "type": "string",
+      "minLength": 1
+    },
+    "remote_url_whitelist": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "uniqueItems": true
+    },
     "rlimit_settings": {
       "type": "object"
     },
+    "max_test_timeout": {
+      "type": "integer",
+      "minimum": 1
+    },
     "workers": {
       "type": "array",
       "minItems": 1,