tests

Author: xzrderek

eval_protocol/cli.py

@@ -447,6 +447,16 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: if multiple tests exist and no --entry, fails with guidance",
     )
+    local_test_parser.add_argument(
+        "--docker-build-extra",
+        default="",
+        help="Extra flags to pass to 'docker build' (quoted string, e.g. \"--no-cache --pull --progress=plain\")",
+    )
+    local_test_parser.add_argument(
+        "--docker-run-extra",
+        default="",
+        help="Extra flags to pass to 'docker run' (quoted string, e.g. \"--env-file .env --memory=8g\")",
+    )
 
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.

eval_protocol/cli_commands/local_test.py

@@ -2,6 +2,7 @@
 import os
 import subprocess
 import sys
+import shlex
 from typing import List
 
 from .upload import _discover_tests, _prompt_select
@@ -24,24 +25,25 @@ def _run_pytest_host(pytest_target: str) -> int:
     return proc.returncode
 
 
-def _build_docker_image(dockerfile_path: str, image_tag: str) -> bool:
+def _build_docker_image(dockerfile_path: str, image_tag: str, build_extras: List[str] | None = None) -> bool:
     context_dir = os.path.dirname(dockerfile_path)
     print(f"Building Docker image '{image_tag}' from {dockerfile_path} ...")
     try:
-        proc = subprocess.run(
-            ["docker", "build", "-t", image_tag, "-f", dockerfile_path, context_dir],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-        )
+        base_cmd = ["docker", "build"]
+        if build_extras:
+            base_cmd += build_extras
+        base_cmd += ["-t", image_tag, "-f", dockerfile_path, context_dir]
+        proc = subprocess.run(base_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
         print(proc.stdout)
         return proc.returncode == 0
     except FileNotFoundError:
         print("Error: docker not found in PATH. Install Docker or use --ignore-docker.")
         return False
 
 
-def _run_pytest_in_docker(project_root: str, image_tag: str, pytest_target: str) -> int:
+def _run_pytest_in_docker(
+    project_root: str, image_tag: str, pytest_target: str, run_extras: List[str] | None = None
+) -> int:
     workdir = "/workspace"
     # Host HOME logs directory to map into container
     host_home = os.path.expanduser("~")
@@ -73,6 +75,8 @@ def _run_pytest_in_docker(project_root: str, image_tag: str, pytest_target: str)
         cmd += ["--user", f"{uid}:{gid}"]
     except Exception:
         pass
+    if run_extras:
+        cmd += run_extras
     cmd += [image_tag, "pytest", pytest_target, "-vs"]
     print("Running in Docker:", " ".join(cmd))
     try:
@@ -126,6 +130,10 @@ def local_test_command(args: argparse.Namespace) -> int:
         pytest_target = rel
 
     ignore_docker = bool(getattr(args, "ignore_docker", False))
+    build_extras_str = getattr(args, "docker_build_extra", "") or ""
+    run_extras_str = getattr(args, "docker_run_extra", "") or ""
+    build_extras = shlex.split(build_extras_str) if build_extras_str else []
+    run_extras = shlex.split(run_extras_str) if run_extras_str else []
     if ignore_docker:
         if not pytest_target:
             print("Error: Failed to resolve a pytest target to run.")
@@ -146,14 +154,14 @@ def local_test_command(args: argparse.Namespace) -> int:
         except Exception:
             pass
         image_tag = "ep-evaluator:local"
-        ok = _build_docker_image(dockerfiles[0], image_tag)
+        ok = _build_docker_image(dockerfiles[0], image_tag, build_extras=build_extras)
         if not ok:
             print("Docker build failed. See logs above.")
             return 1
         if not pytest_target:
             print("Error: Failed to resolve a pytest target to run.")
             return 1
-        return _run_pytest_in_docker(project_root, image_tag, pytest_target)
+        return _run_pytest_in_docker(project_root, image_tag, pytest_target, run_extras=run_extras)
 
     # No Dockerfile: run on host
     if not pytest_target:

tests/test_cli_local_test.py

@@ -95,11 +95,11 @@ def test_local_test_builds_and_runs_in_docker(tmp_path, monkeypatch):
     from eval_protocol.cli_commands import local_test as lt
 
     monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
-    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, platform=None: True)
+    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, build_extras=None: True)
 
     captured = {"target": "", "image": ""}
 
-    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, platform=None) -> int:
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
         captured["target"] = pytest_target
         captured["image"] = image_tag
         return 0
@@ -143,3 +143,82 @@ def _fake_host(target: str) -> int:
     rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
     assert rc == 0
     assert called["host"] is True
+
+
+def test_local_test_passes_docker_build_extra(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    test_file = project / "metric" / "test_build_extra.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
+
+    from eval_protocol.cli_commands import local_test as lt
+
+    monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
+
+    captured = {"extras": None}
+
+    def _fake_build(dockerfile, tag, build_extras=None):
+        captured["extras"] = build_extras
+        return True
+
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
+        return 0
+
+    monkeypatch.setattr(lt, "_build_docker_image", _fake_build)
+    monkeypatch.setattr(lt, "_run_pytest_in_docker", _fake_run_docker)
+
+    # Extras string with multiple flags and equals-arg
+    args = SimpleNamespace(
+        entry=str(test_file),
+        ignore_docker=False,
+        yes=True,
+        docker_build_extra="--no-cache --pull --progress=plain --build-arg KEY=VAL",
+        docker_run_extra="",
+    )
+    rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
+    assert rc == 0
+    # Expect split list preserving tokens order
+    assert captured["extras"] == ["--no-cache", "--pull", "--progress=plain", "--build-arg", "KEY=VAL"]
+
+
+def test_local_test_passes_docker_run_extra(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    test_file = project / "metric" / "test_run_extra.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
+
+    from eval_protocol.cli_commands import local_test as lt
+
+    monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
+    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, build_extras=None: True)
+
+    captured = {"extras": None}
+
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
+        captured["extras"] = run_extras
+        return 0
+
+    monkeypatch.setattr(lt, "_run_pytest_in_docker", _fake_run_docker)
+
+    args = SimpleNamespace(
+        entry=str(test_file),
+        ignore_docker=False,
+        yes=True,
+        docker_build_extra="",
+        docker_run_extra="--env-file .env --memory=8g --cpus=2 --add-host=host.docker.internal:host-gateway",
+    )
+    rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
+    assert rc == 0
+    assert captured["extras"] == [
+        "--env-file",
+        ".env",
+        "--memory=8g",
+        "--cpus=2",
+        "--add-host=host.docker.internal:host-gateway",
+    ]