Fix(eval): block build-script internet for submissions

CLAUDE.md

@@ -24,6 +24,10 @@ tests/
 
 The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies.
 
+## Build-time internet isolation
+
+During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network).
+
 ## Style guide
 
 1. Target python 3.10 or higher

src/programbench/eval/eval.py

@@ -42,6 +42,7 @@
 )
 from programbench.container import ContainerEnvironment, remove_image
 from programbench.exceptions import EmptyTestResultError, EvalStepError, XmlParseError
+from programbench.utils.internet_control import block_build_internet_dns, restore_build_internet_dns
 
 log = logging.getLogger(__name__)
 
@@ -519,13 +520,20 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) ->
             log_buf=log_buf,
             step_name="seed_git",
         )
-        self._run_step(
-            "chmod +x ./compile.sh && ./compile.sh",
-            env=env,
-            log_buf=log_buf,
-            step_name="compile",
-            timeout=900,
-        )
+        # Block internet during compile.sh so a submission can't smuggle
+        # install/download steps into its build. Test-execution containers
+        # are never touched (they may legitimately need network).
+        try:
+            block_build_internet_dns(env)
+            self._run_step(
+                "chmod +x ./compile.sh && ./compile.sh",
+                env=env,
+                log_buf=log_buf,
+                step_name="compile",
+                timeout=900,
+            )
+        finally:
+            restore_build_internet_dns(env)
         self._run_step(
             f"mv ./executable {self._stashed_executable}",
             env=env,

src/programbench/utils/internet_control.py

@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Block internet for the eval build script via an in-container DNS blackhole.
+
+A submission's ``compile.sh`` runs as root inside the build container. Without
+isolation it could smuggle ``pip install`` / download steps into the build,
+which the eval is meant to forbid. We block from *inside* the container by
+overwriting ``/etc/resolv.conf`` with an unroutable nameserver, so hostname
+resolution (pip/cargo/npm/go/apt/git-over-https/...) fails. This needs no host
+privileges and behaves identically locally and under docker-in-docker.
+
+Accepted trade-offs for the build threat model: it does not block raw-IP
+connections, and a root process inside the container could rewrite resolv.conf
+to undo it. Test-execution containers are left untouched (they may legitimately
+need network).
+"""
+
+from programbench.container import ContainerEnvironment
+
+_RESOLV_CONF = "/etc/resolv.conf"
+_RESOLV_BACKUP = "/etc/resolv.conf.programbench-build-bak"
+_BLACKHOLE_NS = "nameserver 0.0.0.0"
+
+
+def block_build_internet_dns(env: ContainerEnvironment) -> None:
+    """Blackhole DNS inside the container so the build script can't download.
+
+    Backs up ``/etc/resolv.conf`` and replaces it with an unroutable nameserver.
+    Restore with :func:`restore_build_internet_dns`. Raises if the rewrite did
+    not take effect, so callers never run a build believing internet is blocked
+    when it isn't.
+    """
+    r = env.execute(
+        f"cp -f {_RESOLV_CONF} {_RESOLV_BACKUP} && "
+        f"printf '%s\\n' '{_BLACKHOLE_NS}' > {_RESOLV_CONF} && "
+        f"cat {_RESOLV_CONF}",
+        timeout=20,
+    )
+    if r["returncode"] != 0 or _BLACKHOLE_NS not in r["output"]:
+        detail = (r["output"] or r["exception_info"] or "").strip()
+        raise RuntimeError(f"Failed to blackhole DNS for build isolation: {detail}")
+
+
+def restore_build_internet_dns(env: ContainerEnvironment) -> None:
+    """Restore ``/etc/resolv.conf`` from the backup taken by the block call.
+
+    Best-effort and idempotent: if the backup is missing (block never ran or was
+    already restored), this is a no-op, so it's safe to call unconditionally from
+    a ``finally`` block.
+    """
+    env.execute(
+        f"if [ -f {_RESOLV_BACKUP} ]; then cat {_RESOLV_BACKUP} > {_RESOLV_CONF} && rm -f {_RESOLV_BACKUP}; fi",
+        timeout=20,
+    )

tests/test_internet_control.py

@@ -0,0 +1,117 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tests for the build-time DNS-blackhole internet block."""
+
+from unittest import mock
+
+import pytest
+
+from programbench.eval.eval import Evaluator
+from programbench.utils.internet_control import (
+    _BLACKHOLE_NS,
+    block_build_internet_dns,
+    restore_build_internet_dns,
+)
+
+
+class FakeEnv:
+    """Records executed commands and replays canned responses."""
+
+    def __init__(self, responses: list[dict] | None = None):
+        self.commands: list[str] = []
+        self._responses = list(responses or [])
+
+    def execute(self, command: str, *, timeout: int | None = None) -> dict:
+        self.commands.append(command)
+        if self._responses:
+            return self._responses.pop(0)
+        return {"output": "", "returncode": 0, "exception_info": ""}
+
+
+class TestBlockBuildInternetDns:
+    def test_block_writes_blackhole_and_backs_up(self):
+        env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 0, "exception_info": ""}])
+        block_build_internet_dns(env)
+        cmd = env.commands[0]
+        assert "/etc/resolv.conf.programbench-build-bak" in cmd
+        assert _BLACKHOLE_NS in cmd
+        assert "/etc/resolv.conf" in cmd
+
+    def test_block_raises_when_write_not_confirmed(self):
+        env = FakeEnv([{"output": "something else", "returncode": 0, "exception_info": ""}])
+        with pytest.raises(RuntimeError, match="blackhole DNS"):
+            block_build_internet_dns(env)
+
+    def test_block_raises_on_nonzero_returncode(self):
+        env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 1, "exception_info": ""}])
+        with pytest.raises(RuntimeError, match="blackhole DNS"):
+            block_build_internet_dns(env)
+
+    def test_restore_copies_backup_back(self):
+        env = FakeEnv()
+        restore_build_internet_dns(env)
+        cmd = env.commands[0]
+        assert "/etc/resolv.conf.programbench-build-bak" in cmd
+        assert "rm -f" in cmd
+
+
+class TestEvaluatorBuildInternetWiring:
+    def _make_evaluator(self, tmp_path) -> Evaluator:
+        archive = tmp_path / "submission.tar.gz"
+        archive.write_bytes(b"")
+        ev = Evaluator(tests_branches=[], submission_archive=archive)
+        ev._remove_hashed_files = lambda env, log_buf: None  # type: ignore[method-assign]
+        return ev
+
+    def test_block_restore_wrap_compile(self, tmp_path):
+        ev = self._make_evaluator(tmp_path)
+        env = FakeEnv()
+        env.copy_in_tar = lambda *a, **k: None  # type: ignore[attr-defined]
+        calls: list[str] = []
+        with (
+            mock.patch(
+                "programbench.eval.eval.block_build_internet_dns",
+                side_effect=lambda e: calls.append("block"),
+            ),
+            mock.patch(
+                "programbench.eval.eval.restore_build_internet_dns",
+                side_effect=lambda e: calls.append("restore"),
+            ),
+            mock.patch.object(
+                ev,
+                "_run_step",
+                side_effect=lambda *a, **k: calls.append(k["step_name"]) or {"output": "h x", "returncode": 0},
+            ),
+        ):
+            ev._compile_executable(env, [])
+        assert calls.index("block") < calls.index("compile") < calls.index("restore")
+
+    def test_restore_runs_even_when_compile_fails(self, tmp_path):
+        from programbench.exceptions import EvalStepError
+
+        ev = self._make_evaluator(tmp_path)
+        env = FakeEnv()
+        env.copy_in_tar = lambda *a, **k: None  # type: ignore[attr-defined]
+        calls: list[str] = []
+
+        def fake_run_step(*a, **k):
+            calls.append(k["step_name"])
+            if k["step_name"] == "compile":
+                raise EvalStepError("compile_failed", "boom")
+            return {"output": "h x", "returncode": 0}
+
+        with (
+            mock.patch("programbench.eval.eval.block_build_internet_dns", side_effect=lambda e: calls.append("block")),
+            mock.patch(
+                "programbench.eval.eval.restore_build_internet_dns", side_effect=lambda e: calls.append("restore")
+            ),
+            mock.patch.object(ev, "_run_step", side_effect=fake_run_step),
+            pytest.raises(EvalStepError),
+        ):
+            ev._compile_executable(env, [])
+        assert "restore" in calls
+        assert calls.index("block") < calls.index("restore")