diff --git a/CLAUDE.md b/CLAUDE.md index 5aaee0d..f4871c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,10 @@ tests/ The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies. +## Build-time internet isolation + +During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network). + ## Style guide 1. Target python 3.10 or higher diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index b423f3d..55c9e62 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -42,6 +42,7 @@ ) from programbench.container import ContainerEnvironment, remove_image from programbench.exceptions import EmptyTestResultError, EvalStepError, XmlParseError +from programbench.utils.internet_control import block_build_internet_dns, restore_build_internet_dns log = logging.getLogger(__name__) @@ -519,13 +520,20 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) - self._run_step( - "chmod +x ./compile.sh && ./compile.sh", - env=env, - log_buf=log_buf, - step_name="compile", - timeout=900, - ) + # Block internet during compile.sh so a submission can't smuggle + # install/download steps into its build. Test-execution containers + # are never touched (they may legitimately need network). + try: + block_build_internet_dns(env) + self._run_step( + "chmod +x ./compile.sh && ./compile.sh", + env=env, + log_buf=log_buf, + step_name="compile", + timeout=900, + ) + finally: + restore_build_internet_dns(env) self._run_step( f"mv ./executable {self._stashed_executable}", env=env, diff --git a/src/programbench/utils/internet_control.py b/src/programbench/utils/internet_control.py new file mode 100644 index 0000000..9237d58 --- /dev/null +++ b/src/programbench/utils/internet_control.py @@ -0,0 +1,58 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Block internet for the eval build script via an in-container DNS blackhole. + +A submission's ``compile.sh`` runs as root inside the build container. Without +isolation it could smuggle ``pip install`` / download steps into the build, +which the eval is meant to forbid. We block from *inside* the container by +overwriting ``/etc/resolv.conf`` with an unroutable nameserver, so hostname +resolution (pip/cargo/npm/go/apt/git-over-https/...) fails. This needs no host +privileges and behaves identically locally and under docker-in-docker. + +Accepted trade-offs for the build threat model: it does not block raw-IP +connections, and a root process inside the container could rewrite resolv.conf +to undo it. Test-execution containers are left untouched (they may legitimately +need network). +""" + +from programbench.container import ContainerEnvironment + +_RESOLV_CONF = "/etc/resolv.conf" +_RESOLV_BACKUP = "/etc/resolv.conf.programbench-build-bak" +_BLACKHOLE_NS = "nameserver 0.0.0.0" + + +def block_build_internet_dns(env: ContainerEnvironment) -> None: + """Blackhole DNS inside the container so the build script can't download. + + Backs up ``/etc/resolv.conf`` and replaces it with an unroutable nameserver. + Restore with :func:`restore_build_internet_dns`. Raises if the rewrite did + not take effect, so callers never run a build believing internet is blocked + when it isn't. + """ + r = env.execute( + f"cp -f {_RESOLV_CONF} {_RESOLV_BACKUP} && " + f"printf '%s\\n' '{_BLACKHOLE_NS}' > {_RESOLV_CONF} && " + f"cat {_RESOLV_CONF}", + timeout=20, + ) + if r["returncode"] != 0 or _BLACKHOLE_NS not in r["output"]: + detail = (r["output"] or r["exception_info"] or "").strip() + raise RuntimeError(f"Failed to blackhole DNS for build isolation: {detail}") + + +def restore_build_internet_dns(env: ContainerEnvironment) -> None: + """Restore ``/etc/resolv.conf`` from the backup taken by the block call. + + Best-effort and idempotent: if the backup is missing (block never ran or was + already restored), this is a no-op, so it's safe to call unconditionally from + a ``finally`` block. + """ + env.execute( + f"if [ -f {_RESOLV_BACKUP} ]; then cat {_RESOLV_BACKUP} > {_RESOLV_CONF} && rm -f {_RESOLV_BACKUP}; fi", + timeout=20, + ) diff --git a/tests/test_internet_control.py b/tests/test_internet_control.py new file mode 100644 index 0000000..9d3e9f2 --- /dev/null +++ b/tests/test_internet_control.py @@ -0,0 +1,117 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Tests for the build-time DNS-blackhole internet block.""" + +from unittest import mock + +import pytest + +from programbench.eval.eval import Evaluator +from programbench.utils.internet_control import ( + _BLACKHOLE_NS, + block_build_internet_dns, + restore_build_internet_dns, +) + + +class FakeEnv: + """Records executed commands and replays canned responses.""" + + def __init__(self, responses: list[dict] | None = None): + self.commands: list[str] = [] + self._responses = list(responses or []) + + def execute(self, command: str, *, timeout: int | None = None) -> dict: + self.commands.append(command) + if self._responses: + return self._responses.pop(0) + return {"output": "", "returncode": 0, "exception_info": ""} + + +class TestBlockBuildInternetDns: + def test_block_writes_blackhole_and_backs_up(self): + env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 0, "exception_info": ""}]) + block_build_internet_dns(env) + cmd = env.commands[0] + assert "/etc/resolv.conf.programbench-build-bak" in cmd + assert _BLACKHOLE_NS in cmd + assert "/etc/resolv.conf" in cmd + + def test_block_raises_when_write_not_confirmed(self): + env = FakeEnv([{"output": "something else", "returncode": 0, "exception_info": ""}]) + with pytest.raises(RuntimeError, match="blackhole DNS"): + block_build_internet_dns(env) + + def test_block_raises_on_nonzero_returncode(self): + env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 1, "exception_info": ""}]) + with pytest.raises(RuntimeError, match="blackhole DNS"): + block_build_internet_dns(env) + + def test_restore_copies_backup_back(self): + env = FakeEnv() + restore_build_internet_dns(env) + cmd = env.commands[0] + assert "/etc/resolv.conf.programbench-build-bak" in cmd + assert "rm -f" in cmd + + +class TestEvaluatorBuildInternetWiring: + def _make_evaluator(self, tmp_path) -> Evaluator: + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + return ev + + def test_block_restore_wrap_compile(self, tmp_path): + ev = self._make_evaluator(tmp_path) + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + with ( + mock.patch( + "programbench.eval.eval.block_build_internet_dns", + side_effect=lambda e: calls.append("block"), + ), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", + side_effect=lambda e: calls.append("restore"), + ), + mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: calls.append(k["step_name"]) or {"output": "h x", "returncode": 0}, + ), + ): + ev._compile_executable(env, []) + assert calls.index("block") < calls.index("compile") < calls.index("restore") + + def test_restore_runs_even_when_compile_fails(self, tmp_path): + from programbench.exceptions import EvalStepError + + ev = self._make_evaluator(tmp_path) + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + + def fake_run_step(*a, **k): + calls.append(k["step_name"]) + if k["step_name"] == "compile": + raise EvalStepError("compile_failed", "boom") + return {"output": "h x", "returncode": 0} + + with ( + mock.patch("programbench.eval.eval.block_build_internet_dns", side_effect=lambda e: calls.append("block")), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", side_effect=lambda e: calls.append("restore") + ), + mock.patch.object(ev, "_run_step", side_effect=fake_run_step), + pytest.raises(EvalStepError), + ): + ev._compile_executable(env, []) + assert "restore" in calls + assert calls.index("block") < calls.index("restore")