From ee20e71431f93a54a8a0febf5401b36ee9203c22 Mon Sep 17 00:00:00 2001 From: Kilian Lieret Date: Thu, 18 Jun 2026 02:33:59 +0000 Subject: [PATCH 1/3] Feat(eval): block build-script internet for submissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A submission's compile.sh runs as root in the build container and could otherwise smuggle install/download steps into the build. Block internet unconditionally during compile.sh via an in-container DNS blackhole (overwrite /etc/resolv.conf with nameserver 0.0.0.0, restore after) — no host privileges, works under docker-in-docker. Test-execution containers are left untouched. Internal-reference: b993951da7a644d49af19073b745c16c513cc316 Internal-reference: fad6005e0b633e103308ef4bc339848ad4bbc569 --- CLAUDE.md | 4 + src/programbench/eval/eval.py | 22 ++-- src/programbench/utils/internet_control.py | 57 ++++++++++ tests/test_internet_control.py | 117 +++++++++++++++++++++ 4 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 src/programbench/utils/internet_control.py create mode 100644 tests/test_internet_control.py diff --git a/CLAUDE.md b/CLAUDE.md index 5aaee0d..f4871c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,10 @@ tests/ The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies. +## Build-time internet isolation + +During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network). + ## Style guide 1. Target python 3.10 or higher diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index b423f3d..9561b7b 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -42,6 +42,7 @@ ) from programbench.container import ContainerEnvironment, remove_image from programbench.exceptions import EmptyTestResultError, EvalStepError, XmlParseError +from programbench.utils.internet_control import block_build_internet_dns, restore_build_internet_dns log = logging.getLogger(__name__) @@ -519,13 +520,20 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) - self._run_step( - "chmod +x ./compile.sh && ./compile.sh", - env=env, - log_buf=log_buf, - step_name="compile", - timeout=900, - ) + # Block internet during compile.sh so a submission can't smuggle + # install/download steps into its build. Test-execution containers + # are never touched (they may legitimately need network). + block_build_internet_dns(env) + try: + self._run_step( + "chmod +x ./compile.sh && ./compile.sh", + env=env, + log_buf=log_buf, + step_name="compile", + timeout=900, + ) + finally: + restore_build_internet_dns(env) self._run_step( f"mv ./executable {self._stashed_executable}", env=env, diff --git a/src/programbench/utils/internet_control.py b/src/programbench/utils/internet_control.py new file mode 100644 index 0000000..2f7bd49 --- /dev/null +++ b/src/programbench/utils/internet_control.py @@ -0,0 +1,57 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Block internet for the eval build script via an in-container DNS blackhole. + +A submission's ``compile.sh`` runs as root inside the build container. Without +isolation it could smuggle ``pip install`` / download steps into the build, +which the eval is meant to forbid. We block from *inside* the container by +overwriting ``/etc/resolv.conf`` with an unroutable nameserver, so hostname +resolution (pip/cargo/npm/go/apt/git-over-https/...) fails. This needs no host +privileges and behaves identically locally and under docker-in-docker. + +Accepted trade-offs for the build threat model: it does not block raw-IP +connections, and a root process inside the container could rewrite resolv.conf +to undo it. Test-execution containers are left untouched (they may legitimately +need network). +""" + +from programbench.container import ContainerEnvironment + +_RESOLV_CONF = "/etc/resolv.conf" +_RESOLV_BACKUP = "/etc/resolv.conf.programbench-build-bak" +_BLACKHOLE_NS = "nameserver 0.0.0.0" + + +def block_build_internet_dns(env: ContainerEnvironment) -> None: + """Blackhole DNS inside the container so the build script can't download. + + Backs up ``/etc/resolv.conf`` and replaces it with an unroutable nameserver. + Restore with :func:`restore_build_internet_dns`. Raises if the rewrite did + not take effect, so callers never run a build believing internet is blocked + when it isn't. + """ + r = env.execute( + f"cp -f {_RESOLV_CONF} {_RESOLV_BACKUP} && " + f"printf '%s\\n' '{_BLACKHOLE_NS}' > {_RESOLV_CONF} && " + f"cat {_RESOLV_CONF}", + timeout=20, + ) + if r["returncode"] != 0 or _BLACKHOLE_NS not in r["output"]: + raise RuntimeError(f"Failed to blackhole DNS for build isolation: {r['output'].strip()}") + + +def restore_build_internet_dns(env: ContainerEnvironment) -> None: + """Restore ``/etc/resolv.conf`` from the backup taken by the block call. + + Best-effort and idempotent: if the backup is missing (block never ran or was + already restored), this is a no-op, so it's safe to call unconditionally from + a ``finally`` block. + """ + env.execute( + f"if [ -f {_RESOLV_BACKUP} ]; then cat {_RESOLV_BACKUP} > {_RESOLV_CONF} && rm -f {_RESOLV_BACKUP}; fi", + timeout=20, + ) diff --git a/tests/test_internet_control.py b/tests/test_internet_control.py new file mode 100644 index 0000000..9d3e9f2 --- /dev/null +++ b/tests/test_internet_control.py @@ -0,0 +1,117 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Tests for the build-time DNS-blackhole internet block.""" + +from unittest import mock + +import pytest + +from programbench.eval.eval import Evaluator +from programbench.utils.internet_control import ( + _BLACKHOLE_NS, + block_build_internet_dns, + restore_build_internet_dns, +) + + +class FakeEnv: + """Records executed commands and replays canned responses.""" + + def __init__(self, responses: list[dict] | None = None): + self.commands: list[str] = [] + self._responses = list(responses or []) + + def execute(self, command: str, *, timeout: int | None = None) -> dict: + self.commands.append(command) + if self._responses: + return self._responses.pop(0) + return {"output": "", "returncode": 0, "exception_info": ""} + + +class TestBlockBuildInternetDns: + def test_block_writes_blackhole_and_backs_up(self): + env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 0, "exception_info": ""}]) + block_build_internet_dns(env) + cmd = env.commands[0] + assert "/etc/resolv.conf.programbench-build-bak" in cmd + assert _BLACKHOLE_NS in cmd + assert "/etc/resolv.conf" in cmd + + def test_block_raises_when_write_not_confirmed(self): + env = FakeEnv([{"output": "something else", "returncode": 0, "exception_info": ""}]) + with pytest.raises(RuntimeError, match="blackhole DNS"): + block_build_internet_dns(env) + + def test_block_raises_on_nonzero_returncode(self): + env = FakeEnv([{"output": f"{_BLACKHOLE_NS}\n", "returncode": 1, "exception_info": ""}]) + with pytest.raises(RuntimeError, match="blackhole DNS"): + block_build_internet_dns(env) + + def test_restore_copies_backup_back(self): + env = FakeEnv() + restore_build_internet_dns(env) + cmd = env.commands[0] + assert "/etc/resolv.conf.programbench-build-bak" in cmd + assert "rm -f" in cmd + + +class TestEvaluatorBuildInternetWiring: + def _make_evaluator(self, tmp_path) -> Evaluator: + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + return ev + + def test_block_restore_wrap_compile(self, tmp_path): + ev = self._make_evaluator(tmp_path) + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + with ( + mock.patch( + "programbench.eval.eval.block_build_internet_dns", + side_effect=lambda e: calls.append("block"), + ), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", + side_effect=lambda e: calls.append("restore"), + ), + mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: calls.append(k["step_name"]) or {"output": "h x", "returncode": 0}, + ), + ): + ev._compile_executable(env, []) + assert calls.index("block") < calls.index("compile") < calls.index("restore") + + def test_restore_runs_even_when_compile_fails(self, tmp_path): + from programbench.exceptions import EvalStepError + + ev = self._make_evaluator(tmp_path) + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + + def fake_run_step(*a, **k): + calls.append(k["step_name"]) + if k["step_name"] == "compile": + raise EvalStepError("compile_failed", "boom") + return {"output": "h x", "returncode": 0} + + with ( + mock.patch("programbench.eval.eval.block_build_internet_dns", side_effect=lambda e: calls.append("block")), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", side_effect=lambda e: calls.append("restore") + ), + mock.patch.object(ev, "_run_step", side_effect=fake_run_step), + pytest.raises(EvalStepError), + ): + ev._compile_executable(env, []) + assert "restore" in calls + assert calls.index("block") < calls.index("restore") From e3bad86fc77024ba413d8d2d7db59455eef15265 Mon Sep 17 00:00:00 2001 From: Kilian Lieret Date: Thu, 18 Jun 2026 12:39:46 -0700 Subject: [PATCH 2/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/programbench/eval/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index 9561b7b..55c9e62 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -523,8 +523,8 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> # Block internet during compile.sh so a submission can't smuggle # install/download steps into its build. Test-execution containers # are never touched (they may legitimately need network). - block_build_internet_dns(env) try: + block_build_internet_dns(env) self._run_step( "chmod +x ./compile.sh && ./compile.sh", env=env, From 27126f2a9b2d9c4b476a267bf6e345becaec8909 Mon Sep 17 00:00:00 2001 From: Kilian Lieret Date: Thu, 18 Jun 2026 12:40:21 -0700 Subject: [PATCH 3/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/programbench/utils/internet_control.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/programbench/utils/internet_control.py b/src/programbench/utils/internet_control.py index 2f7bd49..9237d58 100644 --- a/src/programbench/utils/internet_control.py +++ b/src/programbench/utils/internet_control.py @@ -41,7 +41,8 @@ def block_build_internet_dns(env: ContainerEnvironment) -> None: timeout=20, ) if r["returncode"] != 0 or _BLACKHOLE_NS not in r["output"]: - raise RuntimeError(f"Failed to blackhole DNS for build isolation: {r['output'].strip()}") + detail = (r["output"] or r["exception_info"] or "").strip() + raise RuntimeError(f"Failed to blackhole DNS for build isolation: {detail}") def restore_build_internet_dns(env: ContainerEnvironment) -> None: