^B Use snapshot-cloudflare.debian.org CDN mirror (origin fallback) to end mirror-halting CI

docs/injection-policy.md

@@ -139,7 +139,8 @@ policy artifact for this control. On the `colima` target the launcher enforces i
 as a fail-closed, dual-stack firewall: `scripts/workcell` computes
 `ALLOW_ENDPOINTS` (the union of reviewed sources — provider/auth-recovery/broker
 endpoints, credential-derived endpoints, `[network].allow_endpoints`, profile
-`EXTRA_ENDPOINTS`, and `snapshot.debian.org:443`), de-dupes it, subtracts
+`EXTRA_ENDPOINTS`, and the Debian snapshot mirrors
+`snapshot-cloudflare.debian.org:443` and `snapshot.debian.org:443`), de-dupes it, subtracts
 `[network].deny_endpoints`, and hands it to `scripts/colima-egress-allowlist.sh`,
 which programs `iptables`/`ip6tables` `DOCKER-USER` rules that ACCEPT each allowed
 `host:port` (IPv4 and IPv6) and `DROP` the rest. If `ip6tables` is unavailable the
@@ -159,6 +160,10 @@ deny_endpoints  = ["chatgpt.com:443"]                 # remove from the allowlis
   the reviewed provider/credential endpoints).
 - `deny_endpoints` are subtracted by endpoint entry after every allow source, so
   a denied `host:port` is removed even when a provider needs it (deny wins).
+  Because the match is exact, blocking the automatic Debian snapshot egress
+  requires denying **both** mirror endpoints —
+  `snapshot-cloudflare.debian.org:443` and `snapshot.debian.org:443` — since
+  denying only one still leaves the other reachable.
 - Each endpoint must be `host:port` or `[ipv6]:port` (port 1-65535, host
   `^[A-Za-z0-9.-]+$`, no leading dot or `..`, IP-shaped hosts must be real IPs),
   validated with the same grammar `scripts/colima-egress-allowlist.sh` applies.

runtime/container/Dockerfile

@@ -30,11 +30,12 @@ ENV npm_config_update_notifier=false
 
 # hadolint ignore=DL3008
 RUN fetch_snapshot_bootstrap_package() { \
-      local url="$1"; \
+      local path="$1"; \
       local output="$2"; \
-      local attempt=""; \
+      local attempt="" host=""; \
       for attempt in 1 2 3 4 5 6 7 8; do \
-        if node --dns-result-order=ipv4first -e 'const fs = require("fs"); (async () => { const [url, output] = process.argv.slice(1); const response = await fetch(url); if (!response.ok) { throw new Error("unexpected status " + response.status + " for " + url); } const file = fs.createWriteStream(output); for await (const chunk of response.body) { file.write(chunk); } file.end(); await new Promise((resolve, reject) => { file.on("finish", resolve); file.on("error", reject); }); })().catch((error) => { console.error(error.stack || error); process.exit(1); });' "${url}" "${output}"; then \
+        if [[ $((attempt % 2)) -eq 1 ]]; then host="snapshot-cloudflare.debian.org"; else host="snapshot.debian.org"; fi; \
+        if node --dns-result-order=ipv4first -e 'const fs = require("fs"); (async () => { const [url, output] = process.argv.slice(1); const response = await fetch(url); if (!response.ok) { throw new Error("unexpected status " + response.status + " for " + url); } const file = fs.createWriteStream(output); for await (const chunk of response.body) { file.write(chunk); } file.end(); await new Promise((resolve, reject) => { file.on("finish", resolve); file.on("error", reject); }); })().catch((error) => { console.error(error.stack || error); process.exit(1); });' "https://${host}/${path}" "${output}"; then \
           return 0; \
         fi; \
         rm -f "${output}"; \
@@ -48,19 +49,19 @@ RUN fetch_snapshot_bootstrap_package() { \
        local openssl_url="" openssl_sha256="" ca_url="" ca_sha256=""; \
        case "$(dpkg --print-architecture)" in \
          amd64) \
-           openssl_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_amd64.deb"; \
+           openssl_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_amd64.deb"; \
            openssl_sha256="233d6d1f6ff00509cba554365e7cfb3be36db1dea57291eeb248f2c04d62719d"; \
            ;; \
          arm64) \
-           openssl_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_arm64.deb"; \
+           openssl_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_arm64.deb"; \
            openssl_sha256="92dfcdc213cd7a5e6aa34a1ac9660a59e56838d12876f4b34599f8fea39cb432"; \
            ;; \
          *) \
            echo "Unsupported dpkg architecture for snapshot TLS bootstrap: $(dpkg --print-architecture)" >&2; \
            exit 1; \
            ;; \
        esac; \
-       ca_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/c/ca-certificates/ca-certificates_20250419_all.deb"; \
+       ca_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/c/ca-certificates/ca-certificates_20250419_all.deb"; \
        ca_sha256="ef590f89563aa4b46c8260d49d1cea0fc1b181d19e8df3782694706adf05c184"; \
        fetch_snapshot_bootstrap_package "${openssl_url}" /tmp/workcell-bootstrap-openssl.deb \
        && echo "${openssl_sha256}  /tmp/workcell-bootstrap-openssl.deb" | sha256sum -c - \
@@ -71,10 +72,13 @@ RUN fetch_snapshot_bootstrap_package() { \
        rm -f /tmp/workcell-bootstrap-openssl.deb /tmp/workcell-bootstrap-ca-certificates.deb; \
      } \
   && bootstrap_snapshot_tls \
-  && rm -f /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/%s trixie main\n' "${DEBIAN_SNAPSHOT}" > /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/%s trixie-updates main\n' "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/%s trixie-security main\n' "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list \
+  && rm -f /etc/apt/sources.list.d/debian.sources \
+  && set_snapshot_sources() { \
+       local host="$1"; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian/%s trixie main\n' "${host}" "${DEBIAN_SNAPSHOT}" > /etc/apt/sources.list; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian/%s trixie-updates main\n' "${host}" "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian-security/%s trixie-security main\n' "${host}" "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list; \
+     } \
   && printf 'Acquire::Check-Valid-Until "false";\nAcquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' > /etc/apt/apt.conf.d/99workcell-snapshot \
   && apt_indexes_have_runtime_packages() { \
        apt-cache show --no-all-versions \
@@ -116,6 +120,7 @@ RUN fetch_snapshot_bootstrap_package() { \
        && return 0; \
      } \
   && for attempt in 1 2 3 4 5; do \
+       if [[ $((attempt % 2)) -eq 1 ]]; then set_snapshot_sources snapshot-cloudflare.debian.org; else set_snapshot_sources snapshot.debian.org; fi; \
        if DEBIAN_FRONTEND=noninteractive apt-get update \
          && apt_indexes_have_runtime_packages \
          && install_runtime_packages; then \

runtime/container/control-plane-manifest.json

@@ -2,7 +2,7 @@
   "host_artifacts": [
     {
       "repo_path": "scripts/workcell",
-      "sha256": "a26c3cb56a7b82855a6cd5a5ec197814854a8e2bebc101a13bf6b02a1d7fa22b"
+      "sha256": "4895f0722e5bd532542fda4472aef36b5c809c0816017879ea3f8a047c7c537b"
     },
     {
       "repo_path": "scripts/check-publish-commit-signatures.sh",

scripts/verify-invariants.sh

@@ -3056,6 +3056,11 @@ if ! rg -q 'snapshot\.debian\.org:443' "${ROOT_DIR}/scripts/workcell"; then
   exit 1
 fi
 
+if ! rg -q 'snapshot-cloudflare\.debian\.org:443' "${ROOT_DIR}/scripts/workcell"; then
+  echo "Expected scripts/workcell bootstrap endpoints to allow the snapshot-cloudflare.debian.org CDN mirror" >&2
+  exit 1
+fi
+
 if rg -q 'static\.rust-lang\.org:443' "${ROOT_DIR}/scripts/workcell"; then
   echo "Expected scripts/workcell bootstrap endpoints to avoid unused static.rust-lang.org egress" >&2
   exit 1

scripts/workcell

@@ -7844,7 +7844,7 @@ prepare_workspace_control_plane_shadow() {
 }
 
 bootstrap_endpoints() {
-  echo "auth.docker.io:443 docker.io:443 index.docker.io:443 registry-1.docker.io:443 production.cloudflare.docker.com:443 production.cloudfront.docker.com:443 docker-images-prod.6aa30f8b08e16409b46e0173d6de2f56.r2.cloudflarestorage.com:443 github.com:443 objects.githubusercontent.com:443 release-assets.githubusercontent.com:443 registry.npmjs.org:443 storage.googleapis.com:443 snapshot.debian.org:443"
+  echo "auth.docker.io:443 docker.io:443 index.docker.io:443 registry-1.docker.io:443 production.cloudflare.docker.com:443 production.cloudfront.docker.com:443 docker-images-prod.6aa30f8b08e16409b46e0173d6de2f56.r2.cloudflarestorage.com:443 github.com:443 objects.githubusercontent.com:443 release-assets.githubusercontent.com:443 registry.npmjs.org:443 storage.googleapis.com:443 snapshot-cloudflare.debian.org:443 snapshot.debian.org:443"
 }
 
 AGENT=""
@@ -8582,7 +8582,7 @@ fi
 
 ALLOW_ENDPOINTS="$(dedupe_endpoint_list "$(provider_endpoints "${AGENT}") $(target_broker_endpoints "${TARGET_BACKEND}") $(credential_extra_endpoints) $(provider_auth_recovery_extra_endpoints) ${INJECTION_EXTRA_ENDPOINTS} ${EXTRA_ENDPOINTS:-}")"
 if ! target_backend_is_remote_vm_preview "${TARGET_BACKEND}" && [[ "${CONTAINER_MUTABILITY}" == "ephemeral" ]]; then
-  ALLOW_ENDPOINTS="$(dedupe_endpoint_list "${ALLOW_ENDPOINTS} snapshot.debian.org:443")"
+  ALLOW_ENDPOINTS="$(dedupe_endpoint_list "${ALLOW_ENDPOINTS} snapshot-cloudflare.debian.org:443 snapshot.debian.org:443")"
 fi
 # A1: subtract deny_endpoints last so deny wins over every allow source; this
 # only removes endpoints and never changes NETWORK_POLICY.

tools/validator/Dockerfile

@@ -39,11 +39,12 @@ ENV RUSTUP_HOME=/usr/local/rustup
 
 # hadolint ignore=DL3008
 RUN fetch_snapshot_bootstrap_package() { \
-      local url="$1"; \
+      local path="$1"; \
       local output="$2"; \
-      local attempt=""; \
+      local attempt="" host=""; \
       for attempt in 1 2 3 4 5 6 7 8; do \
-        if node --dns-result-order=ipv4first -e 'const fs = require("fs"); (async () => { const [url, output] = process.argv.slice(1); const response = await fetch(url); if (!response.ok) { throw new Error("unexpected status " + response.status + " for " + url); } const file = fs.createWriteStream(output); for await (const chunk of response.body) { file.write(chunk); } file.end(); await new Promise((resolve, reject) => { file.on("finish", resolve); file.on("error", reject); }); })().catch((error) => { console.error(error.stack || error); process.exit(1); });' "${url}" "${output}"; then \
+        if [[ $((attempt % 2)) -eq 1 ]]; then host="snapshot-cloudflare.debian.org"; else host="snapshot.debian.org"; fi; \
+        if node --dns-result-order=ipv4first -e 'const fs = require("fs"); (async () => { const [url, output] = process.argv.slice(1); const response = await fetch(url); if (!response.ok) { throw new Error("unexpected status " + response.status + " for " + url); } const file = fs.createWriteStream(output); for await (const chunk of response.body) { file.write(chunk); } file.end(); await new Promise((resolve, reject) => { file.on("finish", resolve); file.on("error", reject); }); })().catch((error) => { console.error(error.stack || error); process.exit(1); });' "https://${host}/${path}" "${output}"; then \
           return 0; \
         fi; \
         rm -f "${output}"; \
@@ -57,19 +58,19 @@ RUN fetch_snapshot_bootstrap_package() { \
        local openssl_url="" openssl_sha256="" ca_url="" ca_sha256=""; \
        case "$(dpkg --print-architecture)" in \
          amd64) \
-           openssl_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_amd64.deb"; \
+           openssl_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_amd64.deb"; \
            openssl_sha256="233d6d1f6ff00509cba554365e7cfb3be36db1dea57291eeb248f2c04d62719d"; \
            ;; \
          arm64) \
-           openssl_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_arm64.deb"; \
+           openssl_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/o/openssl/openssl_3.5.5-1~deb13u1_arm64.deb"; \
            openssl_sha256="92dfcdc213cd7a5e6aa34a1ac9660a59e56838d12876f4b34599f8fea39cb432"; \
            ;; \
          *) \
            echo "Unsupported dpkg architecture for snapshot TLS bootstrap: $(dpkg --print-architecture)" >&2; \
            exit 1; \
            ;; \
        esac; \
-       ca_url="https://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/pool/main/c/ca-certificates/ca-certificates_20250419_all.deb"; \
+       ca_url="archive/debian/${DEBIAN_SNAPSHOT}/pool/main/c/ca-certificates/ca-certificates_20250419_all.deb"; \
        ca_sha256="ef590f89563aa4b46c8260d49d1cea0fc1b181d19e8df3782694706adf05c184"; \
        fetch_snapshot_bootstrap_package "${openssl_url}" /tmp/workcell-bootstrap-openssl.deb \
        && echo "${openssl_sha256}  /tmp/workcell-bootstrap-openssl.deb" | sha256sum -c - \
@@ -80,10 +81,13 @@ RUN fetch_snapshot_bootstrap_package() { \
        rm -f /tmp/workcell-bootstrap-openssl.deb /tmp/workcell-bootstrap-ca-certificates.deb; \
      } \
   && bootstrap_snapshot_tls \
-  && rm -f /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/%s trixie main\n' "${DEBIAN_SNAPSHOT}" > /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/%s trixie-updates main\n' "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list \
-  && printf 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/%s trixie-security main\n' "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list \
+  && rm -f /etc/apt/sources.list.d/debian.sources \
+  && set_snapshot_sources() { \
+       local host="$1"; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian/%s trixie main\n' "${host}" "${DEBIAN_SNAPSHOT}" > /etc/apt/sources.list; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian/%s trixie-updates main\n' "${host}" "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list; \
+       printf 'deb [check-valid-until=no] https://%s/archive/debian-security/%s trixie-security main\n' "${host}" "${DEBIAN_SNAPSHOT}" >> /etc/apt/sources.list; \
+     } \
   && printf 'Acquire::Check-Valid-Until "false";\nAcquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' > /etc/apt/apt.conf.d/99workcell-validator-snapshot \
   && install_validator_packages() { \
        DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -106,6 +110,7 @@ RUN fetch_snapshot_bootstrap_package() { \
      } \
   && for attempt in 1 2 3; do \
        rm -rf /var/lib/apt/lists/*; \
+       if [[ $((attempt % 2)) -eq 1 ]]; then set_snapshot_sources snapshot-cloudflare.debian.org; else set_snapshot_sources snapshot.debian.org; fi; \
        if DEBIAN_FRONTEND=noninteractive apt-get update \
          && install_validator_packages; then \
          break; \