fix: watchdog recycles the enclave when Gemma is wedged (v0.7.7)

entrypoint.sh

@@ -14,19 +14,31 @@
 # Failure escalation ladder:
 #   1. vLLM crashes          → supervisor restarts it (5 s → 300 s backoff).
 #   2. restarts don't stick  → server.py /healthz turns 503 after
-#      GEMMA_UNHEALTHY_AFTER (default 900 s) of continuous downtime →
-#      Tinfoil's healthcheck fails → the whole attested container is
-#      recycled (also clears wedged GPU state a process restart can't).
+#      GEMMA_UNHEALTHY_AFTER (default 900 s) of continuous downtime, and the
+#      in-container watchdog below then exits PID 1 → the container stops →
+#      Tinfoil `restart: always` recycles the whole attested unit (which
+#      also clears wedged GPU state a process-level restart can't).
 #   3. uvicorn (PID 1) dies  → container exits → Tinfoil `restart: always`.
-# The PII /filter path stays up through 1 and 2 — we deliberately do NOT
-# kill the container on a vLLM crash; redaction availability comes first.
+# The PII /filter path stays up through 1 and 2 until the instant of recycle —
+# we deliberately do NOT kill the container on a transient vLLM crash;
+# redaction availability comes first.
 #
-# History: v0.5.0–v0.7.5 instead ran a watcher doing `wait "$vllm_pid"`
-# from a sibling subshell to kill the container when vLLM died. `wait`
-# only works on children of the calling shell, so it exited 127 at boot
-# (silently, via `set -e`) and a crashed vLLM left /v1/* 502-ing forever
-# while /health stayed green. The supervisor below is vLLM's parent, so
-# its `wait` reaps the real exit status.
+# Why the watchdog (v0.7.7): the healthcheck points at /healthz so Docker
+# marks the container unhealthy on a sustained 503 — but `restart: always`
+# acts on container EXIT, not on an unhealthy healthcheck, so on Tinfoil an
+# unhealthy container is never recycled on its own. v0.7.6 relied on that
+# phantom recycle and a wedged vLLM sat down for ~5 days while /filter stayed
+# green and gemma_down_seconds climbed unbounded. The watchdog makes step 2
+# actionable from inside the container: it turns the /healthz 503 verdict
+# into the container exit that `restart: always` actually honors.
+#
+# History: v0.5.0–v0.7.5 ran a watcher doing `wait "$vllm_pid"` from a
+# sibling subshell to kill the container when vLLM died. `wait` only works
+# on children of the calling shell, so it exited 127 at boot (silently, via
+# `set -e`) and a crashed vLLM left /v1/* 502-ing forever while /health
+# stayed green. The supervisor below is vLLM's parent, so its `wait` reaps
+# the real exit status — but it restarts in-container forever and never
+# escalates on its own; that gap is what the watchdog closes.
 
 set -euo pipefail
 
@@ -44,6 +56,13 @@ GPU_MEM_UTIL="${GEMMA_GPU_MEM_UTIL:-0.25}"
 # minimum → maximum; a run that lasted longer than the maximum resets it.
 GEMMA_RESTART_BACKOFF_MIN="${GEMMA_RESTART_BACKOFF_MIN:-5}"
 GEMMA_RESTART_BACKOFF_MAX="${GEMMA_RESTART_BACKOFF_MAX:-300}"
+# Watchdog: how often to poll /healthz, and how many consecutive 503s before
+# we recycle. /healthz only 503s after GEMMA_UNHEALTHY_AFTER (default 900 s)
+# of continuous gemma downtime, so 3 × 30 s just adds ~90 s of confirmation —
+# a recycle fires ~990 s into an unrecoverable outage, never on a transient
+# crash the supervisor heals in seconds.
+GEMMA_WATCHDOG_INTERVAL="${GEMMA_WATCHDOG_INTERVAL:-30}"
+GEMMA_WATCHDOG_FAILS="${GEMMA_WATCHDOG_FAILS:-3}"
 
 # vLLM supervisor: this subshell is vLLM's parent, so `wait` reaps the
 # real exit status. Output goes to the container log (fd 1/2 of PID 1) so
@@ -83,6 +102,34 @@ GEMMA_RESTART_BACKOFF_MAX="${GEMMA_RESTART_BACKOFF_MAX:-300}"
     done
 ) >/proc/1/fd/1 2>/proc/1/fd/2 &
 
+# Watchdog: enforce escalation step 2. Polls the loopback /healthz, which
+# already encodes the "gemma down past policy" verdict (503 once downtime
+# exceeds GEMMA_UNHEALTHY_AFTER). `restart: always` recycles on EXIT, not on
+# an unhealthy healthcheck, so on a sustained 503 we exit PID 1 ourselves —
+# the container stops and Tinfoil recycles it, clearing wedged GPU state.
+# Only an explicit HTTP 503 counts; connection-refused during boot (uvicorn
+# not bound yet) returns 0 and resets the counter, so we never recycle on
+# cold start. http.client (not urllib) is used so a 503 is a return value,
+# not an exception.
+(
+    fails=0
+    while true; do
+        sleep "${GEMMA_WATCHDOG_INTERVAL}"
+        code="$(python3 -c "import http.client; c=http.client.HTTPConnection('127.0.0.1', ${APP_PORT}, timeout=5); c.request('GET','/healthz'); print(c.getresponse().status)" 2>/dev/null || echo 0)"
+        if [ "${code}" = "503" ]; then
+            fails=$(( fails + 1 ))
+            echo "[watchdog] /healthz=503 (gemma down past policy) ${fails}/${GEMMA_WATCHDOG_FAILS}"
+        else
+            fails=0
+        fi
+        if [ "${fails}" -ge "${GEMMA_WATCHDOG_FAILS}" ]; then
+            echo "[watchdog] gemma unrecoverable in-container — exiting PID 1 so restart:always recycles the CVM (clears wedged GPU)"
+            kill -TERM 1
+            exit 0
+        fi
+    done
+) >/proc/1/fd/1 2>/proc/1/fd/2 &
+
 echo "[entrypoint] launching uvicorn (privacy-filter + /v1 proxy) on :${APP_PORT}"
 cd /app
 exec uvicorn server:app \

tinfoil-config.yml

@@ -39,7 +39,7 @@ containers:
     # Bump flow: dispatch release.yml → copy digest from "Print digest"
     # step → paste here → commit → push a new `v*` tag to trigger
     # tinfoil-build.yml which attests this exact digest.
-    image: "ghcr.io/screenpipe/privacy-filter:0.7.6@sha256:ab774e6c02d7490d906445828fb0d478308068837b99ed3f93b1a86e82c80691"
+    image: "ghcr.io/screenpipe/privacy-filter:0.7.7@sha256:REPLACE_WITH_DIGEST"
     runtime: nvidia
     gpus: all
     # ipc:host is required by some CUDA workloads (PyTorch's shared-mem
@@ -56,9 +56,12 @@ containers:
       # /healthz (not /health) folds in the gemma-restart policy: 200 while
       # gemma is up or while entrypoint.sh's supervisor is restarting a
       # crashed vLLM; 503 once gemma has been continuously down past
-      # GEMMA_UNHEALTHY_AFTER (default 900 s) — i.e. restarts aren't
-      # sticking — so Tinfoil recycles the whole attested container.
-      # urlopen raises on the 503, failing the check.
+      # GEMMA_UNHEALTHY_AFTER (default 900 s) — i.e. restarts aren't sticking.
+      # This marks the container unhealthy for observability, but the actual
+      # recycle is driven by entrypoint.sh's watchdog (v0.7.7), which exits
+      # PID 1 on a sustained 503 so `restart: always` fires — `restart:
+      # always` does NOT recycle on an unhealthy healthcheck alone, only on
+      # container exit. urlopen raises on the 503, failing the check.
       test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=3)"]
       interval: "15s"
       timeout: "5s"