feat(sandbox): add callable python exec API and refresh e2e coverage (!19)

Closes NVIDIA#13

## Summary
- add Python sandbox execution APIs for command and callable workflows
- consolidate sandbox policy fixtures and expand e2e test coverage for policy and Python exec paths
- update CI/build config and images for sandbox e2e execution dependencies

## Test Plan
- mise run pre-commit

Author: drew

.agent/skills/debug-navigator-cluster/SKILL.md

@@ -25,6 +25,13 @@ Diagnose why a navigator cluster failed to start after `nav cluster admin deploy
    - If TLS enabled: `navigator-cli-client` secret exists with cert data
 9. Extract mTLS credentials if TLS is enabled (up to 3 min)
 
+For local deploys, metadata endpoint selection now depends on Docker connectivity:
+
+- default local Docker socket (`unix:///var/run/docker.sock`): `https://127.0.0.1`
+- TCP Docker daemon (`DOCKER_HOST=tcp://<host>:<port>`): `https://<host>` for non-loopback hosts
+
+The TCP host is also added as an extra gateway TLS SAN so mTLS hostname validation succeeds.
+
 The default cluster name is `navigator`. The container is `navigator-cluster-{name}`.
 
 ## Prerequisites
@@ -173,6 +180,8 @@ If ports are missing or conflicting, another process may be using them. Check wi
 ss -tlnp | grep -E ':(6443|80|443|30051)\s'
 ```
 
+If using Docker-in-Docker (`DOCKER_HOST=tcp://docker:2375`), verify metadata points at `https://docker` (not `https://127.0.0.1`).
+
 ### Step 6: Check Image Availability
 
 Component images (server, sandbox, pki-job) can reach k3s containerd via two paths:
@@ -279,9 +288,11 @@ If DNS is broken, all image pulls from the distribution registry will fail, as w
 | Architecture mismatch (remote) | Built on arm64, deploying to amd64 | Cross-build the image for the target architecture |
 | SSH connection failed (remote) | SSH key/host/Docker issues | Test `ssh <host> docker ps` manually |
 | Port conflict | Another service on 6443/80/443/30051 | Stop conflicting service or change port mapping |
+| gRPC connect refused to `127.0.0.1:443` in CI | Docker daemon is remote (`DOCKER_HOST=tcp://...`) but metadata still points to loopback | Verify metadata endpoint host matches `DOCKER_HOST` and includes non-loopback host |
 | DNS failures inside container | Entrypoint DNS detection failed | Check `/etc/rancher/k3s/resolv.conf` and container startup logs |
 | `metrics-server` errors in logs | Normal k3s noise, not the root cause | These errors are benign — look for the actual failing health check component |
 | Stale NotReady nodes from previous deploys | Volume reused across container recreations | The deploy flow now auto-cleans stale nodes; if it still fails, manually delete NotReady nodes (see Step 3) or choose "Recreate" when prompted |
+| gRPC `UNIMPLEMENTED` for newer RPCs in push mode | Helm values still point at older pulled images instead of the pushed refs | Verify rendered `navigator-helmchart.yaml` uses the expected push refs (`server`, `sandbox`, `pki-job`) and not `:latest` |
 
 ## Remote Cluster Debugging
 

.gitignore

@@ -94,6 +94,10 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+# Generated Python protobuf stubs (keep package marker)
+python/navigator/_proto/*
+!python/navigator/_proto/__init__.py
+
 # =============================================================================
 # IDE / Editor
 # =============================================================================

.gitlab-ci.yml

@@ -18,8 +18,7 @@ variables:
 default:
   image: $CI_IMAGE
   tags:
-    - os/linux
-    - type/docker
+    - agent-dev-kit-build
 
 cache:
   key: "$CI_COMMIT_REF_SLUG"
@@ -40,8 +39,7 @@ build_ci_image:
   services:
     - docker:24-dind
   tags:
-    - os/linux
-    - type/docker
+    - agent-dev-kit-build
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
     DOCKER_BUILDKIT: "1"
@@ -90,6 +88,30 @@ python_test:
   script:
     - mise run test:python
 
+python_e2e_sandbox_test:
+  stage: test
+  services:
+    - docker:24-dind
+  tags:
+    - agent-dev-kit-build
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_TLS_VERIFY: ""
+    DOCKER_CERT_PATH: ""
+  before_script:
+    - uv sync --frozen
+    - apt-get update -qq && apt-get install -y -qq socat >/dev/null
+    - curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz | tar xz --strip-components=1 -C /usr/local/bin docker/docker
+    - mkdir -p /usr/local/lib/docker/cli-plugins
+    - curl -fsSL https://github.com/docker/buildx/releases/download/v0.21.1/buildx-v0.21.1.linux-amd64 -o /usr/local/lib/docker/cli-plugins/docker-buildx
+    - chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
+  script:
+    - socat UNIX-LISTEN:/var/run/docker.sock,fork,reuseaddr TCP:docker:2375 &
+    - sleep 1
+    - mise run cluster
+    - mise run test:e2e:sandbox
+
 # =============================================================================
 # Publish Jobs (main branch only)
 # =============================================================================
@@ -98,11 +120,7 @@ publish_ecr_images:
   services:
     - docker:24-dind
   tags:
-    - os/linux
-    - perflab
-    - ran-as/container
-    - size/large
-    - type/docker
+    - agent-dev-kit-build
   variables:
     DOCKER_HOST: tcp://docker:2375
     DOCKER_TLS_CERTDIR: ""

CONTRIBUTING.md

@@ -100,8 +100,17 @@ mise run check           # Quick compile check
 mise run test            # All tests (Rust + Python)
 mise run test:rust       # Rust tests only
 mise run test:python     # Python tests only
+mise run test:e2e:sandbox # Sandbox Python e2e tests
 ```
 
+### Python E2E Test Patterns
+
+- Put sandbox SDK e2e tests in `e2e/python/`.
+- Prefer `Sandbox.exec_python(...)` with Python callables over inline `python -c` strings.
+- Define callable helpers inside the test function when possible so they serialize cleanly in sandbox.
+- Keep scenarios focused: one test for happy path and separate tests for negative/policy enforcement behavior.
+- Use `mise run test:e2e:sandbox` to run this suite locally.
+
 ### Linting & Formatting
 
 ```bash
@@ -177,6 +186,10 @@ mise run python:dev      # Install Python package in development mode (builds CL
 mise run python:build    # Build Python wheel with CLI binary
 ```
 
+Python protobuf stubs in `python/navigator/_proto/` are generated artifacts and are gitignored
+(except `__init__.py`). `mise` Python build/test/lint/typecheck tasks run `python:proto`
+automatically, so you generally do not need to generate stubs manually.
+
 ### Publishing
 
 Versions are derived from git tags using `setuptools_scm`. No version bumps need to be committed.

Cargo.lock

@@ -155,12 +155,13 @@ For the target daemon (local or remote):
 
 1. Ensure bridge network `navigator-cluster` (attachable, bridge driver).
 2. Ensure volume `navigator-cluster-{name}`.
-3. Compute extra TLS SANs for remote deploys:
+3. Compute extra TLS SANs:
+   - For local deploys, when `DOCKER_HOST` is a non-loopback `tcp://` endpoint (for example `tcp://docker:2375` in CI), add that host as an extra SAN.
    - Resolve the SSH host via `ssh -G` to get the canonical hostname/IP.
    - Add resolved host (and original SSH host if different) as extra `--tls-san` arguments.
    - Set container env vars `EXTRA_SANS`, `SSH_GATEWAY_HOST`, `SSH_GATEWAY_PORT=8080`.
 4. Ensure container `navigator-cluster-{name}` with:
-   - k3s server command: `server --disable=traefik --tls-san=127.0.0.1 --tls-san=localhost --tls-san=host.docker.internal` (plus extra SANs for remote).
+   - k3s server command: `server --disable=traefik --tls-san=127.0.0.1 --tls-san=localhost --tls-san=host.docker.internal` (plus computed extra SANs).
    - privileged mode,
    - bind mount of volume to `/var/lib/rancher/k3s`,
    - network mode `navigator-cluster`,
@@ -212,7 +213,7 @@ Write is performed atomically through temp + backup directory swap.
 
 Bootstrap writes metadata JSON for the cluster:
 
-- local: endpoint `https://127.0.0.1`, `is_remote=false`
+- local: endpoint `https://127.0.0.1` by default, or `https://{docker_host}` when `DOCKER_HOST` is a non-loopback `tcp://` endpoint; `is_remote=false`
 - remote: endpoint `https://{resolved_host}`, `is_remote=true`, plus SSH destination and resolved host
 
 Metadata fields:
@@ -304,6 +305,9 @@ nav cluster admin tunnel --name <name> --remote user@host
 | `XDG_CONFIG_HOME` | Base config directory (default: `$HOME/.config`) |
 | `KUBECONFIG` | Target kubeconfig path for merge (first colon-separated path; default: `$HOME/.kube/config`) |
 
+When `NAVIGATOR_PUSH_IMAGES` is enabled, the entrypoint rewrites HelmChart image tags from `latest` to `IMAGE_TAG` and now handles both quoted and unquoted `tag: latest` formats.
+In push mode, bootstrap also passes exact imported image refs (`server`, `sandbox`, `pki-job`) to the entrypoint, which rewrites Helm values to those refs directly before the tag/pull-policy overrides. Image import uses the `k8s.io` containerd namespace so kubelet resolves the pushed refs without falling back to pulled registry tags. After import, bootstrap restarts `deployment/navigator` and waits for rollout completion so running pods pick up the imported image references.
+
 Container-level env vars set for remote deploys:
 
 | Variable | Value |

architecture/cluster-bootstrap.md

@@ -57,6 +57,7 @@ The server container runs the Navigator orchestration service.
 - Exposes gRPC/HTTP on port 8080
 - Health checks at `/healthz`
 - SQLx migrations copied from source
+- Uses an embedded Rust SSH client (`russh`) for sandbox exec
 
 ### navigator-cluster
 
@@ -79,6 +80,7 @@ An airgapped k3s image with all components pre-loaded for single-container deplo
 When running k3s in Docker, the container's `/etc/resolv.conf` contains Docker's internal DNS (127.0.0.11), which is not reachable from k3s pods. While k3s auto-detects this and falls back to 8.8.8.8, external UDP traffic doesn't work reliably on Docker Desktop.
 
 The `cluster-entrypoint.sh` script solves this by:
+
 1. Detecting the Docker host gateway IP from `/etc/hosts` (requires `--add-host=host.docker.internal:host-gateway`)
 2. Writing a custom resolv.conf with the host gateway as the nameserver
 3. Passing `--resolv-conf` to k3s to use this configuration

architecture/containers.md

@@ -2,21 +2,25 @@
 
 ["python:dev"]
 description = "Install Python package in development mode (builds CLI binary)"
+depends = ["python:proto"]
 run = "uv sync --group dev && uv pip install ."
 
 ["python:build"]
 description = "Build Python wheel with CLI binary (native)"
+depends = ["python:proto"]
 run = "uv run maturin build --release"
 
 ["python:build:linux-x86"]
 description = "Build Python wheel for Linux x86_64"
+depends = ["python:proto"]
 run = """
 docker run --rm --platform linux/amd64 -v $(pwd):/io -w /io \
   ghcr.io/pyo3/maturin build --release
 """
 
 ["python:build:linux-arm"]
 description = "Build Python wheel for Linux aarch64"
+depends = ["python:proto"]
 run = """
 docker run --rm --platform linux/arm64 -v $(pwd):/io -w /io \
   ghcr.io/pyo3/maturin build --release
@@ -28,6 +32,7 @@ depends = ["python:build", "python:build:linux-x86", "python:build:linux-arm"]
 
 ["python:lint"]
 description = "Lint Python code with ruff"
+depends = ["python:proto"]
 env = { UV_NO_SYNC = "1" }
 run = "uv run ruff check {{vars.python_paths}}"
 
@@ -37,6 +42,7 @@ run = "uv run ruff format {{vars.python_paths}}"
 
 ["python:typecheck"]
 description = "Type check Python code with ty"
+depends = ["python:proto"]
 run = "uv run ty check {{vars.python_paths}}"
 
 ["python:proto"]
@@ -50,8 +56,42 @@ uv run python -m grpc_tools.protoc \
   --python_out=python/navigator/_proto \
   --pyi_out=python/navigator/_proto \
   --grpc_python_out=python/navigator/_proto \
-  proto/inference.proto
-# Fix absolute imports in generated gRPC stubs to use relative imports
-sed -i '' 's/^import inference_pb2/from . import inference_pb2/' \
-  python/navigator/_proto/inference_pb2_grpc.py
-"""
+  proto/inference.proto \
+  proto/navigator.proto \
+  proto/datamodel.proto \
+  proto/sandbox.proto
+# Fix absolute imports in generated stubs to use package-relative imports
+uv run python - <<'PY'
+from pathlib import Path
+
+replacements = {
+    "python/navigator/_proto/inference_pb2_grpc.py": [
+        ("import inference_pb2", "from . import inference_pb2"),
+    ],
+    "python/navigator/_proto/navigator_pb2_grpc.py": [
+        ("import navigator_pb2", "from . import navigator_pb2"),
+        ("import sandbox_pb2", "from . import sandbox_pb2"),
+    ],
+    "python/navigator/_proto/navigator_pb2.py": [
+        ("import datamodel_pb2", "from . import datamodel_pb2"),
+        ("import sandbox_pb2", "from . import sandbox_pb2"),
+    ],
+    "python/navigator/_proto/datamodel_pb2.py": [
+        ("import sandbox_pb2", "from . import sandbox_pb2"),
+    ],
+    "python/navigator/_proto/datamodel_pb2_grpc.py": [
+        ("import datamodel_pb2", "from . import datamodel_pb2"),
+    ],
+    "python/navigator/_proto/sandbox_pb2_grpc.py": [
+        ("import sandbox_pb2", "from . import sandbox_pb2"),
+    ],
+}
+
+for path, rules in replacements.items():
+    file_path = Path(path)
+    text = file_path.read_text()
+    for before, after in rules:
+        text = text.replace(before, after)
+    file_path.write_text(text)
+PY
+"""

build/python.toml

@@ -10,5 +10,12 @@ run = "cargo test --workspace"
 
 ["test:python"]
 description = "Run Python tests"
+depends = ["python:proto"]
 env = { UV_NO_SYNC = "1" }
 run = "uv run pytest python/"
+
+["test:e2e:sandbox"]
+description = "Run sandbox end-to-end tests"
+depends = ["python:proto"]
+env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
+run = "uv run pytest -o python_files='test_*.py' e2e/python"

build/test.toml

@@ -306,6 +306,11 @@ pub async fn ensure_container(
             .ok()
             .filter(|v| !v.trim().is_empty())
             .unwrap_or_else(|| "dev".to_string());
+        if let Ok(images) = std::env::var("NAVIGATOR_PUSH_IMAGES")
+            && !images.trim().is_empty()
+        {
+            env_vars.push(format!("PUSH_IMAGE_REFS={images}"));
+        }
         env_vars.push(format!("IMAGE_TAG={tag}"));
         env_vars.push("IMAGE_PULL_POLICY=IfNotPresent".to_string());
     } else if let Ok(tag) = std::env::var("IMAGE_TAG")