feat(vm): deploy NemoClaw helm chart inside gateway VM

Enable full NemoClaw control plane deployment inside the libkrun
microVM so e2e tests can run against the VM instead of Docker.

Build-time (build-rootfs.sh):
- Package helm chart and inject into k3s static charts directory
- Copy HelmChart CR and agent-sandbox manifests into rootfs
- Pull and save arm64 container images as tarballs for airgap boot

Boot-time (gateway-init.sh):
- Enable flannel CNI (remove --flannel-backend=none and related flags)
- Deploy bundled manifests to k3s auto-deploy directory
- Patch HelmChart CR for VM context (pullPolicy, SSH placeholders)
- Ensure DNS fallback when DHCP doesn't configure resolv.conf

Post-boot (lib.rs):
- Wait for navigator namespace created by Helm controller
- Generate PKI and apply TLS secrets via host kubectl
- Store cluster metadata and mTLS creds for CLI/SDK access
- Set 'gateway' as active cluster for e2e test discovery

Also bump VM to 8GB RAM / 4 vCPUs, add port 30051 forwarding,
fix nemoclaw wrapper fingerprint to include navigator-vm crate,
and add test:e2e:vm mise task.

Author: drew

Cargo.lock

@@ -4,13 +4,13 @@
 pub mod build;
 pub mod image;
 
-mod constants;
+pub mod constants;
 mod docker;
 mod kubeconfig;
 mod metadata;
-mod mtls;
+pub mod mtls;
 pub mod paths;
-mod pki;
+pub mod pki;
 pub(crate) mod push;
 mod runtime;
 

crates/navigator-bootstrap/src/lib.rs

@@ -15,8 +15,11 @@ name = "navigator_vm"
 path = "src/lib.rs"
 
 [dependencies]
+base64 = "0.22"
 libc = "0.2"
 miette = { workspace = true }
+navigator-bootstrap = { path = "../navigator-bootstrap" }
+serde_json = "1"
 thiserror = { workspace = true }
 
 [lints]

crates/navigator-vm/Cargo.toml

@@ -4,13 +4,13 @@
 
 # Build an aarch64 Ubuntu rootfs for the gateway microVM.
 #
-# Produces a rootfs with k3s pre-installed, plus the gateway-init.sh script
-# that runs as PID 1 inside the libkrun VM.
+# Produces a rootfs with k3s pre-installed, the NemoClaw helm chart and
+# manifests baked in, and container images pre-loaded for airgap boot.
 #
 # Usage:
 #   ./crates/navigator-vm/scripts/build-rootfs.sh [output_dir]
 #
-# Requires: Docker (or compatible container runtime), curl
+# Requires: Docker (or compatible container runtime), curl, helm
 
 set -euo pipefail
 
@@ -25,8 +25,19 @@ IMAGE_TAG="krun-rootfs:gateway"
 K3S_VERSION="${K3S_VERSION:-v1.29.8+k3s1}"
 K3S_VERSION="${K3S_VERSION//-k3s/+k3s}"
 
+# Project root (two levels up from crates/navigator-vm/scripts/)
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+
+# Container images to pre-load into k3s (arm64).
+IMAGE_REPO_BASE="${IMAGE_REPO_BASE:-d1i0nduu2f6qxk.cloudfront.net/navigator}"
+IMAGE_TAG="${IMAGE_TAG:-latest}"
+SERVER_IMAGE="${IMAGE_REPO_BASE}/server:${IMAGE_TAG}"
+SANDBOX_IMAGE="${IMAGE_REPO_BASE}/sandbox:${IMAGE_TAG}"
+AGENT_SANDBOX_IMAGE="registry.k8s.io/agent-sandbox/agent-sandbox-controller:v0.1.0"
+
 echo "==> Building gateway rootfs"
 echo "    k3s version: ${K3S_VERSION}"
+echo "    Images:      ${SERVER_IMAGE}, ${SANDBOX_IMAGE}"
 echo "    Output:      ${ROOTFS_DIR}"
 
 # ── Download k3s binary (outside Docker — much faster) ─────────────────
@@ -92,6 +103,69 @@ chmod +x "${ROOTFS_DIR}/srv/gateway-init.sh"
 cp "${SCRIPT_DIR}/hello-server.py" "${ROOTFS_DIR}/srv/hello-server.py"
 chmod +x "${ROOTFS_DIR}/srv/hello-server.py"
 
+# ── Package and inject helm chart ────────────────────────────────────
+
+HELM_CHART_DIR="${PROJECT_ROOT}/deploy/helm/navigator"
+CHART_DEST="${ROOTFS_DIR}/var/lib/rancher/k3s/server/static/charts"
+
+if [ -d "${HELM_CHART_DIR}" ]; then
+    echo "==> Packaging helm chart..."
+    mkdir -p "${CHART_DEST}"
+    helm package "${HELM_CHART_DIR}" -d "${CHART_DEST}"
+    echo "    $(ls "${CHART_DEST}"/*.tgz 2>/dev/null | xargs -I{} basename {})"
+else
+    echo "WARNING: Helm chart not found at ${HELM_CHART_DIR}, skipping"
+fi
+
+# ── Inject Kubernetes manifests ──────────────────────────────────────
+# These are copied to /opt/navigator/manifests/ (staging). gateway-init.sh
+# moves them to /var/lib/rancher/k3s/server/manifests/ at boot so the
+# k3s Helm Controller auto-deploys them.
+
+MANIFEST_SRC="${PROJECT_ROOT}/deploy/kube/manifests"
+MANIFEST_DEST="${ROOTFS_DIR}/opt/navigator/manifests"
+
+echo "==> Injecting Kubernetes manifests..."
+mkdir -p "${MANIFEST_DEST}"
+
+for manifest in navigator-helmchart.yaml agent-sandbox.yaml; do
+    if [ -f "${MANIFEST_SRC}/${manifest}" ]; then
+        cp "${MANIFEST_SRC}/${manifest}" "${MANIFEST_DEST}/"
+        echo "    ${manifest}"
+    else
+        echo "WARNING: ${manifest} not found in ${MANIFEST_SRC}"
+    fi
+done
+
+# ── Pre-load container images ────────────────────────────────────────
+# Pull arm64 images and save as tarballs in the k3s airgap images
+# directory. k3s auto-imports from /var/lib/rancher/k3s/agent/images/
+# on startup, so no internet access is needed at boot time.
+
+IMAGES_DIR="${ROOTFS_DIR}/var/lib/rancher/k3s/agent/images"
+mkdir -p "${IMAGES_DIR}"
+
+echo "==> Pre-loading container images (arm64)..."
+
+pull_and_save() {
+    local image="$1"
+    local output="$2"
+
+    if [ -f "${output}" ]; then
+        echo "    cached: $(basename "${output}")"
+        return 0
+    fi
+
+    echo "    pulling: ${image}..."
+    docker pull --platform linux/arm64 "${image}" --quiet
+    echo "    saving:  $(basename "${output}")..."
+    docker save "${image}" -o "${output}"
+}
+
+pull_and_save "${SERVER_IMAGE}" "${IMAGES_DIR}/navigator-server.tar"
+pull_and_save "${SANDBOX_IMAGE}" "${IMAGES_DIR}/navigator-sandbox.tar"
+pull_and_save "${AGENT_SANDBOX_IMAGE}" "${IMAGES_DIR}/agent-sandbox-controller.tar"
+
 # ── Verify ────────────────────────────────────────────────────────────
 
 if [ ! -f "${ROOTFS_DIR}/usr/local/bin/k3s" ]; then
@@ -102,6 +176,14 @@ fi
 echo ""
 echo "==> Rootfs ready at: ${ROOTFS_DIR}"
 echo "    Size: $(du -sh "${ROOTFS_DIR}" | cut -f1)"
+
+# Show image sizes
+echo "    Images:"
+for img in "${IMAGES_DIR}"/*.tar; do
+    [ -f "$img" ] || continue
+    echo "      $(basename "$img"): $(du -sh "$img" | cut -f1)"
+done
+
 echo ""
 echo "Next steps:"
 echo "  1. Run:  ncl gateway"

crates/navigator-vm/scripts/build-rootfs.sh

@@ -4,7 +4,8 @@
 
 # Init script for the gateway microVM. Runs as PID 1 inside the libkrun VM.
 #
-# Mounts essential virtual filesystems, then execs k3s server.
+# Mounts essential virtual filesystems, deploys bundled manifests (helm chart,
+# agent-sandbox controller), then execs k3s server.
 
 set -e
 
@@ -80,6 +81,14 @@ DHCP_SCRIPT
         ip route add default via 192.168.127.1 2>/dev/null || true
     fi
 
+    # Ensure DNS is configured. DHCP should have set /etc/resolv.conf,
+    # but if it didn't (or static fallback was used), provide a default.
+    if [ ! -s /etc/resolv.conf ]; then
+        echo "[gateway-init] no DNS configured, using public DNS"
+        echo "nameserver 8.8.8.8" > /etc/resolv.conf
+        echo "nameserver 8.8.4.4" >> /etc/resolv.conf
+    fi
+
     # Read back the IP we got (from DHCP or static).
     NODE_IP=$(ip -4 addr show eth0 | grep -oP 'inet \K[^/]+' || echo "192.168.127.2")
     echo "[gateway-init] eth0 IP: $NODE_IP"
@@ -110,15 +119,53 @@ rm -f  /var/lib/rancher/k3s/server/kine.sock           2>/dev/null || true
 find /var/lib/rancher/k3s -name '*.sock' -delete 2>/dev/null || true
 find /run -name '*.sock' -delete 2>/dev/null || true
 
+# ── Deploy bundled manifests ────────────────────────────────────────────
+# Copy manifests from the staging directory to the k3s auto-deploy path.
+# This mirrors the approach in cluster-entrypoint.sh for the Docker path.
+
+K3S_MANIFESTS="/var/lib/rancher/k3s/server/manifests"
+BUNDLED_MANIFESTS="/opt/navigator/manifests"
+
+mkdir -p "$K3S_MANIFESTS"
+
+if [ -d "$BUNDLED_MANIFESTS" ]; then
+    echo "[gateway-init] deploying bundled manifests..."
+    for manifest in "$BUNDLED_MANIFESTS"/*.yaml; do
+        [ ! -f "$manifest" ] && continue
+        cp "$manifest" "$K3S_MANIFESTS/"
+        echo "  $(basename "$manifest")"
+    done
+
+    # Remove stale navigator-managed manifests from previous boots.
+    for existing in "$K3S_MANIFESTS"/navigator-*.yaml \
+                    "$K3S_MANIFESTS"/agent-*.yaml; do
+        [ ! -f "$existing" ] && continue
+        basename=$(basename "$existing")
+        if [ ! -f "$BUNDLED_MANIFESTS/$basename" ]; then
+            echo "  removing stale: $basename"
+            rm -f "$existing"
+        fi
+    done
+fi
+
+# Patch the HelmChart manifest for VM deployment.
+HELMCHART="$K3S_MANIFESTS/navigator-helmchart.yaml"
+if [ -f "$HELMCHART" ]; then
+    echo "[gateway-init] patching HelmChart manifest..."
+    # Use pre-loaded images — don't pull from registry.
+    sed -i 's|pullPolicy: Always|pullPolicy: IfNotPresent|' "$HELMCHART"
+    # Clear SSH gateway placeholders (default 127.0.0.1 is correct for local VM).
+    sed -i 's|sshGatewayHost: __SSH_GATEWAY_HOST__|sshGatewayHost: ""|g' "$HELMCHART"
+    sed -i 's|sshGatewayPort: __SSH_GATEWAY_PORT__|sshGatewayPort: 0|g' "$HELMCHART"
+fi
+
 # ── Start k3s ──────────────────────────────────────────────────────────
 
 echo "[gateway-init] starting k3s server..."
 exec /usr/local/bin/k3s server \
     --disable=traefik \
     --write-kubeconfig-mode=644 \
     --node-ip="$NODE_IP" \
-    --flannel-backend=none \
-    --disable-network-policy \
-    --disable-kube-proxy \
     --kube-apiserver-arg=bind-address=0.0.0.0 \
+    --resolv-conf=/etc/resolv.conf \
     --tls-san=localhost,127.0.0.1,10.0.2.15,192.168.127.2