fix(vm): restore sandboxes after gateway restart (#1407)

Author: drew

.github/workflows/driver-vm-linux.yml

@@ -134,24 +134,10 @@ jobs:
         run: |
           set -euo pipefail
           COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed"
-          mkdir -p "$COMPRESSED_DIR"
-
-          EXTRACT_DIR=$(mktemp -d)
-          zstd -d "runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" --stdout \
-            | tar -xf - -C "$EXTRACT_DIR"
-
-          echo "Extracted runtime files:"
-          ls -lah "$EXTRACT_DIR"
-
-          for file in "$EXTRACT_DIR"/*; do
-            [ -f "$file" ] || continue
-            name=$(basename "$file")
-            [ "$name" = "provenance.json" ] && continue
-            zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
-          done
-
-          echo "Staged compressed runtime artifacts:"
-          ls -lah "$COMPRESSED_DIR"
+          VM_RUNTIME_TARBALL="${PWD}/runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" \
+          VM_RUNTIME_PLATFORM="${{ matrix.platform }}" \
+          OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$COMPRESSED_DIR" \
+            tasks/scripts/vm/compress-vm-runtime.sh
 
       - name: Build bundled supervisor
         run: |

.github/workflows/driver-vm-macos.yml

@@ -165,24 +165,10 @@ jobs:
         run: |
           set -euo pipefail
           COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed-macos"
-          mkdir -p "$COMPRESSED_DIR"
-
-          EXTRACT_DIR=$(mktemp -d)
-          zstd -d "runtime-download/vm-runtime-darwin-aarch64.tar.zst" --stdout \
-            | tar -xf - -C "$EXTRACT_DIR"
-
-          echo "Extracted darwin runtime files:"
-          ls -lah "$EXTRACT_DIR"
-
-          for file in "$EXTRACT_DIR"/*; do
-            [ -f "$file" ] || continue
-            name=$(basename "$file")
-            [ "$name" = "provenance.json" ] && continue
-            zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
-          done
-
-          echo "Staged macOS compressed runtime artifacts:"
-          ls -lah "$COMPRESSED_DIR"
+          VM_RUNTIME_TARBALL="${PWD}/runtime-download/vm-runtime-darwin-aarch64.tar.zst" \
+          VM_RUNTIME_PLATFORM="darwin-aarch64" \
+          OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$COMPRESSED_DIR" \
+            tasks/scripts/vm/compress-vm-runtime.sh
 
       - name: Download bundled supervisor
         uses: actions/download-artifact@v4

Cargo.lock

@@ -29,7 +29,7 @@ reason strings.
 | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. |
 | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. |
 | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. |
-| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Gateway spawns `openshell-driver-vm` as a subprocess over a private, state-local Unix socket. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. |
+| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Gateway spawns `openshell-driver-vm` as a subprocess over a private, state-local Unix socket. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. |
 
 Per-sandbox CPU and memory values currently enter the driver layer through
 template resource limits. Docker and Podman apply them as runtime limits.

architecture/compute-runtimes.md

@@ -25,6 +25,7 @@ openshell-vfio = { path = "../openshell-vfio" }
 bollard = { version = "0.20", features = ["ssh"] }
 tokio = { workspace = true }
 tonic = { workspace = true, features = ["transport"] }
+prost = { workspace = true }
 prost-types = { workspace = true }
 futures = { workspace = true }
 tokio-stream = { workspace = true, features = ["net"] }

crates/openshell-driver-vm/Cargo.toml

@@ -189,6 +189,13 @@ the overlay while cached image disks remain unchanged. The overlay disk must be
 large enough to hold the compressed payload, unpacked rootfs, and sandbox writes
 during the first prepare.
 
+The driver also writes the accepted `DriverSandbox` launch request to
+`<state-dir>/sandboxes/<id>/sandbox.pb`. If the gateway restarts, it starts a
+new VM driver process; that process scans the sandbox state directories,
+restarts each persisted VM launcher, and preserves any existing `overlay.ext4`
+instead of cloning a fresh overlay template. If a restart happened before the
+overlay was created, the driver creates it during the resume attempt.
+
 ## Logs and debugging
 
 Raise log verbosity for both processes: