feat(sandboxes): add nvidia gpu sandbox image (#72)

drew · web-flow · commit 13239b634a0a · 2026-05-12T21:14:59.000-07:00
Signed-off-by: Drew Newberry &lt;anewberry@nvidia.com&gt;
diff --git a/.github/workflows/build-sandboxes.yml b/.github/workflows/build-sandboxes.yml
@@ -176,33 +176,47 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Determine parent sandbox
+        id: parent
+        run: |
+          set -euo pipefail
+          DOCKERFILE="sandboxes/${{ matrix.sandbox }}/Dockerfile"
+          DEFAULT_BASE=""
+          if grep -q '^ARG BASE_IMAGE=' "$DOCKERFILE"; then
+            DEFAULT_BASE=$(grep '^ARG BASE_IMAGE=' "$DOCKERFILE" | head -1 | cut -d= -f2-)
+          fi
+
+          PARENT=""
+          if [ -n "$DEFAULT_BASE" ]; then
+            PARENT=$(echo "$DEFAULT_BASE" | sed -n 's|.*/sandboxes/\([^:]*\).*|\1|p')
+            if [ -z "$PARENT" ]; then
+              PARENT="base"
+            fi
+          fi
+
+          echo "sandbox=$PARENT" >> "$GITHUB_OUTPUT"
+          if [ -n "$PARENT" ]; then
+            echo "Parent for ${{ matrix.sandbox }}: $PARENT"
+          else
+            echo "${{ matrix.sandbox }} is a standalone sandbox image"
+          fi
+
       # On PRs the base image is not in GHCR. Build it locally, push to the
-      # local registry, and override BASE_IMAGE to point there.
+      # local registry, and override BASE_IMAGE to point there for dependent
+      # sandbox images. Standalone images do not need this step.
       - name: Build base image locally (PR only)
-        if: github.ref != 'refs/heads/main'
+        if: github.ref != 'refs/heads/main' && steps.parent.outputs.sandbox != ''
         uses: docker/build-push-action@v6
         with:
           context: sandboxes/base
           push: true
           tags: localhost:5000/sandboxes/base:latest
           cache-from: type=gha,scope=base
 
-      - name: Determine parent sandbox
-        id: parent
-        run: |
-          set -euo pipefail
-          DEFAULT_BASE=$(grep '^ARG BASE_IMAGE=' "sandboxes/${{ matrix.sandbox }}/Dockerfile" | head -1 | cut -d= -f2-)
-          PARENT=$(echo "$DEFAULT_BASE" | sed -n 's|.*/sandboxes/\([^:]*\).*|\1|p')
-          if [ -z "$PARENT" ]; then
-            PARENT="base"
-          fi
-          echo "sandbox=$PARENT" >> "$GITHUB_OUTPUT"
-          echo "Parent for ${{ matrix.sandbox }}: $PARENT"
-
       # When a sandbox depends on another sandbox (not base), build that
       # intermediate parent locally so it is available to the buildx build.
       - name: Build parent sandbox locally (PR only)
-        if: github.ref != 'refs/heads/main' && steps.parent.outputs.sandbox != 'base'
+        if: github.ref != 'refs/heads/main' && steps.parent.outputs.sandbox != '' && steps.parent.outputs.sandbox != 'base'
         uses: docker/build-push-action@v6
         with:
           context: sandboxes/${{ steps.parent.outputs.sandbox }}
@@ -216,7 +230,9 @@ jobs:
         id: base
         run: |
           PARENT="${{ steps.parent.outputs.sandbox }}"
-          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+          if [ -z "$PARENT" ]; then
+            echo "image=" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
             echo "image=${{ env.REGISTRY }}/${{ steps.repo.outputs.image_prefix }}/sandboxes/${PARENT}:latest" >> "$GITHUB_OUTPUT"
           else
             echo "image=localhost:5000/sandboxes/${PARENT}:latest" >> "$GITHUB_OUTPUT"
@@ -231,11 +247,20 @@ jobs:
             type=sha,prefix=
             type=raw,value=latest,enable={{is_default_branch}}
 
+      - name: Set build platforms
+        id: platforms
+        run: |
+          if [ "${{ github.ref }}" = "refs/heads/main" ] || [ "${{ matrix.sandbox }}" = "nvidia-gpu" ]; then
+            echo "value=linux/amd64,linux/arm64" >> "$GITHUB_OUTPUT"
+          else
+            echo "value=linux/amd64" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
           context: sandboxes/${{ matrix.sandbox }}
-          platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+          platforms: ${{ steps.platforms.outputs.value }}
           push: ${{ github.ref == 'refs/heads/main' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -244,4 +269,3 @@ jobs:
           cache-from: type=gha,scope=${{ matrix.sandbox }}
           cache-to: type=gha,mode=max,scope=${{ matrix.sandbox }}
 
-
diff --git a/README.md b/README.md
@@ -26,6 +26,7 @@ This repo is the community ecosystem around OpenShell -- a hub for contributed s
 | `sandboxes/ollama/`     | Ollama for local and cloud LLMs with Claude Code, Codex, OpenCode pre-installed |
 | `sandboxes/sdg/`        | Synthetic data generation workflows                          |
 | `sandboxes/openclaw/`   | OpenClaw -- open agent manipulation and control              |
+| `sandboxes/nvidia-gpu/` | GPU-enabled VM sandbox image with NVIDIA userspace tooling   |
 
 ## Getting Started
 
diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES
@@ -19,6 +19,10 @@ Image: docker/dockerfile:1.4 (BuildKit frontend)
 License: Apache-2.0
 URL: https://github.com/moby/buildkit
 
+Image: nvidia/cuda:12.8.1-base-ubuntu22.04
+License: NVIDIA CUDA Toolkit End User License Agreement and Ubuntu component licenses
+URL: https://hub.docker.com/r/nvidia/cuda
+
 ================================================================================
 System Packages (APT — Ubuntu 24.04)
 ================================================================================
diff --git a/sandboxes/nvidia-gpu/Dockerfile b/sandboxes/nvidia-gpu/Dockerfile
@@ -0,0 +1,106 @@
+# syntax=docker/dockerfile:1
+
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# GPU-enabled sandbox image for OpenShell VM driver.
+#
+# Provides userspace GPU tooling (nvidia-smi, NVML, CUDA driver libs, kmod)
+# on top of a minimal Ubuntu base with the full NVIDIA driver userspace
+# installed via the official .run installer (no kernel modules -- those are
+# injected at rootfs preparation time by the VM driver).
+#
+# Usage:
+#   openshell sandbox create --gpu --from ./sandboxes/nvidia-gpu/Dockerfile
+#   openshell sandbox create --gpu --from nvidia-gpu   # once published
+#
+# Build-time args:
+#   CUDA_VERSION          - CUDA toolkit version (default: 12.8.1)
+#   UBUNTU_VERSION        - Ubuntu release (default: 22.04)
+#   NVIDIA_DRIVER_VERSION - Must match the kernel modules built by
+#                           `mise run vm:nvidia-modules` in the OpenShell
+#                           core repo (default: 580.159.03)
+#   TARGETARCH            - Set automatically by BuildKit (amd64 or arm64)
+
+ARG CUDA_VERSION=12.8.1
+ARG UBUNTU_VERSION=22.04
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION}
+
+ARG CUDA_VERSION
+ARG UBUNTU_VERSION
+# Must match NVIDIA_DRIVER_VERSION in sandboxes/nvidia-gpu/versions.env
+# and NVIDIA_OPEN_VERSION in the OpenShell core VM module build.
+ARG NVIDIA_DRIVER_VERSION=580.159.03
+ARG TARGETARCH
+
+# ── System packages required by the sandbox init script ──────────────
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        bash \
+        busybox-static \
+        ca-certificates \
+        curl \
+        iproute2 \
+        iptables \
+        kmod \
+        pciutils \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /usr/share/udhcpc && ln -sf /bin/busybox /sbin/udhcpc
+
+# ── NVIDIA driver userspace ──────────────────────────────────────────
+# The nvidia/cuda base image does NOT include the driver (nvidia-smi,
+# libcuda.so, libnvidia-ml.so). It relies on the NVIDIA Container
+# Runtime to mount them from the host. In a VM there is no container
+# runtime, so we install the driver userspace via the .run installer
+# with --no-kernel-module (kernel modules are injected separately).
+# TODO(gpu): Pin SHA-256 checksum for reproducible builds. Compute with:
+#   curl -fsSL <url> | sha256sum
+RUN set -eux; \
+    case "${TARGETARCH}" in \
+        amd64) nvidia_arch="x86_64" ;; \
+        arm64) nvidia_arch="aarch64" ;; \
+        *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    curl -fsSL \
+        "https://download.nvidia.com/XFree86/Linux-${nvidia_arch}/${NVIDIA_DRIVER_VERSION}/NVIDIA-Linux-${nvidia_arch}-${NVIDIA_DRIVER_VERSION}.run" \
+        -o /tmp/nvidia.run \
+    && chmod +x /tmp/nvidia.run \
+    && /tmp/nvidia.run \
+        --silent \
+        --no-kernel-module \
+        --no-drm \
+        --no-x-check \
+        --no-systemd \
+        --no-nvidia-modprobe \
+        --no-distro-scripts \
+    && rm -f /tmp/nvidia.run
+
+# Ensure library paths are indexed for dlopen.
+RUN set -eux; \
+    case "${TARGETARCH}" in \
+        amd64) deb_arch="x86_64-linux-gnu" ;; \
+        arm64) deb_arch="aarch64-linux-gnu" ;; \
+        *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    mkdir -p /etc/ld.so.conf.d; \
+    printf "/usr/local/cuda/lib64\n/usr/lib/%s\n" "${deb_arch}" > /etc/ld.so.conf.d/cuda.conf; \
+    ldconfig 2>/dev/null || true
+
+# ── Kernel modules ───────────────────────────────────────────────────
+# NVIDIA kernel modules (.ko) must match the guest VM kernel (libkrunfw).
+# They are NOT in this image -- the VM driver injects them at rootfs
+# preparation time via `inject_gpu_modules`.
+#
+# GSP firmware (.bin) IS provided by the .run installer above. The VM
+# driver detects its presence and skips firmware injection, avoiding
+# version mismatches when the host driver differs from this image's.
+RUN mkdir -p /lib/modules
+
+LABEL org.opencontainers.image.title="OpenShell GPU Sandbox" \
+      org.opencontainers.image.description="GPU-enabled sandbox for OpenShell VM driver with CUDA support" \
+      org.opencontainers.image.version="${NVIDIA_DRIVER_VERSION}" \
+      io.openshell.sandbox.cuda-version="${CUDA_VERSION}" \
+      io.openshell.sandbox.ubuntu-version="${UBUNTU_VERSION}" \
+      io.openshell.sandbox.nvidia-driver-version="${NVIDIA_DRIVER_VERSION}" \
+      io.openshell.sandbox.gpu="true"
diff --git a/sandboxes/nvidia-gpu/README.md b/sandboxes/nvidia-gpu/README.md
@@ -0,0 +1,105 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# GPU Sandbox Image
+
+GPU-enabled sandbox image for the OpenShell VM driver. Provides NVIDIA
+userspace tooling (nvidia-smi, NVML, CUDA driver libraries) on top of a
+minimal Ubuntu base. Kernel modules are injected separately by the VM
+driver at sandbox creation time. The image publishes for `linux/amd64` and
+`linux/arm64`.
+
+## Architecture
+
+The GPU sandbox splits responsibility between the container image and the
+VM driver:
+
+| Layer | Source | Contents |
+|-------|--------|----------|
+| **Userspace** | This Dockerfile | nvidia-smi, libcuda.so, libnvidia-ml.so, kmod, iproute2 |
+| **Kernel modules** | OpenShell VM driver injection | nvidia.ko, nvidia_uvm.ko, nvidia_modeset.ko (built for guest kernel 6.12.76) |
+| **GSP firmware** | `.run` installer in image OR host fallback | gsp_ga10x.bin, gsp_tu10x.bin |
+
+The kernel modules must be compiled against the exact guest kernel version
+used by libkrunfw. The VM driver injects them into each sandbox's rootfs
+at creation time via `inject_gpu_modules()`.
+
+## Prerequisites
+
+- Linux host with an NVIDIA GPU
+- IOMMU enabled (for VFIO GPU passthrough)
+- Docker (for building the sandbox image)
+- OpenShell core checkout for VM runtime/module tasks
+- Guest kernel built with `CONFIG_MODULES=y` in the OpenShell core checkout (`mise run vm:setup`)
+
+## Quick Start
+
+```shell
+# 1. In the OpenShell core repo: build the VM runtime
+mise run vm:setup
+
+# 2. In the OpenShell core repo: build NVIDIA modules for the guest kernel
+mise run vm:nvidia-modules
+
+# 3. Start the gateway with GPU support
+sudo mise run gateway:vm -- --gpu
+
+# 4. Create a GPU sandbox from the published community image
+openshell sandbox create --gpu --from nvidia-gpu
+```
+
+## Version Coupling
+
+The NVIDIA driver version must match across the image and the VM guest
+kernel modules:
+
+| Component | Variable | Default |
+|-----------|----------|---------|
+| Dockerfile userspace | `NVIDIA_DRIVER_VERSION` | `580.159.03` |
+| Image version reference | `sandboxes/nvidia-gpu/versions.env` | `580.159.03` |
+| OpenShell core module build | `NVIDIA_OPEN_VERSION` | `580.159.03` |
+
+A mismatch causes `modprobe` "version magic" errors or nvidia-smi ABI
+failures at sandbox boot time.
+
+## Customization
+
+### Changing the CUDA version
+
+```shell
+docker build \
+  --platform linux/amd64 \
+  --build-arg CUDA_VERSION=12.6.0 \
+  --build-arg UBUNTU_VERSION=22.04 \
+  -t my-gpu-sandbox:latest \
+  ./sandboxes/nvidia-gpu/
+```
+
+To build an arm64 variant locally, use `--platform linux/arm64`. Published
+images include both `linux/amd64` and `linux/arm64` manifests.
+
+### Changing the NVIDIA driver version
+
+Update the image version reference and rebuild matching VM guest modules in
+the OpenShell core repo:
+
+1. `sandboxes/nvidia-gpu/versions.env`
+2. `sandboxes/nvidia-gpu/Dockerfile` ARG `NVIDIA_DRIVER_VERSION`
+3. In the OpenShell core repo, rebuild kernel modules:
+   `NVIDIA_OPEN_VERSION=<version> mise run vm:nvidia-modules`
+
+### Adding packages
+
+Add packages to the `apt-get install` line in the Dockerfile. The image
+must retain `bash`, `kmod`, `iproute2`, and `busybox-static` — the VM
+driver validates these at rootfs preparation time.
+
+## Troubleshooting
+
+| Symptom | Cause | Fix |
+|---------|-------|-----|
+| "No GPU kernel modules found" | Modules not built | `mise run vm:nvidia-modules` |
+| "kmod not found in rootfs" | Image missing kmod package | Add `kmod` to Dockerfile `apt-get install` |
+| `modprobe nvidia` fails | Kernel version mismatch | Rebuild modules after `mise run vm:setup` |
+| nvidia-smi "driver/library mismatch" | Userspace/module version mismatch | Ensure Dockerfile and module versions match |
+| "kernel version mismatch: expected X, got Y" | Guest kernel was rebuilt | Rebuild modules: `mise run vm:nvidia-modules` |
diff --git a/sandboxes/nvidia-gpu/versions.env b/sandboxes/nvidia-gpu/versions.env
@@ -0,0 +1,4 @@
+# NVIDIA driver userspace version for the GPU sandbox image.
+# Must match the NVIDIA_OPEN_VERSION used by OpenShell core's
+# `mise run vm:nvidia-modules` workflow.
+NVIDIA_DRIVER_VERSION=580.159.03
