fix(ci,publish): harden publish flow and cache nemoclaw wheel builds (NVIDIA#55)

Author: drew

.github/workflows/publish.yml

@@ -9,6 +9,10 @@ permissions:
   contents: read
   packages: write
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   # ---------------------------------------------------------------------------
   # Build container images to GHCR (same as E2E pipeline)
@@ -28,21 +32,12 @@ jobs:
     with:
       component: cluster
 
-  # ---------------------------------------------------------------------------
-  # Run E2E tests against the built images
-  # ---------------------------------------------------------------------------
-  e2e:
-    needs: [build-server, build-sandbox, build-cluster]
-    uses: ./.github/workflows/e2e-test.yml
-    with:
-      image-tag: ${{ github.sha }}
-
   # ---------------------------------------------------------------------------
   # Publish multi-arch container images to ECR
   # ---------------------------------------------------------------------------
   publish-containers:
     name: Publish Containers
-    needs: [e2e]
+    needs: [build-server, build-sandbox, build-cluster]
     runs-on: build-amd64
     timeout-minutes: 120
     container:
@@ -60,21 +55,6 @@ jobs:
       AWS_DEFAULT_REGION: us-west-2
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # Full history needed for setuptools_scm version
-
-      - name: Install Python dependencies
-        run: uv sync --frozen
-
-      - name: Compute version
-        id: version
-        run: |
-          VERSION_DOCKER=$(uv run python build/scripts/release.py get-version --docker)
-          echo "docker=$VERSION_DOCKER" >> "$GITHUB_OUTPUT"
-          echo "Docker image version: $VERSION_DOCKER"
-
-      - name: Set version in source files
-        run: mise run --no-prepare version:set
 
       - name: Log in to GHCR
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
@@ -90,26 +70,148 @@ jobs:
           DOCKER_BUILDER: navigator
           IMAGE_TAG: dev
           TAG_LATEST: "true"
-          EXTRA_DOCKER_TAGS: ${{ steps.version.outputs.docker }}
         run: mise run --no-prepare docker:publish:cluster:multiarch
 
   # ---------------------------------------------------------------------------
-  # Publish Python packages (placeholder)
+  # Build Python wheels and stage them in S3
   # ---------------------------------------------------------------------------
-  publish-python:
-    name: Publish Python
-    needs: [e2e]
+  build-python-wheels:
+    name: Stage Python Wheels
+    needs: [build-server, build-sandbox, build-cluster]
     runs-on: build-amd64
-    if: false  # TODO: Enable when Python packaging is ready
+    timeout-minutes: 120
+    outputs:
+      wheel_version: ${{ steps.version.outputs.wheel_version }}
+      s3_prefix: ${{ steps.upload.outputs.s3_prefix }}
     container:
       image: ghcr.io/nvidia/nv-agent-env/ci:latest
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+      options: --privileged
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+    env:
+      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-west-2
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Placeholder
-        run: echo "Python packaging not yet implemented"
+      - name: Log in to GHCR
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Set up Docker Buildx
+        uses: ./.github/actions/setup-buildx
+
+      - name: Mark workspace safe for git
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Fetch tags
+        run: git fetch --tags --force
+
+      - name: Compute Python version
+        id: version
+        run: |
+          set -euo pipefail
+          WHEEL_VERSION=$(uv run python build/scripts/release.py get-version --python)
+          echo "wheel_version=${WHEEL_VERSION}" >> "$GITHUB_OUTPUT"
+
+      - name: Build Python wheels
+        run: |
+          set -euo pipefail
+          WHEEL_VERSION="${{ steps.version.outputs.wheel_version }}"
+          CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
+          NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:multiarch
+          NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:macos
+          ls -la target/wheels/*.whl
+
+      - name: Upload wheels to S3
+        id: upload
+        run: |
+          set -euo pipefail
+          WHEEL_VERSION="${{ steps.version.outputs.wheel_version }}"
+          S3_PREFIX="nemoclaw/${WHEEL_VERSION}"
+          aws s3 cp target/wheels/ "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" --recursive --exclude "*" --include "*.whl"
+          aws s3 ls "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/"
+          echo "s3_prefix=${S3_PREFIX}" >> "$GITHUB_OUTPUT"
+
+  # ---------------------------------------------------------------------------
+  # Publish Python wheels to Artifactory from S3 staging
+  # ---------------------------------------------------------------------------
+  publish-python:
+    name: Publish Python
+    needs: [build-python-wheels]
+    runs-on: [self-hosted, nv]
+    timeout-minutes: 10
+    env:
+      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts
+      NAV_PYPI_REPOSITORY_URL: https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi-local
+      NAV_PYPI_USERNAME: ${{ secrets.NAV_PYPI_USERNAME }}
+      NAV_PYPI_PASSWORD: ${{ secrets.NAV_PYPI_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-west-2
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install publish dependencies
+      run: |
+        set -euo pipefail
+        python -m pip install --upgrade pip uv
+        
+        if ! command -v aws >/dev/null 2>&1; then
+          ARCH="$(uname -m)"
+          case "$ARCH" in
+            x86_64|amd64) AWSCLI_ARCH="x86_64" ;;
+            aarch64|arm64) AWSCLI_ARCH="aarch64" ;;
+            *)
+              echo "Unsupported architecture for AWS CLI installer: $ARCH" >&2
+              exit 1
+              ;;
+          esac
+        
+          rm -rf aws awscliv2.zip
+          curl --fail --silent --show-error --location \
+            "https://awscli.amazonaws.com/awscli-exe-linux-${AWSCLI_ARCH}.zip" \
+            --output awscliv2.zip
+          unzip -q awscliv2.zip
+          ./aws/install --install-dir "$HOME/.local/aws-cli" --bin-dir "$HOME/.local/bin" --update
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          export PATH="$HOME/.local/bin:$PATH"
+        fi
+        
+        aws --version
+        uv --version
+
+    - name: List and download versioned wheels from S3
+      run: |
+        set -euo pipefail
+        WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}"
+        S3_PREFIX="${{ needs.build-python-wheels.outputs.s3_prefix }}"
+        OBJECT_COUNT=$(aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "length(Contents)" --output text)
+        if [ "$OBJECT_COUNT" = "None" ] || [ "$OBJECT_COUNT" = "0" ]; then
+          echo "No wheel artifacts found for ${WHEEL_VERSION} at s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" >&2
+          exit 1
+        fi
+        aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "Contents[].Key" --output text
+        mkdir -p target/wheels
+        aws s3 cp "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" target/wheels/ --recursive --exclude "*" --include "*.whl"
+        ls -la target/wheels/*.whl
+
+    - name: Publish wheels to Artifactory
+      run: |
+        set -euo pipefail
+        WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}"
+        uv run python build/scripts/release.py python-publish --version "$WHEEL_VERSION"

CONTRIBUTING.md

@@ -327,14 +327,16 @@ automatically, so you generally do not need to generate stubs manually.
 ### Publishing
 
 Versions are derived from git tags using `setuptools_scm`. No version bumps need to be committed.
+Python wheel builds inject version at build time via
+`NAVIGATOR_CARGO_VERSION` (Cargo/SemVer), applied inside wheel-builder Docker
+layers, so publish flows do not edit `Cargo.toml`/`Cargo.lock` in the working
+tree.
 
 **Version commands:**
 
 ```bash
 mise run version:print             # Show computed versions (python, cargo, docker)
 mise run version:print -- --cargo  # Show cargo version only
-mise run version:set               # Update Cargo.toml with git-derived version (or specified with --version)
-mise run version:reset             # Restore Cargo.toml to git state
 ```
 
 **Publishing credentials (one-time setup):**
@@ -345,18 +347,24 @@ NAV_PYPI_USERNAME=$USER
 NAV_PYPI_PASSWORD=$ARTIFACTORY_PASSWORD" >> .env
 ```
 
-Docker publishing in CI uses AWS credentials for ECR. Python publishing uses
-`NAV_PYPI_*` credentials for Artifactory.
+Docker publishing in CI uses AWS credentials for ECR. Python publishing uses a
+two-stage flow: wheels are uploaded to S3, then an internal-network runner
+publishes them to Artifactory with `NAV_PYPI_*` credentials.
 
 **Main branch publish (CI):**
 
 - Publishes Docker multiarch images to ECR as `:dev`, `:latest`, and a versioned dev tag.
+- Builds Linux + macOS (arm64) Python wheels and uploads them to
+  `s3://navigator-pypi-artifacts/navigator/<wheel-version>/`.
+- Runs a publish job on the `nv` runner to list that version prefix, download
+  the wheels, and publish them to Artifactory.
 
 **Tag release publish (CI):**
 
 - Push a semver tag (`vX.Y.Z`) to trigger release jobs.
 - CI publishes Docker multiarch images to ECR as `:X.Y.Z` (no `:latest`).
-- CI publishes Linux + macOS (arm64) Python wheels to Artifactory and creates GitHub release notes.
+- CI stages Linux + macOS (arm64) Python wheels in S3 and publishes to
+  Artifactory from the `nv` runner.
 
 **Tagging a release:**
 

architecture/build-containers.md

@@ -121,8 +121,8 @@ A k3s image with bundled Helm charts and Kubernetes manifests for single-contain
 
 Two Dockerfiles produce Python wheels for the CLI package distribution. These are not deployed as running containers.
 
-- **`Dockerfile.python-wheels`** -- Builds Linux amd64/arm64 wheels using Maturin. Installs Rust toolchain and cross-compilation targets. Output stage is `scratch` with only the `.whl` files.
-- **`Dockerfile.python-wheels-macos`** -- Builds macOS arm64 wheels using osxcross (cross-compiling from Linux). Uses `crazymax/osxcross:latest` as the cross-toolchain source. The `OSXCROSS_IMAGE` build arg allows using a mirrored registry image instead of Docker Hub.
+- **`Dockerfile.python-wheels`** -- Builds Linux amd64/arm64 wheels using Maturin with a two-pass Rust build (dependency prebuild + final wheel build), BuildKit cache mounts for cargo registry/git/target and sccache, and `cross-build.sh` for conditional cross-toolchain installation. The final build step patches workspace version inside the container layer from `NAVIGATOR_CARGO_VERSION` (computed before Docker build), preserving cacheable dependency layers and avoiding dirty working-tree edits. Output stage is `scratch` with only the `.whl` files.
+- **`Dockerfile.python-wheels-macos`** -- Builds macOS arm64 wheels using osxcross (cross-compiling from Linux) with the same two-pass dependency caching pattern and cargo cache mounts. Version injection uses the same in-container workspace-version patch from `NAVIGATOR_CARGO_VERSION`, avoiding host-side file edits that break Docker layer caching. Uses `crazymax/osxcross:latest` as the cross-toolchain source. The `OSXCROSS_IMAGE` build arg allows using a mirrored registry image instead of Docker Hub.
 
 ### CI Runner Image (`navigator-ci`)
 
@@ -135,7 +135,7 @@ A pre-built Ubuntu 24.04 image for CI pipeline jobs, defined in `deploy/docker/D
 | Docker CLI + buildx plugin | DinD-based image build/publish jobs |
 | AWS CLI v2 | ECR authentication and image publishing |
 | kubectl, helm, protoc | Kubernetes operations, chart packaging, proto compilation |
-| mise | Task runner with Rust, Python, and cargo-edit toolchains |
+| mise | Task runner with Rust and Python toolchains |
 | uv | Python package management (installed from Astral's installer to avoid GitHub API rate limits) |
 | sccache | Rust compilation cache (amd64 only; skipped on arm64) |
 | socat | Docker socket forwarding in sandbox e2e tests |
@@ -385,6 +385,7 @@ Container builds use Docker BuildKit with local cache directories:
 
 - `build/scripts/docker-build-component.sh` stores per-component caches in `.cache/buildkit/<component>`.
 - `build/scripts/docker-build-cluster.sh` stores the cluster image cache in `.cache/buildkit/cluster`.
+- `mise run python:build:multiarch` stores per-platform wheel caches in `.cache/buildkit/python-wheels/<platform>` for local builds when using a `docker-container` buildx driver.
 - Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry and target directories, keyed by image name and `TARGETARCH`, with `sharing=locked` to prevent concurrent cache corruption in parallel CI builds.
 - When the active buildx driver is `docker` (not `docker-container`), local cache import/export flags are skipped automatically because the docker driver cannot export local caches. In CI, cache export is also skipped.
 - For local single-arch builds, the scripts auto-select a builder with the native `docker` driver (matching the active Docker context) so images land directly in the Docker image store without slow tarball export.
@@ -450,6 +451,13 @@ mise run docker:publish:cluster:multiarch
 mise run publish:main
 ```
 
+GitHub Actions stages Python wheels in S3 before final publication to
+Artifactory:
+
+- Wheels are uploaded to `s3://navigator-pypi-artifacts/navigator/<wheel-version>/`.
+- A follow-up job on the `nv` runner lists that version prefix, downloads the
+  wheels, and publishes them to Artifactory.
+
 ### Auto-Deployed Components in Cluster
 
 When the cluster container starts, k3s automatically deploys these HelmChart CRs from `/var/lib/rancher/k3s/server/manifests/`:

build/publish.toml

@@ -6,9 +6,8 @@ run = """
 #!/usr/bin/env bash
 set -euo pipefail
 VERSION=$(uv run python build/scripts/release.py get-version --python)
-mise run version:set
-trap "mise run version:reset" EXIT
-NAVIGATOR_VERSION="$VERSION" mise run python:build:all
+CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:all
 uv run python build/scripts/release.py python-publish --version "$VERSION"
 """
 
@@ -18,9 +17,8 @@ run = """
 #!/usr/bin/env bash
 set -euo pipefail
 VERSION=$(uv run python build/scripts/release.py get-version --python)
-mise run version:set
-trap "mise run version:reset" EXIT
-NAVIGATOR_VERSION="$VERSION" mise run python:build:macos
+CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:macos
 uv run python build/scripts/release.py python-publish --version "$VERSION" --wheel-glob "*macosx*arm64.whl"
 """
 
@@ -31,25 +29,23 @@ run = """
 set -euo pipefail
 VERSION_DOCKER=$(uv run python build/scripts/release.py get-version --docker)
 VERSION_PYTHON=$(uv run python build/scripts/release.py get-version --python)
-mise run version:set
-trap "mise run version:reset" EXIT
+CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
 IMAGE_TAG=dev TAG_LATEST=true EXTRA_DOCKER_TAGS="$VERSION_DOCKER" mise run docker:publish:cluster:multiarch
-NAVIGATOR_VERSION="$VERSION_PYTHON" mise run python:build:multiarch
-NAVIGATOR_VERSION="$VERSION_PYTHON" mise run python:build:macos
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:multiarch
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:macos
 uv run python build/scripts/release.py python-publish --version "$VERSION_PYTHON"
 """
 
 ["publish:tag"]
-description = "Tag release publish: versioned Docker to ECR and Python to Artifactory"
+description = "Tag release publish: versioned Docker to ECR and Python to GitHub Packages"
 run = """
 #!/usr/bin/env bash
 set -euo pipefail
 VERSION_DOCKER=$(uv run python build/scripts/release.py get-version --docker)
 VERSION_PYTHON=$(uv run python build/scripts/release.py get-version --python)
-mise run version:set
-trap "mise run version:reset" EXIT
+CARGO_VERSION=$(uv run python build/scripts/release.py get-version --cargo)
 IMAGE_TAG="$VERSION_DOCKER" TAG_LATEST=false mise run docker:publish:cluster:multiarch
-NAVIGATOR_VERSION="$VERSION_PYTHON" mise run python:build:multiarch
-NAVIGATOR_VERSION="$VERSION_PYTHON" mise run python:build:macos
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:multiarch
+NAVIGATOR_CARGO_VERSION="$CARGO_VERSION" mise run python:build:macos
 uv run python build/scripts/release.py python-publish --version "$VERSION_PYTHON"
 """