From b82dc4d0011d76fb1a617862e4cb56a6c9c40445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 22 Dec 2025 15:09:01 +0100 Subject: [PATCH] [CI] Merge docker images across platforms With the current setup, images for different platforms are pushed with the same tag, overriding each other. With this change, instead, images built for different architectures are merged together. [skip ci] --- .github/workflows/Container.yml | 117 +++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 18 deletions(-) diff --git a/.github/workflows/Container.yml b/.github/workflows/Container.yml index 840c008bc7..26ad1dae5b 100644 --- a/.github/workflows/Container.yml +++ b/.github/workflows/Container.yml @@ -1,3 +1,8 @@ +# Based on +# * +# * +# * . + name: Publish Docker container on: @@ -18,12 +23,20 @@ on: branches: - master +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: - push_to_registry: - name: Container for ${{ matrix.platform }} - Julia ${{ matrix.julia }} - CUDA ${{ matrix.cuda }} + build: + name: Build Container for ${{ matrix.platform }} - Julia ${{ matrix.julia }} - CUDA ${{ matrix.cuda }} permissions: contents: read packages: write + attestations: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write strategy: matrix: @@ -46,6 +59,12 @@ jobs: runs-on: ${{ matrix.os }} steps: + # Docker is terrible and doesn't like uppercase image names. + - name: Lowercase image name + run: | + IMAGE_NAME=$(echo ${IMAGE_NAME} | tr A-Z a-z) + echo "IMAGE_NAME=${IMAGE_NAME}" >> "${GITHUB_ENV}" + - name: Check out the repo uses: actions/checkout@v6 @@ -90,37 +109,99 @@ jobs: - name: Log in to registry uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=raw,value=${{ steps.pkg.outputs.name }}-julia${{ matrix.julia }}-cuda${{ steps.cuda.outputs.major }} - type=raw,value=${{ steps.pkg.outputs.name }},enable=${{ matrix.default == true && (github.ref_type == 'tag' || inputs.tag != '') }} - type=raw,value=latest,enable=${{ matrix.default == true && (github.ref_type == 'tag' || (inputs.tag != '' && inputs.mark_as_latest)) }} - type=raw,value=dev,enable=${{ matrix.default == true && github.ref_type == 'branch' && inputs.tag == '' }} - labels: | - org.opencontainers.image.version=${{ steps.pkg.outputs.version }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push image + id: build uses: docker/build-push-action@v6 with: context: . push: true provenance: false # the build fetches the repo again, so provenance tracking is not useful platforms: ${{ matrix.platform }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pkg.outputs.name }}-julia${{ matrix.julia }}-cuda${{ steps.cuda.outputs.major }} + labels: | + org.opencontainers.image.version=${{ steps.pkg.outputs.version }} build-args: | JULIA_VERSION=${{ matrix.julia }} CUDA_VERSION=${{ matrix.cuda }} PACKAGE_SPEC=CUDA#${{ steps.pkg.outputs.ref }} JULIA_CPU_TARGET=${{ steps.cpu_target.outputs.target }} + + - name: Export digest + id: export-digest + run: | + mkdir -p /tmp/digests + digest_name=${{ steps.pkg.outputs.name }}-julia${{ matrix.julia }}-cuda${{ steps.cuda.outputs.major }}-$(echo ${{ matrix.platform }} | tr / -) + echo "digest_name=${digest_name}" | tee "${GITHUB_OUTPUT}" + echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${digest_name}" + + - name: Upload digest + uses: actions/upload-artifact@v6 + with: + name: digests-${{ steps.export-digest.outputs.digest_name }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + if: github.event_name != 'pull_request' + + steps: + # Docker is terrible and doesn't like uppercase image names. + - name: Lowercase image name + run: | + IMAGE_NAME=$(echo ${IMAGE_NAME} | tr A-Z a-z) + echo "IMAGE_NAME=${IMAGE_NAME}" >> "${GITHUB_ENV}" + + - name: Download digests + uses: actions/download-artifact@v7 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Log into the registry. + # https://github.com/docker/login-action + - name: Log into registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create manifest lists for each tag + run: | + for file in /tmp/digests/*; do + TAG=${file##*/} # e.g. "dev-julia1.11-cuda11-linux-amd64" + VARIANT=$(echo ${TAG} | cut -d- -f1-3) # e.g. "dev-julia1.11-cuda11" + # buildx imagetools needs: -t registry/image:tag and sources + SOURCES="" + for f in /tmp/digests/${VARIANT}-*; do + # each f contains a digest per arch + DIG=$(cat "${f}") + PLATFORM=${f##*/} # e.g. "docs-linux-amd64" + SOURCES="${SOURCES} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${DIG}" + done + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${VARIANT} \ + ${SOURCES} + done