Fix and test containers

.github/workflows/basic-install.yml

@@ -37,23 +37,9 @@ jobs:
       - name: List environment
         run: python -m pip list installed
       - name: Test imports
+        run: bash test/ci_test_imports.sh
+      - name: Test entry points
         run: |
-          python -c "import bilby"
-          python -c "import bilby.bilby_mcmc"
-          python -c "import bilby.core"
-          python -c "import bilby.core.prior"
-          python -c "import bilby.core.sampler"
-          python -c "import bilby.core.utils"
-          python -c "import bilby.gw"
-          python -c "import bilby.gw.detector"
-          python -c "import bilby.gw.eos"
-          python -c "import bilby.gw.likelihood"
-          python -c "import bilby.gw.sampler"
-          python -c "import bilby.hyper"
-          python -c "import cli_bilby"
-          python test/import_test.py
-      # - if: ${{ matrix.os != "windows-latest" }}
-      #   run: |
-      #     for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
-      #         ${script} --help;
-      #     done
+          for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
+              ${script} --help;
+          done

.github/workflows/build-containers.yml

@@ -2,24 +2,22 @@ name: Build images
 
 on:
   schedule:
-    - cron: "15 0 * * *" 
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 jobs:
   build-container:
     if: ${{ github.repository == 'bilby-dev/bilby' }}
     permissions:
-      attestations: write
       contents: read
-      id-token: write
-      packages: write
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["11", "12", "13"]
     env:
       LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
+      IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
     steps:
       - uses: actions/checkout@v4
 
@@ -30,33 +28,114 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           df . -h
 
-      - name: Login to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Docker image
-        id: push
+      - name: Build Docker image archive
         uses: docker/build-push-action@v6
         with:
           context: .
           build-args: |
             python_minor_version=${{ matrix.python-version }}
             ENV_FILE=containers/environment.yml
-          push: true
+          push: false
           file: containers/Dockerfile
           tags: ${{ env.LABEL }}:latest
-          cache-from: type=registry,ref=${{ env.LABEL }}:buildcache
-          cache-to: type=registry,ref=${{ env.LABEL }}:buildcache,mode=max
-
+          outputs: type=docker,dest=/tmp/${{ env.IMAGE_ARCHIVE }}
+          cache-from: type=gha,scope=bilby-python3${{ matrix.python-version }}
+          cache-to: type=gha,scope=bilby-python3${{ matrix.python-version }},mode=max
+
+      - name: Upload Docker image archive
+        uses: actions/upload-artifact@v4
+        with:
+          name: bilby-python3${{ matrix.python-version }}-image
+          path: /tmp/${{ env.IMAGE_ARCHIVE }}
+          if-no-files-found: error
+
+  test-container:
+    if: ${{ github.repository == 'bilby-dev/bilby' }}
+    needs: build-container
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["11", "12", "13"]
+    env:
+      LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
+      IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Docker image archive
+        uses: actions/download-artifact@v4
+        with:
+          name: bilby-python3${{ matrix.python-version }}-image
+          path: /tmp
+
+      - name: Load Docker image
+        run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }}
+
+      - name: Smoke test and import checks
+        run: |
+          docker run --rm \
+            -v "$PWD:/workspaces/bilby" \
+            -w /workspaces/bilby \
+            ${{ env.LABEL }}:latest \
+            bash -lc '
+              set -e
+              python -m pip install -e .
+              bilby_result --help
+              bash test/ci_test_imports.sh
+              for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
+                  ${script} --help;
+              done
+            '
+
+  push-container:
+    if: ${{ github.repository == 'bilby-dev/bilby' }}
+    needs: test-container
+    permissions:
+      attestations: write
+      contents: read
+      id-token: write
+      packages: write
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["11", "12", "13"]
+    env:
+      LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
+      IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
+    steps:
+      - name: Download Docker image archive
+        uses: actions/download-artifact@v4
+        with:
+          name: bilby-python3${{ matrix.python-version }}-image
+          path: /tmp
+
+      - name: Load Docker image
+        run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }}
+
+      - name: Login to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push Docker image
+        id: push
+        run: |
+          docker push ${{ env.LABEL }}:latest
+          digest=$(docker image inspect --format='{{index .RepoDigests 0}}' ${{ env.LABEL }}:latest | sed 's/.*@//')
+          echo "digest=${digest}" >> "$GITHUB_OUTPUT"
+
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v3
         with:

.github/workflows/unit-tests.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   CONDA_PATH: /opt/conda/
   BILBY_ALLOW_PARAMETERS_AS_STATE: FALSE
+  SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER: 1
 
 jobs:
   build:

containers/Dockerfile

@@ -28,6 +28,7 @@ RUN  wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_lin
 #############################
 
 FROM os-deps AS build-env
+WORKDIR /bldenv
 ARG ENV_FILE=environment.yml
 COPY ${ENV_FILE} env.yml
 ARG python_major_version=3
@@ -40,6 +41,7 @@ RUN mamba env create -f env.yml -n ${conda_env}
 RUN echo "source activate ${conda_env}" > ~/.bashrc
 ENV PATH=/opt/conda/envs/${conda_env}/bin:$PATH
 RUN /bin/bash -c "source activate ${conda_env}"
+RUN cd / && rm -rf /bldenv
 RUN mamba clean --all
 
 #############################
@@ -48,5 +50,7 @@ RUN mamba clean --all
 
 FROM build-env AS final
 COPY --from=data /roq_basis /roq_basis
+# otherwise CI jobs don't like installing bilby
+ENV SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER=1
 RUN mamba info
 RUN python --version

containers/environment.yml

@@ -21,7 +21,7 @@ dependencies:
   - pytest-cov
   - pytest-requires
   - pytest-rerunfailures
-  - arviz
+  - arviz<1  # arviz removed InferenceData in version 1
   - parameterized
   - scikit-image
   - celerite

test/ci_test_imports.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# test that the various imports within the package works and can be reused across workflows
+
+set -euo pipefail
+
+python -c "import bilby"
+python -c "import bilby.bilby_mcmc"
+python -c "import bilby.core"
+python -c "import bilby.core.prior"
+python -c "import bilby.core.sampler"
+python -c "import bilby.core.utils"
+python -c "import bilby.gw"
+python -c "import bilby.gw.detector"
+python -c "import bilby.gw.eos"
+python -c "import bilby.gw.likelihood"
+python -c "import bilby.gw.sampler"
+python -c "import bilby.hyper"
+python -c "import cli_bilby"
+python test/import_test.py