From 73fa9afa31c76b1841944cbcc3b22ad1c9c51298 Mon Sep 17 00:00:00 2001 From: Colm Talbot Date: Wed, 1 Apr 2026 15:14:13 -0400 Subject: [PATCH 1/5] BLD: add basic testing to containers --- .github/workflows/basic-install.yml | 24 ++---- .github/workflows/build-containers.yml | 113 +++++++++++++++++++++---- test/ci_test_imports.sh | 20 +++++ 3 files changed, 121 insertions(+), 36 deletions(-) create mode 100644 test/ci_test_imports.sh diff --git a/.github/workflows/basic-install.yml b/.github/workflows/basic-install.yml index 1919042cc..7d8d7a3fe 100644 --- a/.github/workflows/basic-install.yml +++ b/.github/workflows/basic-install.yml @@ -37,23 +37,9 @@ jobs: - name: List environment run: python -m pip list installed - name: Test imports + run: bash test/ci_test_imports.sh + - name: Test entry points run: | - python -c "import bilby" - python -c "import bilby.bilby_mcmc" - python -c "import bilby.core" - python -c "import bilby.core.prior" - python -c "import bilby.core.sampler" - python -c "import bilby.core.utils" - python -c "import bilby.gw" - python -c "import bilby.gw.detector" - python -c "import bilby.gw.eos" - python -c "import bilby.gw.likelihood" - python -c "import bilby.gw.sampler" - python -c "import bilby.hyper" - python -c "import cli_bilby" - python test/import_test.py - # - if: ${{ matrix.os != "windows-latest" }} - # run: | - # for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do - # ${script} --help; - # done + for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do + ${script} --help; + done diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 3bfd7daf2..10c410a5e 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -2,17 +2,14 @@ name: Build images on: schedule: - - cron: "15 0 * * *" + - cron: "15 0 * * *" workflow_dispatch: jobs: build-container: if: ${{ github.repository == 'bilby-dev/bilby' }} permissions: - attestations: write contents: read - id-token: write - packages: write runs-on: ubuntu-latest strategy: fail-fast: false @@ -20,6 +17,7 @@ jobs: python-version: ["11", "12", "13"] env: LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }} + IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar steps: - uses: actions/checkout@v4 @@ -30,33 +28,114 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" df . -h - - name: Login to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push Docker image - id: push + - name: Build Docker image archive uses: docker/build-push-action@v6 with: context: . build-args: | python_minor_version=${{ matrix.python-version }} ENV_FILE=containers/environment.yml - push: true + push: false file: containers/Dockerfile tags: ${{ env.LABEL }}:latest - cache-from: type=registry,ref=${{ env.LABEL }}:buildcache - cache-to: type=registry,ref=${{ env.LABEL }}:buildcache,mode=max - + outputs: type=docker,dest=/tmp/${{ env.IMAGE_ARCHIVE }} + cache-from: type=gha,scope=bilby-python3${{ matrix.python-version }} + cache-to: type=gha,scope=bilby-python3${{ matrix.python-version }},mode=max + + - name: Upload Docker image archive + uses: actions/upload-artifact@v4 + with: + name: bilby-python3${{ matrix.python-version }}-image + path: /tmp/${{ env.IMAGE_ARCHIVE }} + if-no-files-found: error + + test-container: + if: ${{ github.repository == 'bilby-dev/bilby' }} + needs: build-container + permissions: + contents: read + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["11", "12", "13"] + env: + LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }} + IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar + steps: + - uses: actions/checkout@v4 + + - name: Download Docker image archive + uses: actions/download-artifact@v4 + with: + name: bilby-python3${{ matrix.python-version }}-image + path: /tmp + + - name: Load Docker image + run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }} + + - name: Smoke test and import checks + run: | + docker run --rm \ + -v "$PWD:/workspaces/bilby" \ + -w /workspaces/bilby \ + ${{ env.LABEL }}:latest \ + bash -lc ' + set -e + python -m pip install -e . + bilby_result --help + bash test/ci_test_imports.sh + for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do + ${script} --help; + done + ' + + push-container: + if: ${{ github.repository == 'bilby-dev/bilby' }} + needs: test-container + permissions: + attestations: write + contents: read + id-token: write + packages: write + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["11", "12", "13"] + env: + LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }} + IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar + steps: + - name: Download Docker image archive + uses: actions/download-artifact@v4 + with: + name: bilby-python3${{ matrix.python-version }}-image + path: /tmp + + - name: Load Docker image + run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }} + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push Docker image + id: push + run: | + docker push ${{ env.LABEL }}:latest + digest=$(docker image inspect --format='{{index .RepoDigests 0}}' ${{ env.LABEL }}:latest | sed 's/.*@//') + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + - name: Generate artifact attestation uses: actions/attest-build-provenance@v3 with: diff --git a/test/ci_test_imports.sh b/test/ci_test_imports.sh new file mode 100644 index 000000000..a62d03edd --- /dev/null +++ b/test/ci_test_imports.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# test that the various imports within the package works and can be reused across workflows + +set -euo pipefail + +python -c "import bilby" +python -c "import bilby.bilby_mcmc" +python -c "import bilby.core" +python -c "import bilby.core.prior" +python -c "import bilby.core.sampler" +python -c "import bilby.core.utils" +python -c "import bilby.gw" +python -c "import bilby.gw.detector" +python -c "import bilby.gw.eos" +python -c "import bilby.gw.likelihood" +python -c "import bilby.gw.sampler" +python -c "import bilby.hyper" +python -c "import cli_bilby" +python test/import_test.py From df15c8a547e9c1cced9dec4941e564fb7af2e91d Mon Sep 17 00:00:00 2001 From: Colm Talbot Date: Wed, 1 Apr 2026 15:44:52 -0400 Subject: [PATCH 2/5] BLD: try cleaning source from docker build --- containers/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/containers/Dockerfile b/containers/Dockerfile index 21935d2e0..67731a6bc 100644 --- a/containers/Dockerfile +++ b/containers/Dockerfile @@ -28,6 +28,7 @@ RUN wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_lin ############################# FROM os-deps AS build-env +WORKDIR /bldenv ARG ENV_FILE=environment.yml COPY ${ENV_FILE} env.yml ARG python_major_version=3 @@ -40,6 +41,7 @@ RUN mamba env create -f env.yml -n ${conda_env} RUN echo "source activate ${conda_env}" > ~/.bashrc ENV PATH=/opt/conda/envs/${conda_env}/bin:$PATH RUN /bin/bash -c "source activate ${conda_env}" +RUN cd / && rm -rf /bldenv RUN mamba clean --all ############################# From 7c6a11db8dddb7f3d067c8cf887d05158257f059 Mon Sep 17 00:00:00 2001 From: Colm Talbot Date: Wed, 1 Apr 2026 16:17:03 -0400 Subject: [PATCH 3/5] BLD: add setuptools_scm catch for dockerfile --- containers/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/containers/Dockerfile b/containers/Dockerfile index 67731a6bc..513f1a0a8 100644 --- a/containers/Dockerfile +++ b/containers/Dockerfile @@ -50,5 +50,7 @@ RUN mamba clean --all FROM build-env AS final COPY --from=data /roq_basis /roq_basis +# otherwise CI jobs don't like installing bilby +ENV SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER=1 RUN mamba info RUN python --version From f13a9142b474317ddb03ca63fc4be27a40f41192 Mon Sep 17 00:00:00 2001 From: Colm Talbot Date: Wed, 1 Apr 2026 18:40:20 -0400 Subject: [PATCH 4/5] Pin arviz version in test environment --- containers/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/environment.yml b/containers/environment.yml index 969155b75..e9cb42f04 100644 --- a/containers/environment.yml +++ b/containers/environment.yml @@ -21,7 +21,7 @@ dependencies: - pytest-cov - pytest-requires - pytest-rerunfailures - - arviz + - arviz<1 # arviz removed InferenceData in version 1 - parameterized - scikit-image - celerite From 8ad24e944856f7b18706d4f3b35b5df0c239d191 Mon Sep 17 00:00:00 2001 From: Colm Talbot Date: Wed, 1 Apr 2026 18:43:05 -0400 Subject: [PATCH 5/5] Fix setuptools dubious owner error in unit test job --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 674a4cd68..3c2eaba5d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -20,6 +20,7 @@ concurrency: env: CONDA_PATH: /opt/conda/ BILBY_ALLOW_PARAMETERS_AS_STATE: FALSE + SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER: 1 jobs: build: