From b66fafa743b7e974cd51de7f9ab74a1023a25085 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:16:38 +0100 Subject: [PATCH 01/11] Added a CI workflow that tries to pip-install in three different ways. Then, tries to run the lpc3d executable with --help. Dependencies are used from EESSI --- .github/workflows/pip_install.yml | 99 +++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/pip_install.yml diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml new file mode 100644 index 0000000..3754c7d --- /dev/null +++ b/.github/workflows/pip_install.yml @@ -0,0 +1,99 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Test installation of LPC3D with 'pip install' +on: [push, pull_request, workflow_dispatch] +permissions: read-all +jobs: + test_pip_install: + # ubuntu <= 20.04 is required for python 3.6 + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # We can test against multiple pystencil-versions, but currently, there is only one in EESSI + # See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow + pystencil-version: ['1.3.4-gfbf-2023b'] + steps: + - name: Check out repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + fetch-depth: 0 + - name: Setup EESSI + uses: eessi/github-action-eessi@v3 + with: + eessi_stack_version: "2023.06" + - name: build LPC3D distribution + # This will test if we can create a distribution tarball/wheel/etc + # Could be used to e.g. put the package on pipy + # See https://setuptools.pypa.io/en/latest/userguide/quickstart.html + run: | + # Create virtualenv for this step + python -m venv build_dist_env + source build_dist_env/bin/activate + + # Install build + pip install --upgrade build + + # Build the source distribution + python -m build + + # Deactivate the virtualenv + deactivate + - name: LPC3D against EESSI deps from distribution tarball + # This will test if we can pip install from the distribution tarball we just made + run: | + # Create virtualenv for this step + python -m venv install_from_dist_env + source install_from_dist_env/bin/activate + + # Install from distribution tarball + python -m pip install dist/lpc3d*.tar.gz + + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + + # Check if the program runs + lpc3d --help + + # Deactivate virtualenv + deactivate + - name: LPC3D against EESSI deps with pip install . + run: | + # Create virtualenv for this step + python -m venv install_from_clone + source install_from_clone/bin/activate + + # Install from clone + pip install . + + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + + # Check if program runs + lpc3d --help + + # Deactivate virtualenv + deactivate + - name: LPC3D against EESSI deps with pip install git+https + run: | + # Create virtualenv for this step + python -m venv install_from_git + source install_from_git/bin/activate + + # Install from clone + pip install "git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git@$GITHUB_SHA" + + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + + # Check if program runs + lpc3d --help + + # Deactivate virtualenv + deactivate From 47677550c1c55a19b78ff3b30201694fe5dee67a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:40:46 +0100 Subject: [PATCH 02/11] Fix testing matrix with all deps in an include --- .github/workflows/pip_install.yml | 57 +++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 3754c7d..4f442aa 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -9,9 +9,18 @@ jobs: strategy: fail-fast: false matrix: - # We can test against multiple pystencil-versions, but currently, there is only one in EESSI - # See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow - pystencil-version: ['1.3.4-gfbf-2023b'] + include: + # We can test against multiple combinations of versions. + # We currently include one set of versions for all dependencies, based on the 2023b toolchain + # See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow + # on how to use matrix: and include: + - pystencil-version: '1.3.4-gfbf-2023b' + python-version: '3.11.5-GCCcore-13.2.0' + matplotlib-version: '3.8.2-gfbf-2023b' + ipython-version: '8.17.2-GCCcore-13.2.0' + # There is no numba in 2023b... we should include this eventually + # For now, the pip install will just pull it in as dependency + # numba-version: '' steps: - name: Check out repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -27,6 +36,11 @@ jobs: # Could be used to e.g. put the package on pipy # See https://setuptools.pypa.io/en/latest/userguide/quickstart.html run: | + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load python/${{ python-version }} + # Create virtualenv for this step python -m venv build_dist_env source build_dist_env/bin/activate @@ -42,17 +56,20 @@ jobs: - name: LPC3D against EESSI deps from distribution tarball # This will test if we can pip install from the distribution tarball we just made run: | - # Create virtualenv for this step + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + module load matplotlib/${{ matplotlib-version }} + module load IPython/${{ ipython-version }} + + # Create virtualenv for this step python -m venv install_from_dist_env source install_from_dist_env/bin/activate # Install from distribution tarball python -m pip install dist/lpc3d*.tar.gz - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils - # Load pystencils to make it available as dependency - module load pystencils/${{ matrix.pystencil-version }} # Check if the program runs lpc3d --help @@ -61,6 +78,13 @@ jobs: deactivate - name: LPC3D against EESSI deps with pip install . run: | + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + module load matplotlib/${{ matplotlib-version }} + module load IPython/${{ ipython-version }} + # Create virtualenv for this step python -m venv install_from_clone source install_from_clone/bin/activate @@ -68,11 +92,6 @@ jobs: # Install from clone pip install . - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils - # Load pystencils to make it available as dependency - module load pystencils/${{ matrix.pystencil-version }} - # Check if program runs lpc3d --help @@ -80,6 +99,13 @@ jobs: deactivate - name: LPC3D against EESSI deps with pip install git+https run: | + # Check which pystencils versions are available, for easier debugging in case of failing CI + module avail pystencils + # Load pystencils to make it available as dependency + module load pystencils/${{ matrix.pystencil-version }} + module load matplotlib/${{ matplotlib-version }} + module load IPython/${{ ipython-version }} + # Create virtualenv for this step python -m venv install_from_git source install_from_git/bin/activate @@ -87,11 +113,6 @@ jobs: # Install from clone pip install "git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git@$GITHUB_SHA" - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils - # Load pystencils to make it available as dependency - module load pystencils/${{ matrix.pystencil-version }} - # Check if program runs lpc3d --help From 4cff0af744d4101d391d2b305bf8c2c1b79cdf6c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:48:04 +0100 Subject: [PATCH 03/11] Add white line after run section --- .github/workflows/pip_install.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 4f442aa..8152a1f 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -53,6 +53,7 @@ jobs: # Deactivate the virtualenv deactivate + - name: LPC3D against EESSI deps from distribution tarball # This will test if we can pip install from the distribution tarball we just made run: | @@ -76,6 +77,7 @@ jobs: # Deactivate virtualenv deactivate + - name: LPC3D against EESSI deps with pip install . run: | # Check which pystencils versions are available, for easier debugging in case of failing CI @@ -97,6 +99,7 @@ jobs: # Deactivate virtualenv deactivate + - name: LPC3D against EESSI deps with pip install git+https run: | # Check which pystencils versions are available, for easier debugging in case of failing CI @@ -118,3 +121,4 @@ jobs: # Deactivate virtualenv deactivate + From 396aa053f8057c969af4cd5d244490626cf3e68d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:49:19 +0100 Subject: [PATCH 04/11] check if deactivate is the issue --- .github/workflows/pip_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 8152a1f..a708391 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -52,7 +52,7 @@ jobs: python -m build # Deactivate the virtualenv - deactivate + # deactivate - name: LPC3D against EESSI deps from distribution tarball # This will test if we can pip install from the distribution tarball we just made From ca02d6095728c20381f5220668e383ed3ef13559 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:54:02 +0100 Subject: [PATCH 05/11] Fix indentation error --- .github/workflows/pip_install.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index a708391..29a0760 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -23,14 +23,16 @@ jobs: # numba-version: '' steps: - name: Check out repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false fetch-depth: 0 + - name: Setup EESSI uses: eessi/github-action-eessi@v3 with: eessi_stack_version: "2023.06" + - name: build LPC3D distribution # This will test if we can create a distribution tarball/wheel/etc # Could be used to e.g. put the package on pipy @@ -52,7 +54,7 @@ jobs: python -m build # Deactivate the virtualenv - # deactivate + deactivate - name: LPC3D against EESSI deps from distribution tarball # This will test if we can pip install from the distribution tarball we just made @@ -64,7 +66,7 @@ jobs: module load matplotlib/${{ matplotlib-version }} module load IPython/${{ ipython-version }} - # Create virtualenv for this step + # Create virtualenv for this step python -m venv install_from_dist_env source install_from_dist_env/bin/activate From 89d47bb638a69339a50ebfb2171ddd3741bd1f17 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:55:59 +0100 Subject: [PATCH 06/11] Fix namespacing --- .github/workflows/pip_install.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 29a0760..cb254b7 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -41,7 +41,7 @@ jobs: # Check which pystencils versions are available, for easier debugging in case of failing CI module avail pystencils # Load pystencils to make it available as dependency - module load python/${{ python-version }} + module load python/${{ matrix.python-version }} # Create virtualenv for this step python -m venv build_dist_env @@ -63,8 +63,8 @@ jobs: module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} - module load matplotlib/${{ matplotlib-version }} - module load IPython/${{ ipython-version }} + module load matplotlib/${{ matrix.matplotlib-version }} + module load IPython/${{ matrix.ipython-version }} # Create virtualenv for this step python -m venv install_from_dist_env @@ -86,8 +86,8 @@ jobs: module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} - module load matplotlib/${{ matplotlib-version }} - module load IPython/${{ ipython-version }} + module load matplotlib/${{ matrix.matplotlib-version }} + module load IPython/${{ matrix.ipython-version }} # Create virtualenv for this step python -m venv install_from_clone @@ -108,8 +108,8 @@ jobs: module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} - module load matplotlib/${{ matplotlib-version }} - module load IPython/${{ ipython-version }} + module load matplotlib/${{ matrix.matplotlib-version }} + module load IPython/${{ matrix.ipython-version }} # Create virtualenv for this step python -m venv install_from_git From 2cc5b492d4ab4876454e22444b13cc862e98eed9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 16:59:37 +0100 Subject: [PATCH 07/11] Fix capitalization --- .github/workflows/pip_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index cb254b7..f6f6e27 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -41,7 +41,7 @@ jobs: # Check which pystencils versions are available, for easier debugging in case of failing CI module avail pystencils # Load pystencils to make it available as dependency - module load python/${{ matrix.python-version }} + module load Python/${{ matrix.python-version }} # Create virtualenv for this step python -m venv build_dist_env From 173ef5bd9338d8653912e30bcebd9e665dafd540 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 17:01:09 +0100 Subject: [PATCH 08/11] Remove the module avails --- .github/workflows/pip_install.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index f6f6e27..d50c346 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -38,8 +38,6 @@ jobs: # Could be used to e.g. put the package on pipy # See https://setuptools.pypa.io/en/latest/userguide/quickstart.html run: | - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils # Load pystencils to make it available as dependency module load Python/${{ matrix.python-version }} @@ -59,8 +57,6 @@ jobs: - name: LPC3D against EESSI deps from distribution tarball # This will test if we can pip install from the distribution tarball we just made run: | - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} @@ -82,8 +78,6 @@ jobs: - name: LPC3D against EESSI deps with pip install . run: | - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} @@ -104,8 +98,6 @@ jobs: - name: LPC3D against EESSI deps with pip install git+https run: | - # Check which pystencils versions are available, for easier debugging in case of failing CI - module avail pystencils # Load pystencils to make it available as dependency module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} From ee80d02e6904510fcb0a101024a5257c4ee14795 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 17:03:23 +0100 Subject: [PATCH 09/11] Try some debugging ls... --- .github/workflows/pip_install.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index d50c346..b9dc825 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -48,6 +48,9 @@ jobs: # Install build pip install --upgrade build + # Check current working dir + ls -al + # Build the source distribution python -m build From a33ef1f4a08a1aa1fe5d8075a519714bc9b1deb0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 21 Nov 2024 17:09:33 +0100 Subject: [PATCH 10/11] Make sure repo is checked out in a clean path, so that python -m build doesn't complain --- .github/workflows/pip_install.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index b9dc825..7cc7c43 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false - fetch-depth: 0 + path: LPC3D - name: Setup EESSI uses: eessi/github-action-eessi@v3 @@ -38,7 +38,7 @@ jobs: # Could be used to e.g. put the package on pipy # See https://setuptools.pypa.io/en/latest/userguide/quickstart.html run: | - # Load pystencils to make it available as dependency + # Load Python to make it available as dependency module load Python/${{ matrix.python-version }} # Create virtualenv for this step @@ -48,8 +48,8 @@ jobs: # Install build pip install --upgrade build - # Check current working dir - ls -al + # Change dir to repo + cd LPC3D # Build the source distribution python -m build @@ -69,6 +69,9 @@ jobs: python -m venv install_from_dist_env source install_from_dist_env/bin/activate + # Change dir to repo + cd LPC3D + # Install from distribution tarball python -m pip install dist/lpc3d*.tar.gz @@ -90,6 +93,9 @@ jobs: python -m venv install_from_clone source install_from_clone/bin/activate + # Change dir to repo + cd LPC3D + # Install from clone pip install . From b3a8ce51043cc1907eaaa941a2a13ee0e2604b57 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 25 Nov 2024 16:10:13 +0100 Subject: [PATCH 11/11] Add explicit numpy dependency --- .github/workflows/pip_install.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 7cc7c43..5ef9f02 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -18,6 +18,7 @@ jobs: python-version: '3.11.5-GCCcore-13.2.0' matplotlib-version: '3.8.2-gfbf-2023b' ipython-version: '8.17.2-GCCcore-13.2.0' + scipy-bundle-version: '2023.11-gfbf-2023b' # for numpy # There is no numba in 2023b... we should include this eventually # For now, the pip install will just pull it in as dependency # numba-version: '' @@ -64,6 +65,7 @@ jobs: module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} module load IPython/${{ matrix.ipython-version }} + module load SciPy-bundle/${{ matrix.scipy-bundle-version }} # Create virtualenv for this step python -m venv install_from_dist_env @@ -88,6 +90,7 @@ jobs: module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} module load IPython/${{ matrix.ipython-version }} + module load SciPy-bundle/${{ matrix.scipy-bundle-version }} # Create virtualenv for this step python -m venv install_from_clone @@ -111,6 +114,7 @@ jobs: module load pystencils/${{ matrix.pystencil-version }} module load matplotlib/${{ matrix.matplotlib-version }} module load IPython/${{ matrix.ipython-version }} + module load SciPy-bundle/${{ matrix.scipy-bundle-version }} # Create virtualenv for this step python -m venv install_from_git