From 53c454c301206a5a12ea38294861ba76fa21d694 Mon Sep 17 00:00:00 2001 From: Patrick Kopper Date: Wed, 17 Jun 2026 13:28:42 +0200 Subject: [PATCH 1/2] Gmsh: Implement fallback mirror for Gmsh download --- pyhope/common/common_vars.py | 8 +++-- pyhope/gmsh/gmsh_install.py | 65 +++++++++++++++++++++++++++--------- pyproject.toml | 3 +- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/pyhope/common/common_vars.py b/pyhope/common/common_vars.py index 12f8055..de0a1de 100644 --- a/pyhope/common/common_vars.py +++ b/pyhope/common/common_vars.py @@ -138,9 +138,13 @@ def commit(self) -> str: @final class Gitlab: # Gitlab "python-gmsh" access - LIB_GITLAB: str = 'gitlab.iag.uni-stuttgart.de' + LIB_GITLAB: tuple[str] = ('gitlab.iag.uni-stuttgart.de', + 'piclas.boltzplatz.eu' , + ) # LIB_PROJECT = 'libs/python-gmsh' - LIB_PROJECT: str = '797' + LIB_PROJECT: tuple[int] = (797, + 26 , + ) LIB_VERSION: dict[str, dict[str, str]] = { # noqa: RUF012 'linux': { 'x86_64' : '4.15.2.post1', diff --git a/pyhope/gmsh/gmsh_install.py b/pyhope/gmsh/gmsh_install.py index 8a1205b..f593586 100644 --- a/pyhope/gmsh/gmsh_install.py +++ b/pyhope/gmsh/gmsh_install.py @@ -28,10 +28,9 @@ import os import platform import shutil -import subprocess from importlib import metadata from packaging.version import Version -from typing import Optional, cast +from typing import Final, Optional, cast # ---------------------------------------------------------------------------------------------------------------------------------- # Third-party libraries # ---------------------------------------------------------------------------------------------------------------------------------- @@ -135,10 +134,13 @@ def PkgsInstallGmsh(system: str, arch: str, version: str) -> None: # Standard libraries ----------------------------------- import sys import hashlib + import subprocess import tempfile + import requests # Local imports ---------------------------------------- import pyhope.output.output as hopout from pyhope.common.common_vars import Gitlab + from pyhope.common.common_progress import ProgressBar # ------------------------------------------------------ # Get our package manager # > Check if 'uv' is available @@ -163,6 +165,9 @@ def PkgsInstallGmsh(system: str, arch: str, version: str) -> None: lfs = 'yes' lib = f'gmsh-{Gitlab.LIB_VERSION[system][arch]}-py3-none-{system}_{arch}.whl' + noPing: Final[int] = 1 # Number of pings + toPing: Final[int] = 5 # Timeout of pings + # Create a temporary directory with tempfile.TemporaryDirectory() as path: # On macOS add major version string to filename an rename darwin to macosx in whl filename @@ -173,20 +178,50 @@ def PkgsInstallGmsh(system: str, arch: str, version: str) -> None: else: pkgs = os.path.join(path, lib) - curl = [f'curl https://{Gitlab.LIB_GITLAB}/api/v4/projects/{Gitlab.LIB_PROJECT}/repository/files/{lib}/raw?lfs={lfs} --output {pkgs}'] # noqa: E501 - _ = subprocess.run(curl, check=True, shell=True) - - # Compare the hash - # > Initialize a new sha256 hash - sha256 = hashlib.sha256() - with open(pkgs, 'rb') as f: - # Read and update hash string value in blocks of 4K - for chunk in iter(lambda: f.read(4096), b""): - sha256.update(chunk) + # Try the hosts in order, we generally expect the first host to be up and up-to-date + for idGit, (gitURL, gitID) in enumerate(zip(Gitlab.LIB_GITLAB, Gitlab.LIB_PROJECT, strict=True)): + try: + subprocess.check_call(('ping', f'-c{noPing}', f'-w{toPing}', gitURL), + stdout = subprocess.DEVNULL, # noqa: E251 + stderr = subprocess.DEVNULL, # noqa: E251 + timeout = toPing) # noqa: E251 + except subprocess.CalledProcessError: # noqa: PERF203 + continue + + # Host is responding, attempt to get the file + # urllib.request.urlretrieve(url = f'https://{gitURL}/api/v4/projects/{Gitlab.LIB_PROJECT}/repository/files/{lib}/raw?lfs={lfs}', # noqa: E251 + # filename = pkgs # noqa: E251 + # ) + request = requests.get(f'https://{gitURL}/api/v4/projects/{gitID}/repository/files/{lib}/raw?lfs={lfs}', + stream=True) + size = int(request.headers['content-length']) + bar = ProgressBar(title = f'│ Downloading Gmsh [v{Gitlab.LIB_VERSION[system][arch]}] from {gitURL}', # noqa: E251 + value = int(size/1024), # noqa: E251 + threshold = 0) # noqa: E251 + + with open(pkgs, 'wb') as f: + for chunk in request.iter_content(chunk_size=1024*50): + if chunk: # filter out keep-alive new chunks + bar.step(int(len(chunk)/1024)) + f.write(chunk) + + bar.close() + + # Compare the hash + # > Initialize a new sha256 hash + sha256 = hashlib.sha256() + with open(pkgs, 'rb') as f: + # Read and update hash string value in blocks of 4K + for chunk in iter(lambda: f.read(4096), b""): + sha256.update(chunk) + + if sha256.hexdigest() == Gitlab.LIB_SUPPORT[system][arch]: + hopout.info('Hash matches, installing Gmsh wheel...') + break + + if idGit < len(Gitlab.LIB_GITLAB) - 1: + hopout.info('Hash mismatch, trying next mirror...') - if sha256.hexdigest() == Gitlab.LIB_SUPPORT[system][arch]: - hopout.info('Hash matches, installing Gmsh wheel...') - else: hopout.error('Hash mismatch, exiting...') # Remove the old version diff --git a/pyproject.toml b/pyproject.toml index 39f339e..6cc23c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,14 +15,15 @@ maintainers = [ ] dependencies = [ 'alive-progress', + 'gmsh', 'h5py', 'hilbertcurve', 'meshio', 'numpy>=2.0.0', 'packaging', 'plotext', + 'requests', 'scipy', - 'gmsh', 'typing-extensions', # 'threadpoolctl' ] From f7e4ade0e81324ee7f4fa0a9cfe528bf270371d6 Mon Sep 17 00:00:00 2001 From: Patrick Kopper Date: Fri, 26 Jun 2026 10:16:29 +0200 Subject: [PATCH 2/2] CI/CD: Implement fallback mirror for Gmsh download --- .github/actions/setup/action.yml | 2 +- .github/workflows/ci.yml | 24 ++++++++++++++-------- .gitlab-ci.yml | 35 ++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a93d159..4b4bc20 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -98,7 +98,7 @@ runs: source .venv/bin/activate if [[ "${{ inputs.minimal }}" != "true" ]]; then - uv pip install --no-cache-dir https://gitlab.iag.uni-stuttgart.de/libs/python-gmsh/-/raw/master/gmsh-4.15.2.post1-py3-none-linux_x86_64.whl + - uv pip install --no-cache-dir https://gitlab.iag.uni-stuttgart.de/libs/python-gmsh/-/raw/master/gmsh-${GMSH_VERSION}-py3-none-linux_x86_64.whl || uv pip install https://piclas.boltzplatz.eu/public-libs/python-gmsh/-/raw/master/gmsh-${GMSH_VERSION}-py3-none-linux_x86_64.whl uv pip install --no-cache-dir -e . fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8021a2..8f01a5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,14 @@ concurrency: # Environment variables env: UV_LINK_MODE: copy + # Define the Gmsh version + GMSH_VERSION: "4.15.2.post1" + # Define the Python version + # PYTHON310_VERSION: 3.10.20-slim-trixie + # PYTHON311_VERSION: 3.11.15-slim-trixie + # PYTHON312_VERSION: 3.12.13-slim-trixie + # PYTHON313_VERSION: 3.13.14-slim-trixie + # PYTHON314_VERSION: 3.14.6-slim-trixie jobs: # ---------------------------------------------------------------------------------------------------------------------------------------------------- @@ -58,7 +66,7 @@ jobs: name: Lint / ruff runs-on: ubuntu-latest container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: - uses: actions/checkout@v6 # with: @@ -78,7 +86,7 @@ jobs: name: Lint / ty runs-on: ubuntu-latest container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: # Git LFS requires git - name: Install Git LFS @@ -102,7 +110,7 @@ jobs: name: Lint / vulture runs-on: ubuntu-latest container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: - uses: actions/checkout@v6 # with: @@ -157,8 +165,8 @@ jobs: - { python-version: "3.10", image: "python:3.10.20-slim-trixie" } - { python-version: "3.11", image: "python:3.11.15-slim-trixie" } - { python-version: "3.12", image: "python:3.12.13-slim-trixie" } - - { python-version: "3.13", image: "python:3.13.13-slim-trixie" } - - { python-version: "3.14", image: "python:3.14.5-slim-trixie" } + - { python-version: "3.13", image: "python:3.13.14-slim-trixie" } + - { python-version: "3.14", image: "python:3.14.6-slim-trixie" } container: image: ${{ matrix.image }} steps: @@ -190,7 +198,7 @@ jobs: runs-on: ubuntu-latest needs: [compatibility] container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: # Git LFS requires git - name: Install Git LFS @@ -264,7 +272,7 @@ jobs: runs-on: ubuntu-latest needs: [compatibility] container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: # Git LFS requires git - name: Install Git LFS @@ -351,7 +359,7 @@ jobs: runs-on: ubuntu-latest needs: [examples, healthcheck] container: - image: python:3.14.5-slim-trixie + image: python:3.14.6-slim-trixie steps: - uses: actions/checkout@v6 # with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 828095e..72d9d86 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,16 +22,25 @@ # Continuous Integration/Continuous Deployment # ================================================================================================================================== -# Official language image. Look for the different tagged releases at: -# > https://hub.docker.com/r/library/python/tags/ -# image: ${CI_REGISTRY_NAME}/nrg-python:3.14.5-trixie-x86_64 -image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.14.5-trixie-x86_64 - # Change pip's cache directory to be inside the project directory since we can only cache local items. variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" # Enable the faster OverlayFS driver DOCKER_DRIVER: overlay2 + # Define the Gmsh version + GMSH_VERSION: "4.15.2.post1" + # Define the Python version + PYTHON310_VERSION: 3.10.20-trixie + PYTHON311_VERSION: 3.11.15-trixie + PYTHON312_VERSION: 3.12.13-trixie + PYTHON313_VERSION: 3.13.13-trixie + PYTHON314_VERSION: 3.14.5-trixie + +# Official language image. Look for the different tagged releases at: +# > https://hub.docker.com/r/library/python/tags/ +# image: ${CI_REGISTRY_NAME}/nrg-python:${PYTHON314_VERSION}-x86_64 +image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON314_VERSION}-x86_64 + # https://pip.pypa.io/en/stable/topics/caching/ # .venv_cache: &venv_cache @@ -85,8 +94,8 @@ variables: # Setup Python virtual environment - uv venv .venv - source .venv/bin/activate - # Regression checks needs NRG Gmsh, pre-install it - - uv pip install --no-cache-dir https://gitlab.iag.uni-stuttgart.de/libs/python-gmsh/-/raw/master/gmsh-4.15.2.post1-py3-none-linux_x86_64.whl + # Regression checks needs NRG Gmsh, pre-install it (try two different mirrors for downloading the .whl file) + - uv pip install --no-cache-dir https://gitlab.iag.uni-stuttgart.de/libs/python-gmsh/-/raw/master/gmsh-${GMSH_VERSION}-py3-none-linux_x86_64.whl || uv pip install https://piclas.boltzplatz.eu/public-libs/python-gmsh/-/raw/master/gmsh-${GMSH_VERSION}-py3-none-linux_x86_64.whl # Install all pre-requisites - uv pip install --no-cache-dir -e . # Install coverage @@ -183,31 +192,31 @@ pre-commit: python:3.10: extends: .defaults stage: compatibility - image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.10.20-trixie-x86_64 + image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON310_VERSION}-x86_64 <<: *compatibility-script # Python 3.11 ---------------------------------------------------------------------------------------------------------------------------------------- python:3.11: extends: .defaults stage: compatibility - image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.11.15-trixie-x86_64 + image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON311_VERSION}-x86_64 <<: *compatibility-script # Python 3.12 ---------------------------------------------------------------------------------------------------------------------------------------- python:3.12: extends: .defaults stage: compatibility - image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.12.13-trixie-x86_64 + image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON312_VERSION}-x86_64 <<: *compatibility-script # Python 3.13 ---------------------------------------------------------------------------------------------------------------------------------------- python:3.13: extends: .defaults stage: compatibility - image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.13.13-trixie-x86_64 + image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON313_VERSION}-x86_64 <<: *compatibility-script # Python 3.14 ---------------------------------------------------------------------------------------------------------------------------------------- python:3.14: extends: .defaults stage: compatibility - # image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:3.14.3-trixie-x86_64 # default version at the moment + # image: registry.iag.uni-stuttgart.de/flexi/codes/pyhope/nrg-python:${PYTHON314_VERSION}-x86_64 # default version at the moment <<: *compatibility-script # ----------------------------------------------------------------------------------------------------------------------------------------------------