From a45aa3d55515c774c8bf5901cbee1b77b37dcc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 11:42:53 +0100 Subject: [PATCH 01/11] fixes for numpy 2 --- notebooks/RWAnalyzer tour.ipynb | 26 ++++++++++++++----- pyproject.toml | 5 ++-- .../calculate_marginalized_integral.py | 16 ++++++------ tramway/tessellation/kdtree/__init__.py | 4 ++- tramway/tessellation/kdtree/dichotomy.py | 2 +- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/notebooks/RWAnalyzer tour.ipynb b/notebooks/RWAnalyzer tour.ipynb index af11b099..3bde2c0c 100644 --- a/notebooks/RWAnalyzer tour.ipynb +++ b/notebooks/RWAnalyzer tour.ipynb @@ -102789,23 +102789,37 @@ "output_type": "stream", "text": [ "Traceback (most recent call last):\n", - " File \"\", line 2, in \n", - "ModuleNotFoundError: No module named 'tramway'\n" + " File \"...\", line 11, in \n", + " a.spt_data = spt_data.from_dataframe(df)\n", + " ^^^^^^^^^^\n", + " File \".../tramway/analyzer/__init__.py\", line 577, in __setattr__\n", + " warnings.warn(\n", + " ~~~~~~~~~~~~~^\n", + " \"attribute '{}' is already initialized; side effects may occur\".format(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ...<2 lines>...\n", + " SideEffectWarning,\n", + " ^^^^^^^^^^^^^^^^^^\n", + " )\n", + " ^\n", + "tramway.core.exceptions.SideEffectWarning: attribute 'spt_data' is already initialized; side effects may occur" ] } ], "source": [ - "%%script python3 --no-raise-error\n", - "\n", "from tramway.analyzer import *\n", + "import traceback\n", "\n", "a = RWAnalyzer()\n", "a.spt_data = spt_data.from_ascii_file('data-examples/demo1.txt')\n", "\n", "df = a.spt_data.dataframe\n", "\n", - "# not allowed!\n", - "a.spt_data = spt_data.from_dataframe(df)" + "try:\n", + " # not allowed!\n", + " a.spt_data = spt_data.from_dataframe(df)\n", + "except:\n", + " traceback.print_exc()" ] }, { diff --git a/pyproject.toml b/pyproject.toml index a6f227f6..f612da97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,10 @@ license = "CECILL-2.1" readme = "README.md" [tool.poetry.dependencies] -python = ">=3.8,<3.13" +python = ">=3.8,<4" setuptools = ">=69.1.0,<99" six = ">=1.16,<2" -numpy = ">=1.24.3,<2" +numpy = ">=1.24.3,<3" scipy = ">=1.9.1,<2" pandas = ">=2.0.2,<3" matplotlib = ">=3.7.1,<4" @@ -28,6 +28,7 @@ nbformat = { version = ">=5.9.0,<6", optional = true } opencv-python = { version = ">=4.7.0.72,<5", optional = true } scikit-image = { version = ">=0.20.0,<1", optional = true } tqdm = { version = ">=4.65.0,<5", optional = true } +ipykernel = "^6.29.5" [tool.poetry.group.dev.dependencies] diff --git a/tramway/inference/bayes_factors/calculate_marginalized_integral.py b/tramway/inference/bayes_factors/calculate_marginalized_integral.py index 2ad9b16c..eef5a603 100644 --- a/tramway/inference/bayes_factors/calculate_marginalized_integral.py +++ b/tramway/inference/bayes_factors/calculate_marginalized_integral.py @@ -16,7 +16,7 @@ def calculate_marginalized_integral(zeta_t, zeta_sp, p, v, E, rel_loc_error, zeta_a=[0, 0], factor_za=0, lamb='int'): - """ + r""" Calculate the marginalized lambda integral >>> Integrate[gamma_inc[p, arg * rel_loc_error] * arg ** (-p), {lambda, 0, 1}] @@ -24,9 +24,9 @@ def calculate_marginalized_integral(zeta_t, zeta_sp, p, v, E, rel_loc_error, zet for the given values of zeta_t and zeta_sp. Here: - arg = (v + factor_za * zeta_a**2 + E * (zeta_t - zeta_a - lambda * zeta_sp)**2); + :math:`arg = (v + factor_za * zeta_a**2 + E * (zeta_t - zeta_a - lambda * zeta_sp)**2)`; IMPORTANT: gamma_inc --- is the normalized lower incomplete gamma function; - rel_loc_error = n * V / (4 * sigma_L^2) --- inverse relative localization error, + :math:`rel_loc_error = n * V / (4 * sigma_L^2)` --- inverse relative localization error, Input: @@ -137,9 +137,9 @@ def calculate_any_lambda_integral(func, break_points=[]): def calculate_integral_ratio(arg_func_up, arg_func_down, pow_up, pow_down, v, rel_loc_error, break_points=[], lamb='marg', rtol=1e-6, atol=1e-32): - """ + r""" Calculate the ratio of two similar lambda integrals, each of which has the form - \int_0^1 d\lambda arg_func(\lambda)**(-pow) * gammainc(pow, rel_loc_error * arg_func(\lambda)) + :math:`\int_0^1 d\lambda arg_func(\lambda)**(-pow) * gammainc(pow, rel_loc_error * arg_func(\lambda))` Input: v --- zeta-independent term inside the argument functions. Must be the same upstairs and downstairs. In Bayes factor calculations, this is v := (1 + n_pi * V_pi / n / V). @@ -194,9 +194,9 @@ def f(l): return f, ln_prefactor def ln_integral_with_loc_error(arg_func, pow, x0): - """Calculate the following expression for a marginalized or fixed lambda: - v**k / Gamma[k] * \int_0^1 d l q(l)**(-k) gammainc(k, q(l) * rel_loc_error), - with q(l) = arg_func(l) + r"""Calculate the following expression for a marginalized or fixed lambda: + :math:`v**k / Gamma[k] * \int_0^1 d l q(l)**(-k) gammainc(k, q(l) * rel_loc_error)` + with :math:`q(l) = arg_func(l)` """ f, ln_prefactor = get_f_with_loc_error(arg_func, pow, x0) if lamb == 'marg': diff --git a/tramway/tessellation/kdtree/__init__.py b/tramway/tessellation/kdtree/__init__.py index 7ec2ee82..2888f81b 100644 --- a/tramway/tessellation/kdtree/__init__.py +++ b/tramway/tessellation/kdtree/__init__.py @@ -211,7 +211,7 @@ def unique_rows(data, *args, **kwargs): for i, v1 in enumerate(self.dichotomy.unit_hypercube): vertices.append( origin - + np.float_(v1) * self.dichotomy.reference_length[level[:, np.newaxis]] + + np.float64(v1) * self.dichotomy.reference_length[level[:, np.newaxis]] ) for jj, v2 in enumerate(self.dichotomy.unit_hypercube[i + 1 :]): if np.sum(v1 != v2) == 1: # neighbors in the voronoi @@ -229,6 +229,8 @@ def unique_rows(data, *args, **kwargs): vertices, I = unique_rows(np.concatenate(vertices, axis=0), return_inverse=True) self._vertices = vertices self._lazy["vertices"] = False + if 1 < len(I.shape): + I = I.flatten() ridge_vertices = I[np.concatenate(ridge_vertices, axis=0)] u, v = ridge_vertices.T nverts = vertices.shape[0] diff --git a/tramway/tessellation/kdtree/dichotomy.py b/tramway/tessellation/kdtree/dichotomy.py index 0ae62595..61da3b3e 100644 --- a/tramway/tessellation/kdtree/dichotomy.py +++ b/tramway/tessellation/kdtree/dichotomy.py @@ -131,7 +131,7 @@ def _split(self, ss_ref, origin, depth): lower = dict() for i, step in enumerate(self.unit_hypercube): # could be parallelized lower[i] = ( - origin + np.float_(step) * self.reference_length[depth] + origin + np.float64(step) * self.reference_length[depth] ) # new origin upper = ( lower[i] + self.reference_length[depth] From 60414371e2eb0ccd6c415cebbe75da701f8f3cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 12:44:36 +0100 Subject: [PATCH 02/11] version increment --- pyproject.toml | 3 +-- setup.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f612da97..a62233a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tramway" -version = "0.6.8" +version = "0.6.9" description = "" authors = ["François Laurent "] license = "CECILL-2.1" @@ -28,7 +28,6 @@ nbformat = { version = ">=5.9.0,<6", optional = true } opencv-python = { version = ">=4.7.0.72,<5", optional = true } scikit-image = { version = ">=0.20.0,<1", optional = true } tqdm = { version = ">=4.65.0,<5", optional = true } -ipykernel = "^6.29.5" [tool.poetry.group.dev.dependencies] diff --git a/setup.py b/setup.py index 167f844b..9ad16ce5 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name = 'tramway', - version = '0.6.8', + version = '0.6.9', description = 'TRamWAy', long_description = long_description, url = 'https://github.com/DecBayComp/TRamWAy', @@ -43,6 +43,7 @@ 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], keywords = '', package_dir = {'tramway': 'tramway'}, From 87421a7284af9513a6f8e82d6bf9a3df78d1b484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 13:06:31 +0100 Subject: [PATCH 03/11] macos ci fix --- .github/workflows/ci.yml | 4 ++-- README.md | 6 +++--- doc/conf.py | 6 +++--- doc/installation.rst | 14 ++++---------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0042eea..03752d70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] fail-fast: true steps: @@ -50,7 +50,7 @@ jobs: if: ${{ runner.os == 'macOS' }} run: | brew update - brew install hdf5 + brew install hdf5 gcc - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/README.md b/README.md index a6b5f367..0184a081 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ An attempt to rewrite the project documentation is available as a [separate proj ## Installation -You will need Python >= 3.6. +You will need Python >= 3.8. ### Windows users @@ -26,10 +26,10 @@ pip install tramway Note that the HDF5 library can usually be installed using any OS' package manager. Only Windows users may have to [manually download and install the HDF5 library](https://tramway.readthedocs.io/en/latest/libhdf5.html), if they do not wish to use Conda instead of pip. -Several installation targets are available, including `full`, that install optional dependencies: +Several installation targets are available; they install optional dependencies. For example (recommended): ``` -pip install tramway[full] +pip install tramway[roi,animate] ``` Most of the functionalities and code examples described in the documentation will run without optional dependencies. diff --git a/doc/conf.py b/doc/conf.py index 2df65ed0..394dd4f5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -21,7 +21,7 @@ sys.path.insert(0, os.path.abspath('..')) -# mocking out rwa (and hopefully h5py) for readthedocs to successfully compile the project and +# mocking out rwa (and hopefully h5py) for readthedocs to successfully compile the project and # generate the doc mock_ok = True try: @@ -75,7 +75,7 @@ def __getattr__(cls, name): # General information about the project. project = u'TRamWAy' -copyright = u'2017-2021, Institut Pasteur' +copyright = u'2017-2025, Institut Pasteur' author = u'François Laurent' # The version info for the project you're documenting, acts as replacement for @@ -85,7 +85,7 @@ def __getattr__(cls, name): # The short X.Y version. version = u'0.6' # The full version, including alpha/beta/rc tags. -release = u'0.6.3' +release = u'0.6.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/installation.rst b/doc/installation.rst index c1d74e14..58b5a3fc 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -3,7 +3,7 @@ Installation ============ -You will need Python >= 3.6. +You will need Python >= 3.8. Windows users ------------- @@ -22,9 +22,9 @@ From PyPI Note that the HDF5 library can usually be installed using any OS' package manager. Only Windows users may have to :ref:`manually download and install the HDF5 library `, if they do not wish to use Conda instead of pip. -Several installation targets are available, including ``full``, that install optional dependencies:: +Several installation targets are available; they install optional dependencies. For example (recommended):: - pip install tramway[full] + pip install tramway[roi,animate] Most of the functionalities and code examples described in this documentation will run without the optional dependencies this target installs in addition to the required dependencies. It is safe to first install |tramway| with minimal requirements and then ``pip install`` the missing dependencies as you hit ``ImportError`` while using |tramway|. @@ -47,7 +47,7 @@ Initial install:: git clone https://github.com/DecBayComp/TRamWAy cd TRamWAy - pip install -e . + pip install -e .[roi,animate] Can be updated with ``git pull`` run in the local repository. @@ -63,9 +63,3 @@ The generated documentation will be available at ``_build/html/index.html`` from Building the documentation requires Sphinx. - -OS and version specific notes ------------------------------ - -Some modules require *Python>=3.7*. -These are *bayes_factor* for force “detection” based on Bayesian statistics, *snr* that extracts signal-to-noise ratios required by the *bayes_factor* module, and *d.conj_prior* which estimates the diffusion similarly to *ddrift* with no regularization but with additional confidence intervals. From 93e3c84d2269d459d10a2159661e81bdc438a9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 13:14:34 +0100 Subject: [PATCH 04/11] macos ci fix (2) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03752d70..492fd087 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: if: ${{ runner.os == 'macOS' }} run: | brew update - brew install hdf5 gcc + brew install hdf5 gcc suite-sparse - name: Install dependencies run: | python -m pip install --upgrade pip From 8042c210a12f891da11743b764ed312da064d6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 13:19:54 +0100 Subject: [PATCH 05/11] macos ci fix (3) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 492fd087..364c5095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,9 @@ jobs: - name: Install HDF5 (macOS) if: ${{ runner.os == 'macOS' }} run: | + xcode-select -p brew update - brew install hdf5 gcc suite-sparse + brew install hdf5 gsl fftw suite-sparse glpk - name: Install dependencies run: | python -m pip install --upgrade pip From 2d2c8bea47896cf07e05dc172625c381b16d0fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 13:33:23 +0100 Subject: [PATCH 06/11] macos ci fix (4) --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 364c5095..8c8ead8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,9 +49,8 @@ jobs: - name: Install HDF5 (macOS) if: ${{ runner.os == 'macOS' }} run: | - xcode-select -p - brew update - brew install hdf5 gsl fftw suite-sparse glpk + brew --prefix + brew install hdf5 openblas gsl fftw suite-sparse glpk - name: Install dependencies run: | python -m pip install --upgrade pip From e517ecafbe745e9dc5cebe4b5fc8660a453e7795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 14:15:24 +0100 Subject: [PATCH 07/11] macos ci fix (5) --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c8ead8b..c6169e17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,12 @@ jobs: } - name: Install HDF5 (macOS) if: ${{ runner.os == 'macOS' }} + env: + CVXOPT_SUITESPARSE_LIB_DIR: /opt/homebrew/lib + CVXOPT_SUITESPARSE_INC_DIR: /opt/homebrew/include/suitesparse + CVXOPT_BLAS_LIB_DIR: /opt/homebrew/opt/openblas/lib + CVXOPT_BLAS_LIB: openblas + CVXOPT_LAPACK_LIB: openblas run: | brew --prefix brew install hdf5 openblas gsl fftw suite-sparse glpk From 2aa33cfb06bda25f7c240fd60cd4dc6693b43983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 14:25:12 +0100 Subject: [PATCH 08/11] macos ci fix (6) --- .github/workflows/ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6169e17..59730174 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,14 +46,11 @@ jobs: rm "${{ github.workspace }}\hdf\HDF5-1.10.7-win64.msi" rm "${{ github.workspace }}\msi.log" } - - name: Install HDF5 (macOS) + - name: Install HDF5 and CVXOPT dependencies (macOS) if: ${{ runner.os == 'macOS' }} env: - CVXOPT_SUITESPARSE_LIB_DIR: /opt/homebrew/lib - CVXOPT_SUITESPARSE_INC_DIR: /opt/homebrew/include/suitesparse - CVXOPT_BLAS_LIB_DIR: /opt/homebrew/opt/openblas/lib - CVXOPT_BLAS_LIB: openblas - CVXOPT_LAPACK_LIB: openblas + LDFLAGS: -L/opt/homebrew/opt/openblas/lib + CPPFLAGS: -I/opt/homebrew/opt/openblas/include run: | brew --prefix brew install hdf5 openblas gsl fftw suite-sparse glpk From fe64a91af225047669abf797eb0bc46d3a5bec93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 14:33:00 +0100 Subject: [PATCH 09/11] macos ci fix (7) --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59730174..a9544095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,14 +46,22 @@ jobs: rm "${{ github.workspace }}\hdf\HDF5-1.10.7-win64.msi" rm "${{ github.workspace }}\msi.log" } - - name: Install HDF5 and CVXOPT dependencies (macOS) + - name: Install HDF5 and CVXOPT (macOS) if: ${{ runner.os == 'macOS' }} env: + # see https://github.com/cvxopt/cvxopt/blob/master/.github/workflows/macos_build.yml + CVXOPT_SUITESPARSE_LIB_DIR: /opt/homebrew/lib + CVXOPT_SUITESPARSE_INC_DIR: /opt/homebrew/include/suitesparse + CVXOPT_BLAS_LIB_DIR: /opt/homebrew/opt/openblas/lib + CVXOPT_BLAS_LIB: openblas + CVXOPT_LAPACK_LIB: openblas LDFLAGS: -L/opt/homebrew/opt/openblas/lib CPPFLAGS: -I/opt/homebrew/opt/openblas/include run: | brew --prefix brew install hdf5 openblas gsl fftw suite-sparse glpk + python -m pip install --upgrade pip + python -m pip install cvxopt - name: Install dependencies run: | python -m pip install --upgrade pip From cfc06897c5b07b9e5fbd7555ac2fe67139af16b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 16:43:09 +0100 Subject: [PATCH 10/11] NaN to nan --- tests/test_commandline.py | 4 ++-- tramway/tessellation/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_commandline.py b/tests/test_commandline.py index 4bae46ad..edf65cb9 100644 --- a/tests/test_commandline.py +++ b/tests/test_commandline.py @@ -31,9 +31,9 @@ assert sys.version_info[0] == 3 -py3_hash = '9X87jkdA' +py3_hash = 'zUhBbH55' data_server = f'http://dl.pasteur.fr/fop/{py3_hash}/' -data_update = '240213' +data_update = '250124' data_file = 'glycine_receptor.trxyt' data_dir = f'test_commandline_py3_{data_update}' diff --git a/tramway/tessellation/base.py b/tramway/tessellation/base.py index 6fd05e28..7eeec1bf 100644 --- a/tramway/tessellation/base.py +++ b/tramway/tessellation/base.py @@ -1504,7 +1504,7 @@ def _postprocess(self, adjacency_label=False): def cell_volume(self): if self._cell_volume is None: adjacency = self.vertex_adjacency.tocsr() - cell_volume = np.full(len(self._cell_centers), np.NaN) + cell_volume = np.full(len(self._cell_centers), np.nan) for i, u in enumerate(self._cell_centers): js = _js = self.cell_vertices[i] # vertex indices From b7c196160cb44f2def50a95e76913d1f9a9d74fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 24 Jan 2025 18:29:09 +0100 Subject: [PATCH 11/11] updated apptainer containers --- containers/available_images.md | 77 ++++++++++++++++++++++++++++ containers/container-pip-freeze.sh | 12 +++-- containers/detect_python.sh | 9 ++-- containers/tramway-hpc-py313 | 74 ++++++++++++++++++++++++++ tramway/analyzer/env/containers.py | 6 +++ tramway/analyzer/env/environments.py | 6 +-- 6 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 containers/tramway-hpc-py313 diff --git a/containers/available_images.md b/containers/available_images.md index 861b89b6..624c0061 100644 --- a/containers/available_images.md +++ b/containers/available_images.md @@ -189,3 +189,80 @@ urllib3==1.25.8 wheel==0.43.0 # py312 only zipp==3.18.1 # py<310 only ``` + +* tramway-hpc-250124-py3{[8](https://dl.pasteur.fr/fop/xVjMGhTa/tramway-hpc-2501024-py38.sif),[9](https://dl.pasteur.fr/fop/AyK3JTC0/tramway-hpc-2501024-py39.sif),[10](https://dl.pasteur.fr/fop/jEmrwWhp/tramway-hpc-2501024-py310.sif),[11](https://dl.pasteur.fr/fop/QmL7TE6w/tramway-hpc-2501024-py311.sif),[12](https://dl.pasteur.fr/fop/elytCrRB/tramway-hpc-2501024-py312.sif),[13](https://dl.pasteur.fr/fop/hSeIMslv/tramway-hpc-2501024-py313.sif)}.sif + + *tramway0.6.9* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. + Python is available as *python3.x*, with *x* any of *8*, *9*, *10*, *11*, *12*, *13* depending on the container file. + The run command admits option *-s*, passed to Python, + resulting in command (e.g.) `python3.x -s -m tramway $@`. + The included Python dependencies are the following: + +``` +certifi==2019.11.28 +chardet==3.0.4 +contourpy==1.1.1 # py == 3.8 +contourpy==1.3.0 # py == 3.9 +contourpy==1.3.1 # py >= 3.10 +cvxopt==1.3.2 +cycler==0.12.1 +dbus-python==1.2.16 +fonttools==4.55.5 +h5py==3.11.0 # py == 3.8 +h5py==3.12.1 # py >= 3.9 +idna==2.8 +imageio==2.35.1 # py == 3.8 +imageio==2.37.0 # py >= 3.9 +importlib_resources==6.4.5 # py == 3.8 +importlib_resources==6.5.2 # py == 3.9 +joblib==1.4.2 +kiwisolver==1.4.7 # py <= 3.9 +kiwisolver==1.4.8 # py >= 3.10 +lazy_loader==0.4 +matplotlib==3.7.5 # py == 3.8 +matplotlib==3.9.4 # py == 3.9 +matplotlib==3.10.0 # py >= 3.10 +networkx==3.1 # py == 3.8 +networkx==3.2.1 # py == 3.9 +networkx==3.4.2 # py >= 3.10 +numpy==1.24.4 # py == 3.8 +numpy==2.0.2 # py == 3.9 +numpy==2.2.2 # py >= 3.10 +opencv-python==4.11.0.86 +packaging==24.2 +pandas==2.0.3 # py == 3.8 +pandas==2.2.3 # py >= 3.9 +pillow==10.4.0 # py == 3.8 +pillow==11.1.0 # py >= 3.9 +polytope==0.2.5 +PyGObject==3.36.0 +pyparsing==3.1.4 # py == 3.8 +pyparsing==3.2.1 # py >= 3.9 +python-apt==2.0.1+ubuntu0.20.4.1 +python-dateutil==2.9.0.post0 +pytz==2024.2 +requests==2.22.0 +requests-unixsocket==0.2.0 +rwa-python==0.9.5 +scikit-image==0.21.0 # py == 3.8 +scikit-image==0.24.0 # py == 3.9 +scikit-image==0.25.0 # py >= 3.10 +scikit-learn==1.3.2 # py == 3.8 +scikit-learn==1.6.1 # py >= 3.9 +scipy==1.10.1 # py == 3.8 +scipy==1.13.1 # py == 3.9 +scipy==1.15.1 # py >= 3.10 +setuptools==75.8.0 # py >= 3.11 +six==1.17.0 +stopit==1.1.2 +threadpoolctl==3.5.0 +tifffile==2023.7.10 # py == 3.8 +tifffile==2024.8.30 # py == 3.9 +tifffile==2025.1.10 # py >= 3.10 +tqdm==4.67.1 +tzdata==2025.1 +urllib3==1.25.8 +zipp==3.20.2 # py == 3.8 +zipp==3.21.0 # py == 3.9 +``` + diff --git a/containers/container-pip-freeze.sh b/containers/container-pip-freeze.sh index 2ddd11c0..df10e026 100755 --- a/containers/container-pip-freeze.sh +++ b/containers/container-pip-freeze.sh @@ -9,13 +9,17 @@ CONTAINER=$1 shift if [ -z $1 ]; then - SINGULARITY="singularity" + SINGULARITY="singularity exec" else - SINGULARITY=$1 + SINGULARITY="$1 exec" fi -PYTHON=$($SINGULARITY exec "$CONTAINER" ./detect_python.sh) +if [ -d /pasteur ]; then + SINGULARITY="$SINGULARITY -B /pasteur" +fi + +PYTHON=$($SINGULARITY "$CONTAINER" ./detect_python.sh) echo $PYTHON -$SINGULARITY exec "$CONTAINER" $PYTHON -s -m pip freeze +$SINGULARITY "$CONTAINER" $PYTHON -s -m pip freeze diff --git a/containers/detect_python.sh b/containers/detect_python.sh index 310e9d36..58b87575 100755 --- a/containers/detect_python.sh +++ b/containers/detect_python.sh @@ -1,9 +1,9 @@ #!/bin/bash -for ((minor=12;6<=minor;minor--)); do +for ((minor=13;6<=minor;minor--)); do py=python3.$minor -if [ -x "$(command -v $py)" ]; then -if [ -z "$($py -m pip show -q tramway 2>&1)" ]; then +if command -v $py &>/dev/null; then +if $py -m pip freeze | grep tramway &>/dev/null; then echo $py exit 0 fi @@ -11,7 +11,7 @@ fi done py=python2.7 -if [ -x "$(command -v $py)" ]; then +if command -v $py &>/dev/null; then if [ -z "$($py -m pip show -q tramway 2>&1)" ]; then echo $py exit 0 @@ -19,4 +19,3 @@ fi fi exit 1 - diff --git a/containers/tramway-hpc-py313 b/containers/tramway-hpc-py313 new file mode 100644 index 00000000..dab60063 --- /dev/null +++ b/containers/tramway-hpc-py313 @@ -0,0 +1,74 @@ +Bootstrap: docker +From: ubuntu:focal + +%help +TRamWAy is available in the python3.13 environment: + python3.13 -m tramway +The container OS is Ubuntu Focal Fossa. + +%setup + + #echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen + echo "en_GB.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen + if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then + wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py + fi + + # TRamWAy can be installed in any of the following 3 ways: + # 1. using `pip install tramway` (stable release) + # 2. using `git clone`+`pip install .` (git version, default) + # 3. copying your own copy of TRamWAy into the container (local version) + + # (3.) test local changes that have not been committed yet: + LOCAL=/path/to/local/TRamWAy + if [ -d $LOCAL ]; then + CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy + mkdir -p ${CONTAINED} + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts + fi + +%post + + ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime + apt-get update -y + apt-get install -y --no-install-recommends locales + locale-gen + apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ + build-essential git software-properties-common libopenblas0 glpk-utils + + # Python 3.13 + add-apt-repository -y ppa:deadsnakes/ppa + apt-get update -y + apt-get install -y --no-install-recommends python3.13 python3.13-venv + rm -rf /var/lib/apt/lists/* + python3.13 /root/get-pip.py + export LC_ALL=C + #pip3.13 install --upgrade pip + pip3.13 uninstall -qy tramway || true + + # (2.) and (3.) + cd /root + if ! [ -d TRamWAy ]; then + git clone https://github.com/DecBayComp/TRamWAy -b dev --single-branch --depth 1 --no-tags + fi + cd TRamWAy + pip3.13 install .[hpc-minimal,animate] + + # (1.) + #pip3.13 install tramway[hpc-minimal,animate] + + pip3.13 install scikit-learn + + mkdir -p /pasteur + +%runscript + + cmd="tramway" + python="python3.13" + if [ -n "$1" -a "$1" = "-s" ]; then + cmd="${python} -s -m tramway" + shift + fi + exec $cmd $@ + diff --git a/tramway/analyzer/env/containers.py b/tramway/analyzer/env/containers.py index 2577d2bd..fa5eb917 100644 --- a/tramway/analyzer/env/containers.py +++ b/tramway/analyzer/env/containers.py @@ -46,4 +46,10 @@ "tramway-hpc-240423-py310.sif": "https://dl.pasteur.fr/fop/zZ2nrnKP/tramway-hpc-240423-py310.sif", "tramway-hpc-240423-py311.sif": "https://dl.pasteur.fr/fop/7F8PPD7j/tramway-hpc-240423-py311.sif", "tramway-hpc-240423-py312.sif": "https://dl.pasteur.fr/fop/lRWEB8oR/tramway-hpc-240423-py312.sif", + "tramway-hpc-250124-py38.sif": "https://dl.pasteur.fr/fop/xVjMGhTa/tramway-hpc-250124-py38.sif", + "tramway-hpc-250124-py39.sif": "https://dl.pasteur.fr/fop/AyK3JTC0/tramway-hpc-250124-py39.sif", + "tramway-hpc-250124-py310.sif": "https://dl.pasteur.fr/fop/jEmrwWhp/tramway-hpc-250124-py310.sif", + "tramway-hpc-250124-py311.sif": "https://dl.pasteur.fr/fop/QmL7TE6w/tramway-hpc-250124-py311.sif", + "tramway-hpc-250124-py312.sif": "https://dl.pasteur.fr/fop/elytCrRB/tramway-hpc-250124-py312.sif", + "tramway-hpc-250124-py313.sif": "https://dl.pasteur.fr/fop/hSeIMslv/tramway-hpc-250124-py313.sif", } diff --git a/tramway/analyzer/env/environments.py b/tramway/analyzer/env/environments.py index cb757d97..dc626a15 100644 --- a/tramway/analyzer/env/environments.py +++ b/tramway/analyzer/env/environments.py @@ -2145,7 +2145,7 @@ def list_uncomplete_task_log_files(self): def select_python_version(major, minor, *args): if major == 3: - minor = max(6, min(11, minor)) + minor = max(6, min(13, minor)) return major, minor @@ -2156,7 +2156,7 @@ class SingularitySlurm(SlurmOverSSH): """ Runs TRamWAy jobs as Slurm jobs in a Singularity container. - The current default Singularity container is *tramway-hpc-240423-py3?.sif*. + The current default Singularity container is *tramway-hpc-250124-py3?.sif*. See also `available_images.md `_. Children classes should define the :meth:`hostname` and :meth:`scratch` methods. @@ -2315,7 +2315,7 @@ def get_container_url(self, container=None): @classmethod def default_container(cls, python_version=PYVER): - return f"tramway-hpc-240423-py{python_version}.sif" + return f"tramway-hpc-250124-py{python_version}.sif" def early_setup(self, *argv, connect=False, ensure_container=True, **kwargs): ret = SlurmOverSSH.early_setup(self, *argv, connect=connect, **kwargs)