From 522f4bf46a543b61e0ba21b86c26dd84ec9eea30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Tue, 14 Jan 2025 18:54:25 +0100 Subject: [PATCH 1/7] version and contract --- pyproject.toml | 4 ++-- setup.py | 4 +++- tests/multi-py_compat.sh | 11 +++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93aaca3..58b18e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "rwa" -version = "0.9.3" +version = "0.9.4" description = "HDF5-based serialization library for Python datatypes" authors = ["François Laurent "] license = "Apache 2.0" readme = "README.md" [tool.poetry.dependencies] -python = ">=3.8,<3.12" +python = ">=3.8,<3.14" six = "^1.16.0" numpy = ">=0" scipy = ">=0" diff --git a/setup.py b/setup.py index 722b44d..9aa2098 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name = 'rwa-python', - version = '0.9.3', + version = '0.9.4', description = 'HDF5-based serialization library for Python datatypes', long_description = long_description, url = 'https://github.com/DecBayComp/RWA-python', @@ -41,6 +41,8 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], packages = ['rwa'], install_requires = install_requires, diff --git a/tests/multi-py_compat.sh b/tests/multi-py_compat.sh index 64bfbbf..f223e96 100755 --- a/tests/multi-py_compat.sh +++ b/tests/multi-py_compat.sh @@ -1,6 +1,6 @@ #!/bin/sh -versions="3.5 3.6 3.7 3.8 3.9 3.10 3.11" +versions="3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13" #versions="3.6 2.7" if [ "$(pwd | rev | cut -d/ -f1 | rev)" = "tests" ]; then @@ -17,12 +17,15 @@ fi if ! [ -f "$container" -o -h "$container" ]; then cd ../containers # if this crashes, $0 is not run from the tests directory as it should be echo "No container found; building one..." - if [ -z "$(which apptainer)" ]; then + if command -v apptainer &>/dev/null; then + echo "apptainer build rwa-openmpi-dev.sif rwa-jammy" + apptainer build rwa-openmpi-dev.sif rwa-jammy || exit + elif command -v singularity &>/dev/null; then echo "singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal" singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal || exit else - echo "apptainer build rwa-openmpi-dev.sif rwa-jammy" - apptainer build rwa-openmpi-dev.sif rwa-jammy || exit + echo "No Singularity-compatible container engines found; aborting" + exit 1 fi echo "======================================" echo "Container ready; starting the tests..." From ba3e35d3f90206df91b622e9c737f265cfb657d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Wed, 15 Jan 2025 18:19:54 +0100 Subject: [PATCH 2/7] np.float_ deprecation --- containers/rwa-jammy | 69 ++++++++++++++++++++++++++-------------- rwa/generic.py | 12 +++++-- tests/multi-py_compat.sh | 17 ++++------ 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/containers/rwa-jammy b/containers/rwa-jammy index c210a11..2494d1d 100644 --- a/containers/rwa-jammy +++ b/containers/rwa-jammy @@ -11,20 +11,22 @@ RWA-python is available in multiple Python environments: python3.9 python3.10 (alias python3) python3.11 + python3.12 + python3.13 The container OS is Ubuntu Jammy. %setup echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen - for OLD in 2.7 3.5 3.6; do - if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then - wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py - mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py - fi - done - if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then - wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py - fi + #for OLD in 2.7 3.5 3.6; do + # if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then + # wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py + # mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py + # fi + #done + #if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then + # wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py + #fi # test local changes that have not been committed yet; # to be run from any subdirectory in the RWA-python directory, e.g. containers, tests... @@ -45,26 +47,38 @@ The container OS is Ubuntu Jammy. locale-gen apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ - build-essential git software-properties-common pkg-config + build-essential git software-properties-common pkg-config \ + gpg-agent wget add-apt-repository -y ppa:deadsnakes/ppa apt-get update -y apt-get install -y --no-install-recommends \ python2.7 python2.7-dev \ - python3.5 python3.5-dev \ - python3.6 python3.6-dev python3.6-distutils \ python3.7 python3.7-dev python3.7-distutils \ python3.8 python3.8-dev python3.8-venv \ - python3.9 python3.9-dev \ - python3.10 python3.10-dev python3.10-venv \ - python3.11 python3.11-dev python3.11-venv + python3.9 python3.9-dev python3.9-distutils \ + python3.10-dev python3.10-venv \ + python3.11 python3.11-dev python3.11-venv \ + python3.12 python3.12-dev python3.12-venv \ + python3.13 python3.13-dev python3.13-venv + #python3.5 python3.5-dev \ + #python3.6 python3.6-dev python3.6-distutils \ # python 3.8 and 3.10 won't work with get-pip.py without package venv - for OLD in 2.7 3.5 3.6; do - python$OLD /root/get-pip$OLD.py + for OLD in 2.7 3.7; do + if ! [ -f /root/get-pip$OLD.py ]; then + wget -P /root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py + mv /root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py + fi + if command -v python$OLD &>/dev/null; then + python$OLD /root/get-pip$OLD.py + fi done - for VER in 3.7 3.8 3.9 3.10 3.11; do + if ! [ -f /root/get-pip.py ]; then + wget -P /root/ -- https://bootstrap.pypa.io/get-pip.py + fi + for VER in 3.8 3.9 3.10 3.11 3.12 3.13; do python$VER /root/get-pip.py done @@ -85,15 +99,18 @@ The container OS is Ubuntu Jammy. cd $cur export LC_ALL=C - for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11; do + for version in 2.7 3.7 3.8 3.9 3.10 3.11 3.12 3.13; do python="python${version}" + if ! command -v $python $>/dev/null; then + continue + fi pip="$python -m pip" pip_install="$pip install -U" - if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then + if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" -o "$version" = "3.12" -o "$version" = "3.13" ]; then $pip_install --force-reinstall setuptools $pip_install pytest fi - if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then + if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" -o "$version" = "3.12" -o "$version" = "3.13" ]; then $pip_install mpi4py #$pip_install --no-binary=h5py h5py fi @@ -112,10 +129,10 @@ The container OS is Ubuntu Jammy. %runscript - python="python2.7" + python="python3" if [ -n "$1" ]; then if [ "$1" = "-2" -o "$1" = "-27" ]; then - # nothing to do + python="python2.7" shift elif [ "$1" = "-3" -o "$1" = "-35" ]; then python="python3.5" @@ -138,6 +155,12 @@ The container OS is Ubuntu Jammy. elif [ "$1" = "-311" ]; then python="python3.11" shift + elif [ "$1" = "-312" ]; then + python="python3.12" + shift + elif [ "$1" = "-313" ]; then + python="python3.13" + shift fi fi exec $python $@ diff --git a/rwa/generic.py b/rwa/generic.py index 47b9335..2d69fdb 100644 --- a/rwa/generic.py +++ b/rwa/generic.py @@ -1110,13 +1110,19 @@ def namedtuple_storable(namedtuple, *args, **kwargs): except ImportError: numpy_storables = [] else: - numpy_basic_types = ( + numpy_basic_types = [ numpy.bool_, numpy.int_, numpy.intc, numpy.intp, numpy.int8, numpy.int16, numpy.int32, numpy.int64, numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64, - numpy.float_, numpy.float16, numpy.float32, numpy.float64, + ] + try: + numpy_basic_types.append(numpy.float_) + except AttributeError: + pass + numpy_basic_types.extend([ + numpy.float16, numpy.float32, numpy.float64, numpy.complex_, numpy.complex64, numpy.complex128, - ) + ]) # numpy.dtype numpy_storables = [\ diff --git a/tests/multi-py_compat.sh b/tests/multi-py_compat.sh index f223e96..52f9e68 100755 --- a/tests/multi-py_compat.sh +++ b/tests/multi-py_compat.sh @@ -1,7 +1,6 @@ #!/bin/sh -versions="3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13" -#versions="3.6 2.7" +versions="3.7 3.8 3.9 3.10 3.11 3.12 3.13" if [ "$(pwd | rev | cut -d/ -f1 | rev)" = "tests" ]; then container="$(pwd)/../containers/rwa-openmpi-dev.sif" @@ -20,12 +19,9 @@ if ! [ -f "$container" -o -h "$container" ]; then if command -v apptainer &>/dev/null; then echo "apptainer build rwa-openmpi-dev.sif rwa-jammy" apptainer build rwa-openmpi-dev.sif rwa-jammy || exit - elif command -v singularity &>/dev/null; then + else echo "singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal" singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal || exit - else - echo "No Singularity-compatible container engines found; aborting" - exit 1 fi echo "======================================" echo "Container ready; starting the tests..." @@ -33,9 +29,10 @@ if ! [ -f "$container" -o -h "$container" ]; then cd ../tests fi -hdf5_file=$(mktemp) || exit -poke_script=$(mktemp) || exit -peek_script=$(mktemp) || exit +tmpdir=$(mktemp -d -p .) || exit +hdf5_file=$(mktemp -p $tmpdir) || exit +poke_script=$(mktemp -p $tmpdir) || exit +peek_script=$(mktemp -p $tmpdir) || exit trap "rm -f -- '$hdf_file' '$poke_script' '$peek_script'" EXIT @@ -93,7 +90,7 @@ done done -rm -f -- '$hdf_file' '$poke_script' '$peek_script' +rm -rf -- '$tmpdir' trap - EXIT exit From e50afcf959d932b07d6aade75ffa1ab231e4186c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Thu, 16 Jan 2025 12:58:03 +0100 Subject: [PATCH 3/7] np.complex_ deprecation --- rwa/generic.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rwa/generic.py b/rwa/generic.py index 2d69fdb..2df9a9e 100644 --- a/rwa/generic.py +++ b/rwa/generic.py @@ -1121,8 +1121,15 @@ def namedtuple_storable(namedtuple, *args, **kwargs): pass numpy_basic_types.extend([ numpy.float16, numpy.float32, numpy.float64, - numpy.complex_, numpy.complex64, numpy.complex128, ]) + try: + numpy_basic_types.append(numpy.complex_) + except AttributeError: + pass + numpy_basic_types.extend([ + numpy.complex64, numpy.complex128, + ]) + numpy_basic_types = tuple(numpy_basic_types) # numpy.dtype numpy_storables = [\ From 3315b7aed45b268639214c70f0fd2bc3fe05b85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Thu, 16 Jan 2025 13:27:05 +0100 Subject: [PATCH 4/7] np.string_ deprecation --- rwa/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rwa/hdf5.py b/rwa/hdf5.py index e5cb51e..7a1a735 100644 --- a/rwa/hdf5.py +++ b/rwa/hdf5.py @@ -28,7 +28,7 @@ def to_binary(s): if isinstance(s, six.text_type): s = s.encode('utf-8') - return numpy.string_(s) + return numpy.bytes_(s) if six.PY3: def from_unicode(s): return s @@ -45,7 +45,7 @@ def from_unicode(s): def from_bytes(b): return b to_str = str -to_attr = numpy.string_ +to_attr = numpy.bytes_ from_attr = from_bytes def native_poke(service, objname, obj, container, *args, **kargs): From e8a8f9965a915c886c96c766d85b5cd8666e8952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 17 Jan 2025 16:10:52 +0100 Subject: [PATCH 5/7] jammy recipe updated for python 3.6 to 3.13 --- containers/rwa-jammy | 39 +++++++++++++++++++++++---------------- tests/multi-py_compat.sh | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/containers/rwa-jammy b/containers/rwa-jammy index 2494d1d..caf7e5c 100644 --- a/containers/rwa-jammy +++ b/containers/rwa-jammy @@ -18,15 +18,6 @@ The container OS is Ubuntu Jammy. %setup echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen - #for OLD in 2.7 3.5 3.6; do - # if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then - # wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py - # mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py - # fi - #done - #if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then - # wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py - #fi # test local changes that have not been committed yet; # to be run from any subdirectory in the RWA-python directory, e.g. containers, tests... @@ -48,7 +39,21 @@ The container OS is Ubuntu Jammy. apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ build-essential git software-properties-common pkg-config \ - gpg-agent wget + gpg-agent wget curl libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils \ + tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev + + curl -fsSL https://pyenv.run | bash #&>/dev/null + export PYENV_ROOT=/root/.pyenv + for VER in 3.5 3.6; do + /root/.pyenv/bin/pyenv install $VER + cat </root/python$VER +#!/bin/bash +PYTHON=\$(ls /root/.pyenv/versions/$VER*/bin/python) +eval "\$PYTHON \$@" +EOF + chmod a+x /root/python$VER + done add-apt-repository -y ppa:deadsnakes/ppa apt-get update -y @@ -99,9 +104,11 @@ The container OS is Ubuntu Jammy. cd $cur export LC_ALL=C - for version in 2.7 3.7 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 2.7 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13; do python="python${version}" - if ! command -v $python $>/dev/null; then + if [ "$version" = "3.5" -o "$version" = "3.6" ]; then + python="/root/$python" + elif ! command -v $python $>/dev/null; then continue fi pip="$python -m pip" @@ -134,11 +141,11 @@ The container OS is Ubuntu Jammy. if [ "$1" = "-2" -o "$1" = "-27" ]; then python="python2.7" shift - elif [ "$1" = "-3" -o "$1" = "-35" ]; then - python="python3.5" + elif [ "$1" = "-35" ]; then + python="/root/python3.5" shift elif [ "$1" = "-36" ]; then - python="python3.6" + python="/root/python3.6" shift elif [ "$1" = "-37" ]; then python="python3.7" @@ -149,7 +156,7 @@ The container OS is Ubuntu Jammy. elif [ "$1" = "-39" ]; then python="python3.9" shift - elif [ "$1" = "-310" ]; then + elif [ "$1" = "-3" -o "$1" = "-310" ]; then python="python3.10" shift elif [ "$1" = "-311" ]; then diff --git a/tests/multi-py_compat.sh b/tests/multi-py_compat.sh index 52f9e68..9416205 100755 --- a/tests/multi-py_compat.sh +++ b/tests/multi-py_compat.sh @@ -1,6 +1,6 @@ #!/bin/sh -versions="3.7 3.8 3.9 3.10 3.11 3.12 3.13" +versions="3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13" if [ "$(pwd | rev | cut -d/ -f1 | rev)" = "tests" ]; then container="$(pwd)/../containers/rwa-openmpi-dev.sif" From 36147a8fecea81eb6a3c1de51243f856f2eddd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 17 Jan 2025 16:26:35 +0100 Subject: [PATCH 6/7] remove another occurence of np.float_ --- tests/test_native_hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_native_hdf5.py b/tests/test_native_hdf5.py index cef40b4..ce5130a 100644 --- a/tests/test_native_hdf5.py +++ b/tests/test_native_hdf5.py @@ -44,7 +44,7 @@ def test_base_types(self, tmpdir): for _type in store.store: val = store.peek(_type) # the assertion immediately below is redundant but helps debug when numpy hides its base types - assert not isinstance(val, (np.bool_, np.int_, np.float_)) + assert not isinstance(val, (np.bool_, np.int_, np.float32, np.float64)) assert type(val) is type(value[_type]) finally: store.close() From 32bbd0dfa6d2f42c13aa774589da14dc27b38a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 17 Jan 2025 16:29:09 +0100 Subject: [PATCH 7/7] ci: add Python versions 3.12 and 3.13 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28ca724..bf61fc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] fail-fast: true steps: