From f709fd11b49e1683fc1f2a22e0ab23e5d9efaf95 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 30 Jul 2021 14:29:22 +0100 Subject: [PATCH 1/6] enable gdal within PV --- packages/paraview/env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/paraview/env.sh b/packages/paraview/env.sh index a933fd5..52d4fca 100644 --- a/packages/paraview/env.sh +++ b/packages/paraview/env.sh @@ -28,6 +28,7 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then cmake_vars[ENABLE_protobuf]=ON cmake_vars[ENABLE_python]=ON cmake_vars[ENABLE_python3]=ON + cmake_vars[ENABLE_gdal]=ON cmake_vars[USE_SYSTEM_python3]=ON cmake_vars[USE_SYSTEM_boost]=ON # PV superbuild uses this instead of CMAKE_INSTALL_PREFIX From c4528abaea226b5517c3f304d520b9e20ed83598 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 11 Aug 2021 17:34:35 +0100 Subject: [PATCH 2/6] superbuild numpy and add compile parallism --- packages/paraview/env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/paraview/env.sh b/packages/paraview/env.sh index 52d4fca..67c7a9c 100644 --- a/packages/paraview/env.sh +++ b/packages/paraview/env.sh @@ -25,6 +25,7 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then cmake_vars[PARAVIEW_BUILD_EDITION]=CATALYST_RENDERING cmake_vars[paraview_SOURCE_SELECTION]=$version cmake_vars[ENABLE_boost]=ON + cmake_vars[ENABLE_numpy]=ON cmake_vars[ENABLE_protobuf]=ON cmake_vars[ENABLE_python]=ON cmake_vars[ENABLE_python3]=ON @@ -33,4 +34,6 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then cmake_vars[USE_SYSTEM_boost]=ON # PV superbuild uses this instead of CMAKE_INSTALL_PREFIX cmake_vars[superbuild_install_location]=$prefix + # Also doesn't use the standard -j flag for parallelism + cmake_vars[SUPERBUILD_PROJECT_PARALLELISM]=$make_parallelism fi From 3fd7f8073a7089c586fef7a2ed95bf35fb6c4d55 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 12 Aug 2021 11:46:52 +0100 Subject: [PATCH 3/6] further paraview --- packages/paraview/configure.sh | 20 +++++++++++++++++++- packages/paraview/env.sh | 7 +++++-- packages/paraview/main-configure.sh | 1 + packages/paraview/modtemplate.tcl | 4 ++++ 4 files changed, 29 insertions(+), 3 deletions(-) mode change 120000 => 100755 packages/paraview/configure.sh create mode 120000 packages/paraview/main-configure.sh diff --git a/packages/paraview/configure.sh b/packages/paraview/configure.sh deleted file mode 120000 index e242f13..0000000 --- a/packages/paraview/configure.sh +++ /dev/null @@ -1 +0,0 @@ -../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/paraview/configure.sh b/packages/paraview/configure.sh new file mode 100755 index 0000000..6f906ef --- /dev/null +++ b/packages/paraview/configure.sh @@ -0,0 +1,19 @@ +#!/bin/bash +if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + echo "Don't source me" + return 1 +fi +set -e + +thisdir=$(readlink -f $(dirname $BASH_SOURCE)) +. $thisdir/env.sh + +# Set up the virtual env first +if [ ! -f $prefix/bin/activate ]; then + python3 -m venv --system-site-packages $prefix +fi +. $prefix/bin/activate + +pip3 install numpy==${numpy_version} + +$thisdir/main-configure.sh diff --git a/packages/paraview/env.sh b/packages/paraview/env.sh index 67c7a9c..84c331f 100644 --- a/packages/paraview/env.sh +++ b/packages/paraview/env.sh @@ -13,6 +13,7 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then name=paraview version=5.9.0 version_tag=v${version} + numpy_version=1.19.2 source_dir_name=paraview-$version_tag build_dir_name=build-$version_tag @@ -25,13 +26,15 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then cmake_vars[PARAVIEW_BUILD_EDITION]=CATALYST_RENDERING cmake_vars[paraview_SOURCE_SELECTION]=$version cmake_vars[ENABLE_boost]=ON + cmake_vars[USE_SYSTEM_boost]=ON cmake_vars[ENABLE_numpy]=ON + cmake_vars[USE_SYSTEM_numpy]=ON cmake_vars[ENABLE_protobuf]=ON cmake_vars[ENABLE_python]=ON cmake_vars[ENABLE_python3]=ON - cmake_vars[ENABLE_gdal]=ON cmake_vars[USE_SYSTEM_python3]=ON - cmake_vars[USE_SYSTEM_boost]=ON + cmake_vars[ENABLE_gdal]=ON + cmake_vars[USE_SYSTEM_zlib]=ON # PV superbuild uses this instead of CMAKE_INSTALL_PREFIX cmake_vars[superbuild_install_location]=$prefix # Also doesn't use the standard -j flag for parallelism diff --git a/packages/paraview/main-configure.sh b/packages/paraview/main-configure.sh new file mode 120000 index 0000000..e242f13 --- /dev/null +++ b/packages/paraview/main-configure.sh @@ -0,0 +1 @@ +../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/paraview/modtemplate.tcl b/packages/paraview/modtemplate.tcl index d008396..045e3a6 100644 --- a/packages/paraview/modtemplate.tcl +++ b/packages/paraview/modtemplate.tcl @@ -1,6 +1,10 @@ #%Module module-whatis "ParaView-$version server (Catalyst edition)" +if {![is-loaded python]} { + module load python/3.8.5 +} +setenv VIRTUAL_ENV "${prefix}" setenv PARAVIEW_DIR "${prefix}" prepend-path PATH "${prefix}/bin" setenv PARAVIEW_VERSION "${version}" From bdd481e195381e06ea0f6abb272faab488e1f4be Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 12 Aug 2021 15:13:03 +0100 Subject: [PATCH 4/6] update TTK to use paraview module --- packages/ttk/all.sh | 2 +- packages/ttk/download.sh | 15 --------------- packages/ttk/env.sh | 16 ++-------------- packages/ttk/patch.sh | 0 packages/ttk/pv/build.sh | 1 - packages/ttk/pv/configure.sh | 1 - packages/ttk/pv/env.sh | 18 ------------------ packages/ttk/pv/install.sh | 1 - packages/ttk/pv/patch.sh | 14 -------------- packages/ttk/python.sh | 19 ------------------- platforms/cirrus/ttk/config.sh | 3 +++ platforms/cirrus/ttk/modtemplate.tcl | 20 +++++++++----------- 12 files changed, 15 insertions(+), 95 deletions(-) mode change 100644 => 100755 packages/ttk/all.sh mode change 100644 => 100755 packages/ttk/patch.sh delete mode 120000 packages/ttk/pv/build.sh delete mode 120000 packages/ttk/pv/configure.sh delete mode 100644 packages/ttk/pv/env.sh delete mode 120000 packages/ttk/pv/install.sh delete mode 100755 packages/ttk/pv/patch.sh delete mode 100755 packages/ttk/python.sh diff --git a/packages/ttk/all.sh b/packages/ttk/all.sh old mode 100644 new mode 100755 index f3ebdb0..80fde55 --- a/packages/ttk/all.sh +++ b/packages/ttk/all.sh @@ -6,6 +6,6 @@ fi set -e thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -for step in download patch "pv/patch" "pv/configure" "pv/build" "pv/install" configure build install module; do +for step in download patch configure build install module; do $thisdir/$step.sh done diff --git a/packages/ttk/download.sh b/packages/ttk/download.sh index 6863bcc..6937360 100755 --- a/packages/ttk/download.sh +++ b/packages/ttk/download.sh @@ -13,18 +13,3 @@ if [ ! -d $source_dir_name ]; then fi tar -xzf $version.tar.gz fi - -if [ ! -f $source_dir_name/paraview/patch/patch-paraview-${pv_version}.sh ]; then - echo "Don't have suitable patch for paraview ${pv_version}" - exit 1 -fi - -if [ ! -d $pv_source_dir_name ]; then - pv_tarball=ParaView-v$pv_version.tar.gz - if [ ! -f $pv_tarball ]; then - IFS='.' read -a semver <<< "$pv_version" - pv_ver_minor="v${semver[0]}.${semver[1]}" - wget -O $pv_tarball "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=${pv_ver_minor}&type=source&os=Sources&downloadFile=ParaView-v$pv_version.tar.gz" - fi - tar -xzf $pv_tarball -fi diff --git a/packages/ttk/env.sh b/packages/ttk/env.sh index 69b4a2e..d8d97e6 100644 --- a/packages/ttk/env.sh +++ b/packages/ttk/env.sh @@ -12,22 +12,10 @@ if [ -z "$INSTALLER_TTK_ENV_SH" ]; then build_dir_name=build-$version # TTK needs a patched version of Paraview - pv_version=5.8.1 - pv_source_dir_name=ParaView-v$pv_version + pv_version=5.9.0 declare -A cmake_vars + module load paraview/$pv_version installer_init - - if [ -x $prefix/bin/pvpython ]; then - # This only possible after install; only needed for module - pvpython_sitepackage_dir=$($prefix/bin/pvpython $thisdir/get_site_packages.py) - fi - if [ -f $prefix/bin/activate ]; then - python_sitepackage_dir=$( - ( . /lustre/home/shared/dc118/sw/ttk/0.9.9/bin/activate; - python -c 'import sysconfig; print(sysconfig.get_path("platlib"))' - ) - ) - fi fi diff --git a/packages/ttk/patch.sh b/packages/ttk/patch.sh old mode 100644 new mode 100755 diff --git a/packages/ttk/pv/build.sh b/packages/ttk/pv/build.sh deleted file mode 120000 index ad9623a..0000000 --- a/packages/ttk/pv/build.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/build.sh \ No newline at end of file diff --git a/packages/ttk/pv/configure.sh b/packages/ttk/pv/configure.sh deleted file mode 120000 index cce7d9f..0000000 --- a/packages/ttk/pv/configure.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/ttk/pv/env.sh b/packages/ttk/pv/env.sh deleted file mode 100644 index ac61e40..0000000 --- a/packages/ttk/pv/env.sh +++ /dev/null @@ -1,18 +0,0 @@ -if [ -z "$INSTALLER_TTK_PV_ENV_SH" ]; then - INSTALLER_TTK_PV_ENV_SH=1 - installer_ttk_pv_dir=$(readlink -f $(dirname $BASH_SOURCE)) - - # Include the parent dir's one and then override stuff - . $installer_ttk_pv_dir/../env.sh - - source_dir_name=$pv_source_dir_name - build_dir_name=pv-build-$pv_version - make_parallelism=64 - unset cmake_vars - declare -A cmake_vars - - cmake_vars[PARAVIEW_USE_PYTHON]=ON - cmake_vars[PARAVIEW_INSTALL_DEVELOPMENT_FILES]=ON - cmake_vars[PARAVIEW_PYTHON_VERSION]=3 - cmake_vars[PARAVIEW_USE_QT]=OFF -fi diff --git a/packages/ttk/pv/install.sh b/packages/ttk/pv/install.sh deleted file mode 120000 index cd1fa16..0000000 --- a/packages/ttk/pv/install.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/install.sh \ No newline at end of file diff --git a/packages/ttk/pv/patch.sh b/packages/ttk/pv/patch.sh deleted file mode 100755 index 9160109..0000000 --- a/packages/ttk/pv/patch.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then - echo "Don't source me" - return 1 -fi - -thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -. $thisdir/env.sh - -pv_src=$PWD/$pv_source_dir_name - -cd $source_dir_name/paraview/patch - -./patch-paraview-${pv_version}.sh $pv_src diff --git a/packages/ttk/python.sh b/packages/ttk/python.sh deleted file mode 100755 index d961812..0000000 --- a/packages/ttk/python.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then - echo "Don't source me" - return 1 -fi -set -e -thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -. $thisdir/env.sh - -# Here we set up a venv and install numpy - -# Ensure that if this is the first version installed, others can also -# write the package dir -mkdir_gw $app_dir/$name - -python3 -m venv --system-site-packages $prefix -. $prefix/bin/activate - -pip3 install numpy diff --git a/platforms/cirrus/ttk/config.sh b/platforms/cirrus/ttk/config.sh index 214c10d..e8c77ee 100644 --- a/platforms/cirrus/ttk/config.sh +++ b/platforms/cirrus/ttk/config.sh @@ -3,3 +3,6 @@ module load gcc/6.3.0 module load python/3.8.5 module load boost/1.73.0 module load eigen/3.3.9 +export Eigen3_ROOT=$EIGEN_DIR +# Ensure that TTK and its python modules are installed consistently +cmake_vars[CMAKE_INSTALL_LIBDIR]=lib diff --git a/platforms/cirrus/ttk/modtemplate.tcl b/platforms/cirrus/ttk/modtemplate.tcl index d8bb636..19b25a8 100644 --- a/platforms/cirrus/ttk/modtemplate.tcl +++ b/platforms/cirrus/ttk/modtemplate.tcl @@ -1,18 +1,16 @@ #%Module -module-whatis "TTK version ${version} including Paraview ${pv_version}" +module-whatis "TTK version ${version}" + +if {![is-loaded paraview]} { + module load paraview/${pv_version} +} + +if {![is-loaded boost]} { + module load boost/1.73.0 +} setenv TTK_DIR "${prefix}" prepend-path PATH "${prefix}/bin" -setenv VIRTUAL_ENV "${prefix}" prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins" -# This is needed to ensure that TTK/Paraview work in another venv with -# this module loaded. -prepend-path PYTHONPATH "${pvpython_sitepackage_dir}" -# And this one is for numpy -prepend-path PYTHONPATH "${python_sitepackage_dir}" - -if {![is-loaded boost]} { - module load boost/1.73.0 -} From 2e0af0b33c83eeec587a3dd012d0235546d5775d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 13 Aug 2021 08:52:08 +0100 Subject: [PATCH 5/6] deal with python and multiple levels of packages --- generic/pip/module_venv_paths.py | 11 +++++++++++ packages/mosquito-topo/env.sh | 1 + packages/mosquito-topo/modtemplate.tcl | 2 +- packages/paraview/env.sh | 5 ++++- packages/paraview/modtemplate.tcl | 4 +++- platforms/cirrus/ttk/modtemplate.tcl | 2 +- 6 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 generic/pip/module_venv_paths.py diff --git a/generic/pip/module_venv_paths.py b/generic/pip/module_venv_paths.py new file mode 100644 index 0000000..dd9c09e --- /dev/null +++ b/generic/pip/module_venv_paths.py @@ -0,0 +1,11 @@ +import os +import sysconfig + +prefix = os.environ["prefix"] +venv = os.environ.get("VIRTUAL_ENV", None) + +if venv: + venv_site_packages = sysconfig.get_path("platlib") + print(f'prepend-path PYTHONPATH "{venv_site_packages}"') + +print(f'setenv VIRTUAL_ENV "{prefix}"') diff --git a/packages/mosquito-topo/env.sh b/packages/mosquito-topo/env.sh index 82245ef..4aed092 100644 --- a/packages/mosquito-topo/env.sh +++ b/packages/mosquito-topo/env.sh @@ -12,4 +12,5 @@ if [ -z "$INSTALLER_MOSQUITO_TOPO_ENV_SH" ]; then download_git_repo="git@github.com:VESTEC-EU/mosquito-topological-processing.git" installer_init + python_layer_venv=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py) fi diff --git a/packages/mosquito-topo/modtemplate.tcl b/packages/mosquito-topo/modtemplate.tcl index dfab068..7fec820 100644 --- a/packages/mosquito-topo/modtemplate.tcl +++ b/packages/mosquito-topo/modtemplate.tcl @@ -7,5 +7,5 @@ module load python/3.8.5 module load ttk/0.9.9 setenv MOSQUITO_TOPO_DIR "${prefix}" -setenv VIRTUAL_ENV "${prefix}" +${python_layer_venv} prepend-path PATH "${prefix}/bin" diff --git a/packages/paraview/env.sh b/packages/paraview/env.sh index 84c331f..07782cd 100644 --- a/packages/paraview/env.sh +++ b/packages/paraview/env.sh @@ -18,9 +18,10 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then source_dir_name=paraview-$version_tag build_dir_name=build-$version_tag + declare -A cmake_vars + installer_init - declare -A cmake_vars cmake_vars[BUILD_TESTING]=OFF cmake_vars[PARAVIEW_BUILD_SHARED_LIBS]=ON cmake_vars[PARAVIEW_BUILD_EDITION]=CATALYST_RENDERING @@ -39,4 +40,6 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then cmake_vars[superbuild_install_location]=$prefix # Also doesn't use the standard -j flag for parallelism cmake_vars[SUPERBUILD_PROJECT_PARALLELISM]=$make_parallelism + + python_layer_env=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py) fi diff --git a/packages/paraview/modtemplate.tcl b/packages/paraview/modtemplate.tcl index 045e3a6..6d60a7b 100644 --- a/packages/paraview/modtemplate.tcl +++ b/packages/paraview/modtemplate.tcl @@ -4,7 +4,9 @@ module-whatis "ParaView-$version server (Catalyst edition)" if {![is-loaded python]} { module load python/3.8.5 } -setenv VIRTUAL_ENV "${prefix}" + +${python_layer_venv} + setenv PARAVIEW_DIR "${prefix}" prepend-path PATH "${prefix}/bin" setenv PARAVIEW_VERSION "${version}" diff --git a/platforms/cirrus/ttk/modtemplate.tcl b/platforms/cirrus/ttk/modtemplate.tcl index 19b25a8..53888ab 100644 --- a/platforms/cirrus/ttk/modtemplate.tcl +++ b/platforms/cirrus/ttk/modtemplate.tcl @@ -13,4 +13,4 @@ setenv TTK_DIR "${prefix}" prepend-path PATH "${prefix}/bin" prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins" - +prepend-path PYTHONPATH "${prefix}/lib/python3.8/site-packages" From 4ebbe9615e3950710a52a907df1d6f56fb16c31d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 20 Aug 2021 13:41:24 +0100 Subject: [PATCH 6/6] Tweak fire sim build to work with TTK --- packages/fireapp/modtemplate.tcl | 3 +++ platforms/cirrus/fireapp/config.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/fireapp/modtemplate.tcl b/packages/fireapp/modtemplate.tcl index ba54663..0747190 100644 --- a/packages/fireapp/modtemplate.tcl +++ b/packages/fireapp/modtemplate.tcl @@ -1,5 +1,8 @@ #%Module module-whatis "Wildfire Analyst VESTEC app" +if {![is-loaded ttk]} { + module load ttk/0.9.9 +} setenv FIREAPP_DIR "${prefix}" prepend-path PATH "${prefix}/bin" diff --git a/platforms/cirrus/fireapp/config.sh b/platforms/cirrus/fireapp/config.sh index d0d947a..d3ff741 100644 --- a/platforms/cirrus/fireapp/config.sh +++ b/platforms/cirrus/fireapp/config.sh @@ -5,6 +5,6 @@ module load netcdf/4.7.4 module load gdal/3.1.0 module load conan/1.31.3 module load mpt/2.22 -module load paraview/5.9.0 -module load python/3.8.5 +module load ttk/0.9.9 + cmake_vars[MPI_CXX_LINK_FLAGS]=-Wl,-rpath=$MPI_ROOT/lib