diff --git a/generic/pip/module_venv_paths.py b/generic/pip/module_venv_paths.py new file mode 100644 index 0000000..dd9c09e --- /dev/null +++ b/generic/pip/module_venv_paths.py @@ -0,0 +1,11 @@ +import os +import sysconfig + +prefix = os.environ["prefix"] +venv = os.environ.get("VIRTUAL_ENV", None) + +if venv: + venv_site_packages = sysconfig.get_path("platlib") + print(f'prepend-path PYTHONPATH "{venv_site_packages}"') + +print(f'setenv VIRTUAL_ENV "{prefix}"') diff --git a/packages/fireapp/modtemplate.tcl b/packages/fireapp/modtemplate.tcl index ba54663..0747190 100644 --- a/packages/fireapp/modtemplate.tcl +++ b/packages/fireapp/modtemplate.tcl @@ -1,5 +1,8 @@ #%Module module-whatis "Wildfire Analyst VESTEC app" +if {![is-loaded ttk]} { + module load ttk/0.9.9 +} setenv FIREAPP_DIR "${prefix}" prepend-path PATH "${prefix}/bin" diff --git a/packages/mosquito-topo/env.sh b/packages/mosquito-topo/env.sh index 82245ef..4aed092 100644 --- a/packages/mosquito-topo/env.sh +++ b/packages/mosquito-topo/env.sh @@ -12,4 +12,5 @@ if [ -z "$INSTALLER_MOSQUITO_TOPO_ENV_SH" ]; then download_git_repo="git@github.com:VESTEC-EU/mosquito-topological-processing.git" installer_init + python_layer_venv=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py) fi diff --git a/packages/mosquito-topo/modtemplate.tcl b/packages/mosquito-topo/modtemplate.tcl index dfab068..7fec820 100644 --- a/packages/mosquito-topo/modtemplate.tcl +++ b/packages/mosquito-topo/modtemplate.tcl @@ -7,5 +7,5 @@ module load python/3.8.5 module load ttk/0.9.9 setenv MOSQUITO_TOPO_DIR "${prefix}" -setenv VIRTUAL_ENV "${prefix}" +${python_layer_venv} prepend-path PATH "${prefix}/bin" diff --git a/packages/paraview/configure.sh b/packages/paraview/configure.sh deleted file mode 120000 index e242f13..0000000 --- a/packages/paraview/configure.sh +++ /dev/null @@ -1 +0,0 @@ -../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/paraview/configure.sh b/packages/paraview/configure.sh new file mode 100755 index 0000000..6f906ef --- /dev/null +++ b/packages/paraview/configure.sh @@ -0,0 +1,19 @@ +#!/bin/bash +if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + echo "Don't source me" + return 1 +fi +set -e + +thisdir=$(readlink -f $(dirname $BASH_SOURCE)) +. $thisdir/env.sh + +# Set up the virtual env first +if [ ! -f $prefix/bin/activate ]; then + python3 -m venv --system-site-packages $prefix +fi +. $prefix/bin/activate + +pip3 install numpy==${numpy_version} + +$thisdir/main-configure.sh diff --git a/packages/paraview/env.sh b/packages/paraview/env.sh index a933fd5..07782cd 100644 --- a/packages/paraview/env.sh +++ b/packages/paraview/env.sh @@ -13,23 +13,33 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then name=paraview version=5.9.0 version_tag=v${version} + numpy_version=1.19.2 source_dir_name=paraview-$version_tag build_dir_name=build-$version_tag + declare -A cmake_vars + installer_init - declare -A cmake_vars cmake_vars[BUILD_TESTING]=OFF cmake_vars[PARAVIEW_BUILD_SHARED_LIBS]=ON cmake_vars[PARAVIEW_BUILD_EDITION]=CATALYST_RENDERING cmake_vars[paraview_SOURCE_SELECTION]=$version cmake_vars[ENABLE_boost]=ON + cmake_vars[USE_SYSTEM_boost]=ON + cmake_vars[ENABLE_numpy]=ON + cmake_vars[USE_SYSTEM_numpy]=ON cmake_vars[ENABLE_protobuf]=ON cmake_vars[ENABLE_python]=ON cmake_vars[ENABLE_python3]=ON cmake_vars[USE_SYSTEM_python3]=ON - cmake_vars[USE_SYSTEM_boost]=ON + cmake_vars[ENABLE_gdal]=ON + cmake_vars[USE_SYSTEM_zlib]=ON # PV superbuild uses this instead of CMAKE_INSTALL_PREFIX cmake_vars[superbuild_install_location]=$prefix + # Also doesn't use the standard -j flag for parallelism + cmake_vars[SUPERBUILD_PROJECT_PARALLELISM]=$make_parallelism + + python_layer_env=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py) fi diff --git a/packages/paraview/main-configure.sh b/packages/paraview/main-configure.sh new file mode 120000 index 0000000..e242f13 --- /dev/null +++ b/packages/paraview/main-configure.sh @@ -0,0 +1 @@ +../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/paraview/modtemplate.tcl b/packages/paraview/modtemplate.tcl index d008396..6d60a7b 100644 --- a/packages/paraview/modtemplate.tcl +++ b/packages/paraview/modtemplate.tcl @@ -1,6 +1,12 @@ #%Module module-whatis "ParaView-$version server (Catalyst edition)" +if {![is-loaded python]} { + module load python/3.8.5 +} + +${python_layer_venv} + setenv PARAVIEW_DIR "${prefix}" prepend-path PATH "${prefix}/bin" setenv PARAVIEW_VERSION "${version}" diff --git a/packages/ttk/all.sh b/packages/ttk/all.sh old mode 100644 new mode 100755 index f3ebdb0..80fde55 --- a/packages/ttk/all.sh +++ b/packages/ttk/all.sh @@ -6,6 +6,6 @@ fi set -e thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -for step in download patch "pv/patch" "pv/configure" "pv/build" "pv/install" configure build install module; do +for step in download patch configure build install module; do $thisdir/$step.sh done diff --git a/packages/ttk/download.sh b/packages/ttk/download.sh index 6863bcc..6937360 100755 --- a/packages/ttk/download.sh +++ b/packages/ttk/download.sh @@ -13,18 +13,3 @@ if [ ! -d $source_dir_name ]; then fi tar -xzf $version.tar.gz fi - -if [ ! -f $source_dir_name/paraview/patch/patch-paraview-${pv_version}.sh ]; then - echo "Don't have suitable patch for paraview ${pv_version}" - exit 1 -fi - -if [ ! -d $pv_source_dir_name ]; then - pv_tarball=ParaView-v$pv_version.tar.gz - if [ ! -f $pv_tarball ]; then - IFS='.' read -a semver <<< "$pv_version" - pv_ver_minor="v${semver[0]}.${semver[1]}" - wget -O $pv_tarball "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=${pv_ver_minor}&type=source&os=Sources&downloadFile=ParaView-v$pv_version.tar.gz" - fi - tar -xzf $pv_tarball -fi diff --git a/packages/ttk/env.sh b/packages/ttk/env.sh index 69b4a2e..d8d97e6 100644 --- a/packages/ttk/env.sh +++ b/packages/ttk/env.sh @@ -12,22 +12,10 @@ if [ -z "$INSTALLER_TTK_ENV_SH" ]; then build_dir_name=build-$version # TTK needs a patched version of Paraview - pv_version=5.8.1 - pv_source_dir_name=ParaView-v$pv_version + pv_version=5.9.0 declare -A cmake_vars + module load paraview/$pv_version installer_init - - if [ -x $prefix/bin/pvpython ]; then - # This only possible after install; only needed for module - pvpython_sitepackage_dir=$($prefix/bin/pvpython $thisdir/get_site_packages.py) - fi - if [ -f $prefix/bin/activate ]; then - python_sitepackage_dir=$( - ( . /lustre/home/shared/dc118/sw/ttk/0.9.9/bin/activate; - python -c 'import sysconfig; print(sysconfig.get_path("platlib"))' - ) - ) - fi fi diff --git a/packages/ttk/patch.sh b/packages/ttk/patch.sh old mode 100644 new mode 100755 diff --git a/packages/ttk/pv/build.sh b/packages/ttk/pv/build.sh deleted file mode 120000 index ad9623a..0000000 --- a/packages/ttk/pv/build.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/build.sh \ No newline at end of file diff --git a/packages/ttk/pv/configure.sh b/packages/ttk/pv/configure.sh deleted file mode 120000 index cce7d9f..0000000 --- a/packages/ttk/pv/configure.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/configure.sh \ No newline at end of file diff --git a/packages/ttk/pv/env.sh b/packages/ttk/pv/env.sh deleted file mode 100644 index ac61e40..0000000 --- a/packages/ttk/pv/env.sh +++ /dev/null @@ -1,18 +0,0 @@ -if [ -z "$INSTALLER_TTK_PV_ENV_SH" ]; then - INSTALLER_TTK_PV_ENV_SH=1 - installer_ttk_pv_dir=$(readlink -f $(dirname $BASH_SOURCE)) - - # Include the parent dir's one and then override stuff - . $installer_ttk_pv_dir/../env.sh - - source_dir_name=$pv_source_dir_name - build_dir_name=pv-build-$pv_version - make_parallelism=64 - unset cmake_vars - declare -A cmake_vars - - cmake_vars[PARAVIEW_USE_PYTHON]=ON - cmake_vars[PARAVIEW_INSTALL_DEVELOPMENT_FILES]=ON - cmake_vars[PARAVIEW_PYTHON_VERSION]=3 - cmake_vars[PARAVIEW_USE_QT]=OFF -fi diff --git a/packages/ttk/pv/install.sh b/packages/ttk/pv/install.sh deleted file mode 120000 index cd1fa16..0000000 --- a/packages/ttk/pv/install.sh +++ /dev/null @@ -1 +0,0 @@ -../../../generic/cmake/install.sh \ No newline at end of file diff --git a/packages/ttk/pv/patch.sh b/packages/ttk/pv/patch.sh deleted file mode 100755 index 9160109..0000000 --- a/packages/ttk/pv/patch.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then - echo "Don't source me" - return 1 -fi - -thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -. $thisdir/env.sh - -pv_src=$PWD/$pv_source_dir_name - -cd $source_dir_name/paraview/patch - -./patch-paraview-${pv_version}.sh $pv_src diff --git a/packages/ttk/python.sh b/packages/ttk/python.sh deleted file mode 100755 index d961812..0000000 --- a/packages/ttk/python.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then - echo "Don't source me" - return 1 -fi -set -e -thisdir=$(readlink -f $(dirname $BASH_SOURCE)) -. $thisdir/env.sh - -# Here we set up a venv and install numpy - -# Ensure that if this is the first version installed, others can also -# write the package dir -mkdir_gw $app_dir/$name - -python3 -m venv --system-site-packages $prefix -. $prefix/bin/activate - -pip3 install numpy diff --git a/platforms/cirrus/fireapp/config.sh b/platforms/cirrus/fireapp/config.sh index d0d947a..d3ff741 100644 --- a/platforms/cirrus/fireapp/config.sh +++ b/platforms/cirrus/fireapp/config.sh @@ -5,6 +5,6 @@ module load netcdf/4.7.4 module load gdal/3.1.0 module load conan/1.31.3 module load mpt/2.22 -module load paraview/5.9.0 -module load python/3.8.5 +module load ttk/0.9.9 + cmake_vars[MPI_CXX_LINK_FLAGS]=-Wl,-rpath=$MPI_ROOT/lib diff --git a/platforms/cirrus/ttk/config.sh b/platforms/cirrus/ttk/config.sh index 214c10d..e8c77ee 100644 --- a/platforms/cirrus/ttk/config.sh +++ b/platforms/cirrus/ttk/config.sh @@ -3,3 +3,6 @@ module load gcc/6.3.0 module load python/3.8.5 module load boost/1.73.0 module load eigen/3.3.9 +export Eigen3_ROOT=$EIGEN_DIR +# Ensure that TTK and its python modules are installed consistently +cmake_vars[CMAKE_INSTALL_LIBDIR]=lib diff --git a/platforms/cirrus/ttk/modtemplate.tcl b/platforms/cirrus/ttk/modtemplate.tcl index d8bb636..53888ab 100644 --- a/platforms/cirrus/ttk/modtemplate.tcl +++ b/platforms/cirrus/ttk/modtemplate.tcl @@ -1,18 +1,16 @@ #%Module -module-whatis "TTK version ${version} including Paraview ${pv_version}" +module-whatis "TTK version ${version}" -setenv TTK_DIR "${prefix}" -prepend-path PATH "${prefix}/bin" - -setenv VIRTUAL_ENV "${prefix}" -prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins" - -# This is needed to ensure that TTK/Paraview work in another venv with -# this module loaded. -prepend-path PYTHONPATH "${pvpython_sitepackage_dir}" -# And this one is for numpy -prepend-path PYTHONPATH "${python_sitepackage_dir}" +if {![is-loaded paraview]} { + module load paraview/${pv_version} +} if {![is-loaded boost]} { module load boost/1.73.0 } + +setenv TTK_DIR "${prefix}" +prepend-path PATH "${prefix}/bin" + +prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins" +prepend-path PYTHONPATH "${prefix}/lib/python3.8/site-packages"