Reworking of Paraview and TTK build: catalyst adaptor working in WFA

generic/pip/module_venv_paths.py

@@ -0,0 +1,11 @@
+import os
+import sysconfig
+
+prefix = os.environ["prefix"]
+venv = os.environ.get("VIRTUAL_ENV", None)
+
+if venv:
+    venv_site_packages = sysconfig.get_path("platlib")
+    print(f'prepend-path PYTHONPATH "{venv_site_packages}"')
+
+print(f'setenv VIRTUAL_ENV "{prefix}"')

packages/fireapp/modtemplate.tcl

@@ -1,5 +1,8 @@
 #%Module
 module-whatis "Wildfire Analyst VESTEC app"
 
+if {![is-loaded ttk]} {
+    module load ttk/0.9.9
+}
 setenv FIREAPP_DIR "${prefix}"
 prepend-path PATH "${prefix}/bin"

packages/mosquito-topo/env.sh

@@ -12,4 +12,5 @@ if [ -z "$INSTALLER_MOSQUITO_TOPO_ENV_SH" ]; then
 
     download_git_repo="git@github.com:VESTEC-EU/mosquito-topological-processing.git"
     installer_init
+    python_layer_venv=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py)
 fi

packages/mosquito-topo/modtemplate.tcl

@@ -7,5 +7,5 @@ module load python/3.8.5
 module load ttk/0.9.9
 
 setenv MOSQUITO_TOPO_DIR "${prefix}"
-setenv VIRTUAL_ENV "${prefix}"
+${python_layer_venv}
 prepend-path PATH "${prefix}/bin"

packages/paraview/configure.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
+    echo "Don't source me"
+    return 1
+fi
+set -e
+
+thisdir=$(readlink -f $(dirname $BASH_SOURCE))
+. $thisdir/env.sh
+
+# Set up the virtual env first
+if [ ! -f $prefix/bin/activate ]; then 
+    python3 -m venv --system-site-packages $prefix
+fi
+. $prefix/bin/activate
+
+pip3 install numpy==${numpy_version}
+
+$thisdir/main-configure.sh

packages/paraview/env.sh

@@ -13,23 +13,33 @@ if [ -z "$INSTALLER_PARAVIEW_ENV_SH" ]; then
     name=paraview
     version=5.9.0 
     version_tag=v${version}
+    numpy_version=1.19.2
 
     source_dir_name=paraview-$version_tag
     build_dir_name=build-$version_tag
 
+    declare -A cmake_vars
+
     installer_init
 
-    declare -A cmake_vars
     cmake_vars[BUILD_TESTING]=OFF
     cmake_vars[PARAVIEW_BUILD_SHARED_LIBS]=ON
     cmake_vars[PARAVIEW_BUILD_EDITION]=CATALYST_RENDERING
     cmake_vars[paraview_SOURCE_SELECTION]=$version
     cmake_vars[ENABLE_boost]=ON
+    cmake_vars[USE_SYSTEM_boost]=ON
+    cmake_vars[ENABLE_numpy]=ON
+    cmake_vars[USE_SYSTEM_numpy]=ON
     cmake_vars[ENABLE_protobuf]=ON
     cmake_vars[ENABLE_python]=ON
     cmake_vars[ENABLE_python3]=ON
     cmake_vars[USE_SYSTEM_python3]=ON
-    cmake_vars[USE_SYSTEM_boost]=ON
+    cmake_vars[ENABLE_gdal]=ON
+    cmake_vars[USE_SYSTEM_zlib]=ON
     # PV superbuild uses this instead of CMAKE_INSTALL_PREFIX
     cmake_vars[superbuild_install_location]=$prefix
+    # Also doesn't use the standard -j flag for parallelism
+    cmake_vars[SUPERBUILD_PROJECT_PARALLELISM]=$make_parallelism
+
+    python_layer_env=$(prefix=$prefix python3 $installer_dir/generic/pip/module_venv_paths.py)
 fi

packages/paraview/main-configure.sh

@@ -0,0 +1 @@
+../../generic/cmake/configure.sh

packages/paraview/modtemplate.tcl

@@ -1,6 +1,12 @@
 #%Module
 module-whatis "ParaView-$version server (Catalyst edition)"
 
+if {![is-loaded python]} {
+    module load python/3.8.5
+}
+
+${python_layer_venv}
+
 setenv PARAVIEW_DIR "${prefix}"
 prepend-path PATH "${prefix}/bin"
 setenv PARAVIEW_VERSION "${version}"

packages/ttk/all.sh

@@ -6,6 +6,6 @@ fi
 set -e
 
 thisdir=$(readlink -f $(dirname $BASH_SOURCE))
-for step in download patch "pv/patch" "pv/configure" "pv/build" "pv/install" configure build install module; do
+for step in download patch configure build install module; do
     $thisdir/$step.sh
 done

packages/ttk/download.sh

@@ -13,18 +13,3 @@ if [ ! -d $source_dir_name ]; then
     fi
     tar -xzf $version.tar.gz
 fi
-
-if [ ! -f $source_dir_name/paraview/patch/patch-paraview-${pv_version}.sh ]; then
-   echo "Don't have suitable patch for paraview ${pv_version}"
-   exit 1
-fi
-
-if [ ! -d $pv_source_dir_name ]; then
-    pv_tarball=ParaView-v$pv_version.tar.gz
-    if [ ! -f $pv_tarball ]; then
-	IFS='.' read -a semver <<< "$pv_version"
-	pv_ver_minor="v${semver[0]}.${semver[1]}"
-	wget -O $pv_tarball "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=${pv_ver_minor}&type=source&os=Sources&downloadFile=ParaView-v$pv_version.tar.gz"
-    fi
-    tar -xzf $pv_tarball
-fi

packages/ttk/env.sh

@@ -12,22 +12,10 @@ if [ -z "$INSTALLER_TTK_ENV_SH" ]; then
     build_dir_name=build-$version
 
     # TTK needs a patched version of Paraview
-    pv_version=5.8.1
-    pv_source_dir_name=ParaView-v$pv_version
+    pv_version=5.9.0
 
     declare -A cmake_vars
 
+    module load paraview/$pv_version
     installer_init
-
-    if [ -x $prefix/bin/pvpython ]; then
-	# This only possible after install; only needed for module
-	pvpython_sitepackage_dir=$($prefix/bin/pvpython $thisdir/get_site_packages.py)
-    fi
-    if [ -f $prefix/bin/activate ]; then
-	python_sitepackage_dir=$(
-	    ( . /lustre/home/shared/dc118/sw/ttk/0.9.9/bin/activate;
-	      python -c 'import sysconfig; print(sysconfig.get_path("platlib"))'
-	    )
-	)
-    fi
 fi

platforms/cirrus/fireapp/config.sh

@@ -5,6 +5,6 @@ module load netcdf/4.7.4
 module load gdal/3.1.0
 module load conan/1.31.3
 module load mpt/2.22
-module load paraview/5.9.0
-module load python/3.8.5
+module load ttk/0.9.9
+
 cmake_vars[MPI_CXX_LINK_FLAGS]=-Wl,-rpath=$MPI_ROOT/lib

platforms/cirrus/ttk/config.sh

@@ -3,3 +3,6 @@ module load gcc/6.3.0
 module load python/3.8.5
 module load boost/1.73.0
 module load eigen/3.3.9
+export Eigen3_ROOT=$EIGEN_DIR
+# Ensure that TTK and its python modules are installed consistently
+cmake_vars[CMAKE_INSTALL_LIBDIR]=lib

platforms/cirrus/ttk/modtemplate.tcl

@@ -1,18 +1,16 @@
 #%Module
-module-whatis "TTK version ${version} including Paraview ${pv_version}"
+module-whatis "TTK version ${version}"
 
-setenv TTK_DIR "${prefix}"
-prepend-path PATH "${prefix}/bin"
-
-setenv VIRTUAL_ENV "${prefix}"
-prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins"
-
-# This is needed to ensure that TTK/Paraview work in another venv with
-# this module loaded.
-prepend-path PYTHONPATH "${pvpython_sitepackage_dir}"
-# And this one is for numpy
-prepend-path PYTHONPATH "${python_sitepackage_dir}"
+if {![is-loaded paraview]} {
+    module load paraview/${pv_version}
+}
 
 if {![is-loaded boost]} {
     module load boost/1.73.0
 }
+
+setenv TTK_DIR "${prefix}"
+prepend-path PATH "${prefix}/bin"
+
+prepend-path PV_PLUGIN_PATH "${prefix}/bin/plugins"
+prepend-path PYTHONPATH "${prefix}/lib/python3.8/site-packages"