diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..99b40c8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.image filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.xdmf filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 0a78929..e05c5f2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ bin/ compile_commands.json test_data/ build/ +benchmarking/pyvenv* doc/_build/ doc/_static/pfire_tutorial_files.zip gitstate.hpp @@ -80,3 +81,18 @@ MANIFEST *.exe *.out *.app + +# IDEs +.cproject +.project +.pydevproject +.settings/ +src/.cproject +src/.project + + +external_libs/ + +testdata/integration/regression/brain_2d/xdmf + +benchmarking/brain2d/brain2d.html diff --git a/CMakeLists.txt b/CMakeLists.txt index 8691bd1..8dd1eef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ cmake_minimum_required(VERSION 3.7.0) project(pFIRE) +include(CMakePrintHelpers) if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}") message(FATAL_ERROR "In-source builds are not permitted. Make a separate folder for " @@ -26,6 +27,9 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED on) set(CMAKE_EXPORT_COMPILE_COMMANDS on) +#[=[ +#add_compile_options(-W -Wall -Werror) +#]=] cmake_policy(SET CMP0074 NEW) if( NOT CMAKE_BUILD_TYPE ) @@ -35,9 +39,10 @@ if( NOT CMAKE_BUILD_TYPE ) endif() set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wuninitialized -O0 -ggdb -march=native -fstack-protector-strong") -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wall -O3 -march=native -fno-fast-math") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -ggdb") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wall -O3 -march=native -fno-fast-math") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -ggdb") +# set( FIND_LIBRARY_USE_LIB64_PATHS property TRUE) option(DEBUG_VERBOSE "Verbose debug outputs") if(DEBUG_VERBOSE) add_definitions(-DDEBUG_VERBOSE) @@ -72,6 +77,10 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${pFIRE_SOURCE_DIR}/bin) # Ensure we can find FindPETSc and friends list(APPEND CMAKE_MODULE_PATH ${pFIRE_SOURCE_DIR}/cmake-modules) +# Define flags to enable dependencies +option(ENABLE_OpenImageIO "Enable Linking OpenImageIO library" [TRUE]) +option(ENABLE_DCMTK "Enable Linking DCMTK library" [TRUE]) + find_package(MPI REQUIRED) include_directories(${MPI_INCLUDES}) set(EXTRA_LIBS ${EXTRA_LIBS} ${MPI_LIBRARIES}) @@ -94,19 +103,67 @@ find_package(Boost REQUIRED COMPONENTS filesystem program_options) include_directories(${BOOST_INCLUDE_DIRS}) set(EXTRA_LIBS ${EXTRA_LIBS} ${Boost_LIBRARIES}) -find_package(DCMTK) +if(ENABLE_DCMTK) +find_package(DCMTK CONFIG) +endif(ENABLE_DCMTK) + if(DCMTK_FOUND) -include_directories(${DCMTK_INCLUDES}) -set(EXTRA_LIBS ${EXTRA_LIBS} ${DCMTK_LIBRARIES}) - add_definitions(-DUSE_DCMTK) + message("DCMTK found") + include_directories(${DCMTK_INCLUDES}) + # FIX the following line to work with DCMTK not in single library + # set(EXTRA_LIBS ${EXTRA_LIBS} ${DCMTK_CMAKE_INSTALL_PREFIX}/lib64/libdcmtk.so )#${DCMTK_CMAKE_INSTALL_PREFIX}/${DCMTK_LIBRARIES}) + set(EXTRA_LIBS ${EXTRA_LIBS} ${DCMTK_CMAKE_INSTALL_PREFIX}/${DCMTK_LIBRARIES}) + add_definitions(-DUSE_DCMTK) + include_directories(${DCMTK_INCLUDE_DIRS}) + cmake_print_variables(DCMTK_INCLUDES) + cmake_print_variables(DCMTK_LIBRARIES) + #cmake_print_variables(DCMTK_CMAKE_INSTALL_PREFIX) + + endif(DCMTK_FOUND) -find_package(OpenImageIO) -if(OPENIMAGEIO_FOUND) -include_directories(${OPENIMAGEIO_INCLUDE_DIR}) -set(EXTRA_LIBS ${EXTRA_LIBS} ${OPENIMAGEIO_LIBRARIES}) - add_definitions(-DUSE_OIIO) -endif(OPENIMAGEIO_FOUND) +#list(APPEND CMAKE_FIND_ROOT_PATH "... insigneo-pfire/docker-pfire/singularity/fix_oiio_dcmtk/oiio/dist/lib64/cmake/OpenImageIO/") + + +if(ENABLE_OpenImageIO) +find_package(OpenImageIO CONFIG) +endif(ENABLE_OpenImageIO) + +cmake_print_variables(OpenImageIO_FOUND) + + + +if(OpenImageIO_FOUND AND ENABLE_OpenImageIO) + + #cmake_print_variables(OpenImageIO_INCLUDE_DIR) + cmake_print_variables(OpenImageIO_LIBRARIES) + #cmake_print_variables(OpenImageIO_LIB_DIR) + # OpenImageIO cmake config is missing library names so declaring here until they fix it +#try modern +set (OpenImageIO_LIBRARIES "${OpenImageIO_LIB_DIR}/libOpenImageIO_Util.so;${OpenImageIO_LIB_DIR}/libOpenImageIO.so") +#try modern +include_directories(${OpenImageIO_INCLUDE_DIR}) + cmake_print_variables(OpenImageIO_LIBRARIES) + #set(EXTRA_LIBS ${EXTRA_LIBS} ${OpenImageIO_LIBRARIES} ) + #add_definitions(-DUSE_OIIO) + + +else(OpenImageIO_FOUND AND ENABLE_OpenImageIO) + message(STATUS "OPENIMAGEIO_FOUND ELSE branch") + + #include_directories(${OpenImageIO_INCLUDE_DIR}) + # danielta set(EXTRA_LIBS ${EXTRA_LIBS} ${OpenImageIO_LIBRARIES} ) #/usr/local/oiio/lib64/libOpenImageIO.a /usr/local/oiio/lib64/libOpenImageIO_Util.a) + + #add_definitions(-DUSE_OIIO) + +endif(OpenImageIO_FOUND AND ENABLE_OpenImageIO) +# include(GNUInstallDirs) +cmake_print_variables(EXTRA_LIBS) + +#find_library(JPEG_LIBRARY NAMES jpeg jpg libjpeg REQUIRED NO_SYSTEM_ENVIRONMENT_PATH PATHS /usr/local/oiio/lib64) +#cmake_print_variables(JPEG_LIBRARY) + +#set(EXTRA_LIBS ${EXTRA_LIBS} ${OpenImageIO_LIBRARIES}) add_subdirectory(${pFIRE_SOURCE_DIR}/src) diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/benchmarking/README.md b/benchmarking/README.md new file mode 100644 index 0000000..873fe95 --- /dev/null +++ b/benchmarking/README.md @@ -0,0 +1,73 @@ +## pFIRE Benchmark suite + +The benchmark suite provides a framework to perform regression and comparison tests of pFire/ShiRT over a test cases organised in folders. + + +# Installation + +The suite is written in Python and can be installed in the standard way. +We recommend to use virtual environments as below: + + + +```bash + # Create virtual environment + + cd pFIRE/benchmarking + + python3 -m venv pyvenv3 + + + # Activate + + source pyvenv3/bin/activate + + # Upgrade pip + pip install --upgrade pip + + # Install dependencies + + pip install -r requirements.txt + + ##python setup.py install . + pip install . + + # add folder to pythonpath + + export PYTHONPATH=`pwd`/:$PYTHONPATH + + + ../../pFIRE/testdata/integration/regression + + # When not needed any more deactivate the virtual environment + + deactivate + + +``` + + +add pfire binary to PATH + +export PATH=//:$PATH + +# Usage + +``` +pfire-integration-test --pfire-executable=/ +``` + +It is optional to provide the pFIRE executable filename. Apptainer allows to use the container as an executable (a wrapper around a container binary configured un the Runscript section) + + +* create a testconf file in each test folder +* pfire_benchmark /mytest.testconf + + + + +TODO add testconf syntax and use cases + + + + diff --git a/benchmarking/pfire_benchmarking/__main__.py b/benchmarking/pfire_benchmarking/__main__.py index 17e1a1c..c681fcf 100644 --- a/benchmarking/pfire_benchmarking/__main__.py +++ b/benchmarking/pfire_benchmarking/__main__.py @@ -12,8 +12,15 @@ def _parse_args(): parser = argparse.ArgumentParser(description="Run pFIRE integration tests") parser.add_argument('dir', nargs='?', default=os.getcwd(), metavar="datadir", - help="Test data directory, will be inspected recursively") + help="Test data directory will be inspected recursively") + + parser.add_argument('--pfire-executable', nargs='?', default="pfire", metavar="pfire_exec_filename", + dest="pfire_exec_filename", + help="pFIRE executable filepath (filename with full path)") + + parser.add_argument('--output', '-o', default='.', metavar="outdir", + dest="output_dir", help="Path at which to output results") return parser.parse_args() @@ -23,13 +30,15 @@ def main(): """ Run the testsuite over a directory tree """ args = _parse_args() - - testsuite = TestDespatcher(output_dir=args.output) + print(args) + print("pfire_exec_filename=", args.pfire_exec_filename ) + + testsuite = TestDespatcher(output_dir=args.output_dir, pfire_exec_filename=args.pfire_exec_filename ) if not os.path.exists(args.dir): print("Error, path {} does not exist".format(args.dir)) return - + # Add tests or tests in directory tree if os.path.isfile(args.dir): testsuite.add_test(args.dir) else: diff --git a/benchmarking/pfire_benchmarking/analysis_routines.py b/benchmarking/pfire_benchmarking/analysis_routines.py index 47a9a83..abbf4f1 100644 --- a/benchmarking/pfire_benchmarking/analysis_routines.py +++ b/benchmarking/pfire_benchmarking/analysis_routines.py @@ -48,8 +48,21 @@ def calculate_mutual_information(data1, data2, resolution=50, Returns a tuple of MI(X,Y), H(X), H(Y), H(X,Y) """ jointmax = max(data1.max(), data2.max()) + + # First calculate probability density - bin_edges = np.linspace(0, 1, num=resolution) + bin_edges = np.linspace(0, 1, num=resolution)#, dtype=object) + + print ("shape(data1.flatten())=" + str(np.shape(data1.flatten()))) + print ("shape(data2.flatten())=" + str(np.shape(data2.flatten()))) + + print ("shape(data1)=" + str(np.shape(data1))) + print ("shape(data2)=" + str(np.shape(data2))) + + print ("shape(bin_edges)=" + str(np.shape(bin_edges))) + + + prob1_2, _, _ = np.histogram2d(data1.flatten()/jointmax, data2.flatten()/jointmax, bins=bin_edges, density=True) @@ -101,11 +114,25 @@ def compare_image_results(fixed_path, moved_path, accepted_path, else: fig_dir = os.path.normpath('.') - mi_start = calculate_proficiency(fixed_path, moved_path) - mi_accepted = calculate_proficiency(fixed_path, accepted_path) - mi_pfire = calculate_proficiency(fixed_path, pfire_path) + + print("fixed_path="+ fixed_path) + print("moved_path="+ moved_path) + print("pfire_path="+ pfire_path) + print("accepted path="+ accepted_path) + + print("MI fixed_path vs moved_path=\n"+ fixed_path + "\n" + moved_path) + mi_start = calculate_proficiency(fixed_path, moved_path) + + print("MI fixed_path vs accepted_path=\n"+ fixed_path + "\n" + accepted_path) + mi_accepted = calculate_proficiency(fixed_path, accepted_path) + + print("MI fixed_path vs pfire_path=\n"+ fixed_path + "\n" +pfire_path) + mi_pfire = calculate_proficiency(fixed_path, pfire_path) + + print("MI accepted_path vs pfire_path=\n"+ accepted_path + "\n" +pfire_path) mi_comparison = calculate_proficiency(accepted_path, pfire_path) + res_table = [("Normalized mutual information (proficiency):", ""), ("Fixed vs. Moved:", "{:.3f}".format(mi_start.mi)), ("{} vs. Fixed:".format(cmpname), "{:.3f}".format(mi_accepted.mi)), diff --git a/benchmarking/pfire_benchmarking/application_routines.py b/benchmarking/pfire_benchmarking/application_routines.py index 030b095..9ddd524 100644 --- a/benchmarking/pfire_benchmarking/application_routines.py +++ b/benchmarking/pfire_benchmarking/application_routines.py @@ -37,7 +37,7 @@ def __init__(self, *args, **kwargs): self.pfire_mask_path = None self.pfire_reg_path = None self.pfire_map_path = None - + self.pfire_exec_filename=None def run_pfire(self, config_path, comm_size=1): """ Run pFIRE using provided config file @@ -52,41 +52,52 @@ def run_pfire(self, config_path, comm_size=1): with open(config_path, 'r') as fh: config = ConfigObj(fh) print("Running pFIRE on {}".format(pfire_config)) - + self.pfire_fixed_path = os.path.join(pfire_workdir, config['fixed']) self.pfire_moved_path = os.path.join(pfire_workdir, config['moved']) - try: - self.pfire_mask_path = os.path.join(pfire_workdir, config['mask']) - except KeyError: - pass - + + # TODO Check Registered image is in xdmf format + # TODO Check registration Map is in xdmf format + + # Log file name self.pfire_logfile = os.path.join( pfire_workdir, "{}_pfire.log".format(os.path.splitext(pfire_config)[0])) - with open(self.pfire_logfile, 'w') as logfile: - pfire_args = ['pfire', pfire_config] + + # Run pFIRE + print("pFIRERunnerMixin.run_pfire:", self.pfire_exec_filename) + with open(self.pfire_logfile, 'w') as logfile: + + #pfire_args = ['pfire', pfire_config] + pfire_args = [self.pfire_exec_filename, pfire_config] res = sp.run(pfire_args, cwd=pfire_workdir, stdout=logfile, stderr=logfile) - if res.returncode != 0: raise RuntimeError("Failed to run pFIRE, check log for details: {}" "".format(self.pfire_logfile)) - with open(self.pfire_logfile, 'r') as logfile: - for line in logfile: - if line.startswith("Saved registered image to "): - reg_path = line.replace("Saved registered image to ", - "").strip() - self.pfire_reg_path = os.path.join(pfire_workdir, reg_path) - elif line.startswith("Saved map to "): - map_path = line.replace("Saved map to ", "").strip() - self.pfire_map_path = os.path.join(pfire_workdir, map_path) + # Get pFIRE registered image path filename + try: + self.pfire_reg_path = os.path.join(pfire_workdir, config['registered']) + print("self.pfire_reg_path="+ self.pfire_reg_path) + except KeyError: + pass + + # Get pFIRE registered map path filename + try: + self.pfire_map_path = os.path.join(pfire_workdir, config['map']) + print("self.pfire_map_path="+ self.pfire_map_path) + except KeyError: + pass + + try: + self.pfire_mask_path = os.path.join(pfire_workdir, config['mask']) + except KeyError: + pass if not (self.pfire_reg_path or self.pfire_map_path): - raise RuntimeError("Failed to extract result path(s) from log") - - - + raise RuntimeError("Failed to find registered image or map named in config file") + class ShIRTRunnerMixin: """ Mixin class to provide a ShIRT runner interface accepted a pFIRE config @@ -126,6 +137,9 @@ def run_shirt(self, config_path): """ config = self._build_shirt_config(config_path) work_dir, config_file = os.path.split(config_path) + + # FIXME workdire is empty if config path is relatove. make it absolute + print("Running ShIRT on {}".format(config_file)) self.shirt_reg_path = 'shirt_{}_registered.image'.format(self.name) @@ -159,12 +173,35 @@ def run_shirt(self, config_path): 'Registered', self._strip_imgname(self.shirt_reg_path), 'Map', self._strip_imgname(self.shirt_map_path)] + + + # ADd test FIXME + #SHIRT_BIN + + shirt_env = {} + SHIRT_DIR="/home/tartarini/LOCAL/app/Shirt/"#~/LOCAL/app/Shirt/bin/ShIRT + + #shirt_env['DATAPATH'] = '/home/tartarini/fixing_bugs_pfire/pFIRE/benchmarking/brain2d/' + shirt_env['DATAPATH'] = './' + shirt_env['DISPLAYPATH'] = shirt_env['DATAPATH'] +"display/" + + #SHIRT_DIR=/home/tartarini/ShIRT/ + shirt_env['SYSPATH']= SHIRT_DIR+"bin/" + shirt_env['SCRIPTPATH']=SHIRT_DIR+"Scripts/" + shirt_env['PATH']= shirt_env['SYSPATH'] + #export PATH=$PATH:$SHIRT_DIR/1.1/bin/ + + self.shirt_logfile = os.path.join( work_dir, "{}_shirt.log".format(os.path.splitext(config_file)[0])) with open(self.shirt_logfile, 'w') as logfile: - sp.run(['ShIRT', 'setpath', 'DataPath', '.'], cwd=work_dir) - res = sp.run(shirt_args, stdout=logfile, stderr=logfile, - cwd=work_dir) + print("work_dir=" + work_dir) + print(shirt_env) + + sp.run(['ShIRT', 'setpath', 'DataPath', '.'])#, cwd=work_dir) + + res = sp.run(shirt_args, stdout=logfile, stderr=logfile, env=shirt_env) + #cwd=work_dir) if res.returncode != 0: raise RuntimeError("Failed to run ShIRT, check log for details: {}" diff --git a/benchmarking/pfire_benchmarking/cross_validate.py b/benchmarking/pfire_benchmarking/cross_validate.py index 8baee9e..9b72c4d 100644 --- a/benchmarking/pfire_benchmarking/cross_validate.py +++ b/benchmarking/pfire_benchmarking/cross_validate.py @@ -49,6 +49,9 @@ def __init__(self, pfire_config, name=None, output_path=None): super().__init__(pfire_config, name=name, output_path=output_path) self.run_errstring = None print("output_path: {}".format(self.output_path)) + + print("\n pFIRE config=" +pfire_config) + exit def run(self): """ Run pfire against provided config diff --git a/benchmarking/pfire_benchmarking/image_routines.py b/benchmarking/pfire_benchmarking/image_routines.py index 501ede5..e35c9fa 100644 --- a/benchmarking/pfire_benchmarking/image_routines.py +++ b/benchmarking/pfire_benchmarking/image_routines.py @@ -6,19 +6,28 @@ from functools import partial import numpy as np import h5py - +import sys import skimage.io as skio import flannel.io as fio +def __LINE__(): # FIXME remove + return str(sys._getframe(1).f_lineno) + def load_image(imagepath): """ Attempt to load image with known image loaders """ + print("\n Load_image = {}".format(imagepath)) + print("imagepath" + imagepath) + image = None - imgloaders = [load_pfire_image, fio.load_image, - partial(skio.imread, as_gray=True)] + imgloaders = [load_pfire_image, fio.load_image + ,partial(skio.imread, as_gray=True) + ] + for loader in imgloaders: + print(" trying loader="+ str(imgloaders.index(loader))) try: image = loader(imagepath) except (ValueError, OSError): @@ -41,6 +50,9 @@ def load_map(mappath): shirtloader = lambda path: fio.load_map(path)[0][0:3] maploaders = [load_pfire_map, shirtloader] + print("Loop over map loaders") + print("mappath={}", mappath) + for loader in maploaders: try: data = loader(mappath) @@ -57,12 +69,25 @@ def load_map(mappath): def load_pfire_image(imagepath): """Load pFIRE image """ + print("\nload_pfire_image filepath={}".format(imagepath) ) + imagepath+=":/registered" # FIXME data group should be read from xdmf file + filename, group = [x.strip() for x in imagepath.split(':')] - if filename.endswith(".xdmf"): + print("\nload_pfire_image filename={}".format(filename) ) + print("\nload_pfire_image group={}".format(group )) + print("load_pfire_image() line="+__LINE__()) + + # Workaround to bypass issue CBM-66 Xdmf lib bug. TO read metadata from XDMF + if filename.endswith(".xdmf"): filename += ".h5" - + + + print("\nload_pfire_image filename={}".format(filename) ) + print("load_pfire_image() line="+__LINE__()) + with h5py.File(filename, 'r') as fh: - imgdata = np.asarray(fh[group]) + print(list(fh.keys())) + imgdata = np.asarray(fh['registered']) return imgdata @@ -70,13 +95,21 @@ def load_pfire_image(imagepath): def load_pfire_map(imagepath): """Load pFIRE map """ + imagepath+=":/map" # FIXME + + print("\nload_pfire imagepath={}", imagepath) filename, group = [x.strip() for x in imagepath.split(':')] if filename.endswith(".xdmf"): filename += ".h5" + print("\nload_pfire_map={}", load_pfire_map) + + print("\nload_pfire_filename={}", filename) + imgdata = [] with h5py.File(filename, 'r') as fh: for dxn in ['x', 'y', 'z']: + print("\n dxn={}"+str(dxn)) try: imgdata.append(np.asarray(fh["{}/{}".format(group, dxn)])) except KeyError: diff --git a/benchmarking/pfire_benchmarking/regression_validate.py b/benchmarking/pfire_benchmarking/regression_validate.py index 2828c11..83b0f02 100644 --- a/benchmarking/pfire_benchmarking/regression_validate.py +++ b/benchmarking/pfire_benchmarking/regression_validate.py @@ -34,7 +34,8 @@ def main(): args = parse_args() - test = RegressionTest(args.pfire_config, name=args.test_name, + test = RegressionTest(args.pfire_config, + name=args.test_name, accepted_image=args.accepted_image, accepted_map=args.accepted_map) @@ -49,18 +50,24 @@ class RegressionTest(TestInstance, pFIRERunnerMixin): """ def __init__(self, pfire_config, name=None, accepted_image=None, - accepted_map=None, output_path=None): + accepted_map=None, output_path=None, + pfire_exec_filename='pfire'): + # Check this before we go anywhere else if not (accepted_map or accepted_image): raise ValueError("At least one of accepted_image or accepted_map " "must be provided") super().__init__(pfire_config, name=name, output_path=output_path) - + self.pfire_exec_filename=pfire_exec_filename + + self.accepted_image_path = accepted_image - self.accepted_map_path = accepted_map + self.accepted_map_path = accepted_map self.run_errstring = None + print("\n pFIRE config=" +pfire_config) + def run(self): """ Run pfire against provided config diff --git a/benchmarking/pfire_benchmarking/testdespatcher.py b/benchmarking/pfire_benchmarking/testdespatcher.py index 08cb829..f98f720 100644 --- a/benchmarking/pfire_benchmarking/testdespatcher.py +++ b/benchmarking/pfire_benchmarking/testdespatcher.py @@ -19,18 +19,21 @@ class TestDespatcher: test_types = {"regression": RegressionTest, "comparison": ComparisonTest} - def __init__(self, output_dir=None): + def __init__(self, output_dir=None, pfire_exec_filename='pfire'): self.tests = [] if output_dir: self.output_dir = os.path.normpath(output_dir) else: self.output_dir = os.path.normpath('.') + + self.pfire_exec_filename=pfire_exec_filename def add_test(self, testconfig_path): - """ Add a new test by parsing configfile + """ Add a new test by parsing test config file in INI format """ testdir = os.path.dirname(testconfig_path) + # opening explicitly causes failure on file nonexistence with open(testconfig_path, 'r') as fh: testconfig = ConfigObj(fh) @@ -64,7 +67,9 @@ def add_test(self, testconfig_path): testkwargs['name'] = None testkwargs['output_path'] = self.output_dir + testkwargs['pfire_exec_filename'] =self.pfire_exec_filename + # Get location of pFIRE configuration file used to generate the outputs to test pfire_config_path = os.path.join(testdir, testconfig['pfire_config']) try: test = testiniter(pfire_config_path, **testkwargs) @@ -72,6 +77,7 @@ def add_test(self, testconfig_path): raise RuntimeError("A pFIRE configuration file (\"pfire_config\") " "must be specified") + # Add found test with config and pFire config to the list of tests print("Found \"{}\"".format(test.name)) self.tests.append(test) diff --git a/benchmarking/requirements.txt b/benchmarking/requirements.txt new file mode 100644 index 0000000..da2552a --- /dev/null +++ b/benchmarking/requirements.txt @@ -0,0 +1,27 @@ +cached-property==1.5.2 +configobj==5.0.6 +cycler==0.10.0 +decorator==4.4.2 +docutils==0.17.1 +dpath==1.4.2 +#./../utils/flannel +git+https://github.com/insigneo-pfire/flannel.git +h5py==3.1.0 +imageio==2.9.0 +kiwisolver==1.3.1 +matplotlib==3.3.4 +networkx==2.5.1 +numpy +Pillow==8.3.2 +pyparsing==2.4.7 +PyQt5==5.15.4 +PyQt5-Qt5==5.15.2 +PyQt5-sip==12.9.0 +python-dateutil==2.8.2 +PyWavelets==1.1.1 +PyYAML==5.4.1 +scikit-image +scipy +six==1.16.0 +tabulate==0.8.9 +tifffile==2020.9.3 diff --git a/benchmarking/setup.py b/benchmarking/setup.py index 340313a..4d7796f 100644 --- a/benchmarking/setup.py +++ b/benchmarking/setup.py @@ -11,9 +11,14 @@ packages=find_packages(''), include_package_data=True, install_requires=['numpy', 'scipy', 'h5py', 'matplotlib', 'flannel', - 'configObj'], + 'configObj', 'docutils', 'tabulate', 'scikit-image'], entry_points={ 'console_scripts': [ 'pfire-integration-test = pfire_benchmarking.__main__:main' ], - }) + }, + #package_dir={"": "pfire_benchmarking"}, + #packages=find_packages(where="pfire_benchmarking"), + python_requires=">=3.6", + + ) diff --git a/doc/_templates/footer.html b/doc/_templates/footer.html deleted file mode 100644 index c93c287..0000000 --- a/doc/_templates/footer.html +++ /dev/null @@ -1,55 +0,0 @@ -
- {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} - - {% endif %} - -
- -
-

- {%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} - {%- else %} - © 2018, University of Sheffield. -

-

- Development of pFIRE is supported by the European Union’s Horizon 2020 - research and innovation programme under grant agreement No 675451. - {%- endif %} - {%- endif %} - - {%- if build_id and build_url %} - {% trans build_url=build_url, build_id=build_id %} - - Build - {{ build_id }}. - - {% endtrans %} - {%- elif commit %} - {% trans commit=commit %} - - Revision {{ commit }}. - - {% endtrans %} - {%- elif last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {%- endif %} - -

-
- - {%- if show_sphinx %} - {% trans %}Built with Sphinx using a theme provided by Read the Docs{% endtrans %}. - {%- endif %} - - {%- block extrafooter %} {% endblock %} - -
diff --git a/doc/index.html b/doc/index.html deleted file mode 100644 index 082b421..0000000 --- a/doc/index.html +++ /dev/null @@ -1,8 +0,0 @@ - - - pFIRE - - - - - diff --git a/oiio.env b/oiio.env new file mode 100644 index 0000000..722bff6 --- /dev/null +++ b/oiio.env @@ -0,0 +1,12 @@ +export LD_LIBRARY_PATH=/usr/local/cuda-11.5/lib64:/usr/local/cuda-11.5/lib64::/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib +export LIBTIFF_ROOT=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist +export pybind11_ROOT=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist +export pugixml_ROOT=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist +export LD_LIBRARY_PATH=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib:/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib64:/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib:/usr/local/cuda-11.5/lib64:/usr/local/cuda-11.5/lib64::/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib + +export LD_LIBRARY_PATH=/usr/local/cuda-11.5/lib64:/usr/local/cuda-11.5/lib64::/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib +export LIBTIFF_ROOT=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist + +export LIBTIFF_ROOT=/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist +export LD_LIBRARY_PATH=/usr/local/cuda-11.5/lib64:/usr/local/cuda-11.5/lib64::/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/external_libs/oiio-master/build/ext/dist/lib + diff --git a/singularity_sandbox/v0.4.0 b/singularity_sandbox/v0.4.0 new file mode 160000 index 0000000..fac19a7 --- /dev/null +++ b/singularity_sandbox/v0.4.0 @@ -0,0 +1 @@ +Subproject commit fac19a78fc654a2b5c46f96f40bbac5d54b851b4 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 537719d..c700e24 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,27 +13,46 @@ # See the License for the specific language governing permissions and # limitations under the License. -file(GLOB libpFIRESOURCES +file(GLOB pfire_libSOURCES "*.cpp") -list(REMOVE_ITEM libpFIRESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/sandbox.cpp") -list(REMOVE_ITEM libpFIRESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/pfire.cpp") +list(REMOVE_ITEM pfire_libSOURCES "${CMAKE_CURRENT_SOURCE_DIR}/sandbox.cpp") +list(REMOVE_ITEM pfire_libSOURCES "${CMAKE_CURRENT_SOURCE_DIR}/pfire.cpp") -if(NOT OPENIMAGEIO_FOUND) - list(REMOVE_ITEM libpFIRESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/oiioloader.cpp") - list(REMOVE_ITEM libpFIRESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/oiiowriter.cpp") -endif(NOT OPENIMAGEIO_FOUND) +if(NOT OpenImageIO_FOUND) + list(REMOVE_ITEM pfire_libSOURCES "${CMAKE_CURRENT_SOURCE_DIR}/oiioloader.cpp") + list(REMOVE_ITEM pfire_libSOURCES "${CMAKE_CURRENT_SOURCE_DIR}/oiiowriter.cpp") +endif(NOT OpenImageIO_FOUND) if(NOT DCMTK_FOUND) - list(REMOVE_ITEM libpFIRESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/dcmloader.cpp") + list(REMOVE_ITEM pfire_libSOURCES "${CMAKE_CURRENT_SOURCE_DIR}/dcmloader.cpp") endif(NOT DCMTK_FOUND) -add_library(libpfire ${libpFIRESOURCES}) -set_target_properties(libpfire PROPERTIES PREFIX "") -target_link_libraries(libpfire ${EXTRA_LIBS}) +cmake_print_variables(${pfire_libSOURCES}) +cmake_print_variables(${EXTRA_LIBS} ) +cmake_print_variables(${DCMTK_LIBRARIES}) + + +#set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +add_library(pfire_lib SHARED ${pfire_libSOURCES} ) + +#set_target_properties(pfire_lib PROPERTIES PREFIX "") +#target_link_libraries(pfire_lib ${EXTRA_LIBS} ) # ${DCMTK_LIBRARIES}) +#target_link_libraries(pfire_lib PUBLIC ${EXTRA_LIBS}) +#target_link_libraries(pfire_lib PUBLIC OpenImageIO::OpenImageIO )#${OpenImageIO_LIBRARIES}) +#target_link_libraries(pfire_lib PUBLIC OpenImageIO::OpenImageIO DCMTK::dcmtk ) #${OpenImageIO_LIBRARIES}) +#target_link_libraries(pfire_lib PUBLIC DCMTK::dcmtk) +# alternative approach based on modern CMAKE +#target_link_libraries(pfire_lib ${DCMTK_LIBRARIES} ${OpenImageIO_LIBRARIES}) +target_link_libraries(pfire_lib DCMTK::DCMTK ${OpenImageIO_LIBRARIES} ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${PETSC_LIBRARIES} ${HDF5_LIBRARIES}) + add_executable(pfire pfire.cpp) set_target_properties(pfire PROPERTIES LINKER_LANGUAGE CXX) -target_link_libraries(pfire libpfire) +target_link_libraries(pfire PUBLIC pfire_lib) + install(TARGETS pfire DESTINATION bin) + +install(TARGETS pfire_lib DESTINATION lib) diff --git a/src/elasticregistration.hpp b/src/elasticregistration.hpp index bc7149c..daee9bf 100644 --- a/src/elasticregistration.hpp +++ b/src/elasticregistration.hpp @@ -15,6 +15,7 @@ #include "baseconfiguration.hpp" #include "types.hpp" +#include class ElasticRegistration { public: diff --git a/src/shirtloader.hpp b/src/shirtloader.hpp index 3e6c37a..1d4150f 100644 --- a/src/shirtloader.hpp +++ b/src/shirtloader.hpp @@ -16,6 +16,7 @@ #ifndef SHIRTLOADER_HPP #define SHIRTLOADER_HPP +#include #include "baseloader.hpp" #include "types.hpp" @@ -39,12 +40,12 @@ class ShIRTLoader: public BaseLoader { using shirt_header_dtype = int32_t; using image_data_dtype = float; - static constexpr MPI_Datatype image_data_mpi_type = MPI_FLOAT; + MPI_Datatype image_data_mpi_type = MPI_FLOAT; static constexpr integer image_header_length = 4; static constexpr integer image_header_bytes = image_header_length * sizeof(shirt_header_dtype); using mask_data_dtype = int16_t; - static constexpr MPI_Datatype mask_data_mpi_type = MPI_SHORT; + MPI_Datatype mask_data_mpi_type = MPI_SHORT; static constexpr integer mask_header_length = 3; static constexpr integer mask_header_bytes = mask_header_length * sizeof(shirt_header_dtype); diff --git a/src/tmatrix.cpp b/src/tmatrix.cpp index a17815c..1dfa91a 100644 --- a/src/tmatrix.cpp +++ b/src/tmatrix.cpp @@ -207,9 +207,10 @@ std::pair calculate_entries(integer row, integer col, intege const std::vector& gradient_data, const Vec& difference_data, MPI_Comm comm) { -#ifdef DEBUG_CHECKS - throw_if_idof_inconsistent(comm, idof) -#endif //DEBUG_CHECKS + +// danielta #ifdef DEBUG_CHECKS + // danielta throw_if_idof_inconsistent(comm, idof) + // danielta #endif //DEBUG_CHECKS int commsize, rank; MPI_Comm_size(comm, &commsize); diff --git a/src/types.hpp b/src/types.hpp index fe60da2..f6446a7 100644 --- a/src/types.hpp +++ b/src/types.hpp @@ -45,7 +45,7 @@ class BaseLoader; class BaseWriter; class ConfigurationBase; -constexpr MPI_Comm COMM = MPI_COMM_WORLD; +static MPI_Comm COMM = MPI_COMM_WORLD; // Useful typedefs using integer = PetscInt; diff --git a/testdata/.gitignore b/testdata/.gitignore new file mode 100644 index 0000000..4332973 --- /dev/null +++ b/testdata/.gitignore @@ -0,0 +1,4 @@ +moved_8000x10000.image +fixed_8000x10000.image +*.html + diff --git a/testdata/integration/comparison/brain_2d/TEST/fixed.image b/testdata/integration/comparison/brain_2d/TEST/fixed.image new file mode 100644 index 0000000..19ef18a Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/fixed.image differ diff --git a/testdata/integration/comparison/brain_2d/TEST/map.map b/testdata/integration/comparison/brain_2d/TEST/map.map new file mode 100644 index 0000000..23094de Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/map.map differ diff --git a/testdata/integration/comparison/brain_2d/TEST/mask.mask b/testdata/integration/comparison/brain_2d/TEST/mask.mask new file mode 100644 index 0000000..4d27e58 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/mask.mask differ diff --git a/testdata/integration/comparison/brain_2d/TEST/moved.image b/testdata/integration/comparison/brain_2d/TEST/moved.image new file mode 100644 index 0000000..bb89593 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/moved.image differ diff --git a/testdata/integration/comparison/brain_2d/TEST/registered.image b/testdata/integration/comparison/brain_2d/TEST/registered.image new file mode 100644 index 0000000..41a44c9 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/registered.image differ diff --git a/testdata/integration/comparison/brain_2d/TEST/test.image.image b/testdata/integration/comparison/brain_2d/TEST/test.image.image new file mode 100644 index 0000000..41a44c9 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/TEST/test.image.image differ diff --git a/testdata/integration/comparison/brain_2d/brain_2d.conf b/testdata/integration/comparison/brain_2d/brain_2d.conf index 794ac83..701a6d6 100644 --- a/testdata/integration/comparison/brain_2d/brain_2d.conf +++ b/testdata/integration/comparison/brain_2d/brain_2d.conf @@ -1,4 +1,7 @@ fixed=fixed.image moved=moved.image +#map=map.xmdf +#registered=registered.xdmf:/registered nodespacing=10 -save_intermediate_frames=true +save_intermediate_frames=false +#registered=registered.h5 diff --git a/testdata/integration/comparison/brain_2d/brain_2d.testconf b/testdata/integration/comparison/brain_2d/brain_2d.testconf new file mode 100644 index 0000000..faae27d --- /dev/null +++ b/testdata/integration/comparison/brain_2d/brain_2d.testconf @@ -0,0 +1,5 @@ +name = brain2d +type = comparison +pfire_config = brain_2d.conf +#accepted_image = # string: the image to compare with ("regression" mode only) +#accepted_map = # string: the map to compare with ("regression" mode only) diff --git a/testdata/integration/comparison/brain_2d/default_mask.mask b/testdata/integration/comparison/brain_2d/default_mask.mask new file mode 100644 index 0000000..b395f71 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/default_mask.mask differ diff --git a/testdata/integration/comparison/brain_2d/map.map b/testdata/integration/comparison/brain_2d/map.map new file mode 100644 index 0000000..23094de Binary files /dev/null and b/testdata/integration/comparison/brain_2d/map.map differ diff --git a/testdata/integration/comparison/brain_2d/mask.mask b/testdata/integration/comparison/brain_2d/mask.mask new file mode 100644 index 0000000..4d27e58 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/mask.mask differ diff --git a/testdata/integration/comparison/brain_2d/registered.h5 b/testdata/integration/comparison/brain_2d/registered.h5 new file mode 100644 index 0000000..a62754c Binary files /dev/null and b/testdata/integration/comparison/brain_2d/registered.h5 differ diff --git a/testdata/integration/comparison/brain_2d/registered.image b/testdata/integration/comparison/brain_2d/registered.image new file mode 100644 index 0000000..41a44c9 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/registered.image differ diff --git a/testdata/integration/comparison/brain_2d/registered.image.image b/testdata/integration/comparison/brain_2d/registered.image.image new file mode 100644 index 0000000..41a44c9 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/registered.image.image differ diff --git a/testdata/integration/comparison/brain_2d/test.image.image b/testdata/integration/comparison/brain_2d/test.image.image new file mode 100644 index 0000000..41a44c9 Binary files /dev/null and b/testdata/integration/comparison/brain_2d/test.image.image differ diff --git a/testdata/integration/regression/brain_2d/TestXdmfRead.py b/testdata/integration/regression/brain_2d/TestXdmfRead.py new file mode 100644 index 0000000..5987a3c --- /dev/null +++ b/testdata/integration/regression/brain_2d/TestXdmfRead.py @@ -0,0 +1,58 @@ +from Xdmf import * + +if __name__ == "__main__": + + fileName ="TestXdmfRead.xmf" + + # create a simple empty file + domain = XdmfDomain.New() + collection = XdmfGridCollection.New() + grid = XdmfUnstructuredGrid.New() + attribute1 = XdmfAttribute.New() + attribute2 = XdmfAttribute.New() + information = XdmfInformation.New() + + domain.insert(collection) + collection.insert(grid) + grid.insert(attribute1) + grid.insert(attribute2) + grid.insert(information) + + writer = XdmfWriter.New(fileName) + domain.accept(writer) + + # read file using XPaths and verify downcasts to appropriate XdmfItems + reader = XdmfReader.New() + + domain = reader.read(fileName, "/Xdmf/Domain") + print(str(len(domain)) + " ?= " + str(1)) + print("? " + str(isinstance(domain[0], XdmfDomain))) + assert(len(domain) == 1) + assert(isinstance(domain[0], XdmfDomain)) + + collection = reader.read(fileName, "/Xdmf/Domain/Grid") + print(str(len(collection)) + " ?= " + str(1)) + print("? " + str(isinstance(collection[0], XdmfGridCollection))) + assert(len(collection) == 1) + assert(isinstance(collection[0], XdmfGridCollection)) + + grid = reader.read(fileName, "/Xdmf/Domain/Grid/Grid") + print(str(len(grid)) + " ?= " + str(1)) + print("? " + str(isinstance(grid[0], XdmfUnstructuredGrid))) + assert(len(grid) == 1) + assert(isinstance(grid[0], XdmfUnstructuredGrid)) + + attributes = reader.read(fileName, "/Xdmf/Domain/Grid/Grid/Attribute") + print(str(len(attributes)) + " ?= " + str(2)) + print("? " + str(isinstance(attributes[0], XdmfAttribute))) + print("? " + str(isinstance(attributes[1], XdmfAttribute))) + assert(len(attributes) == 2) + assert(isinstance(attributes[0], XdmfAttribute)) + assert(isinstance(attributes[1], XdmfAttribute)) + + information = reader.read(fileName, "/Xdmf/Domain/Grid/Grid/Information") + print(str(len(information)) + " ?= " + str(1)) + print("? " + str(isinstance(information[0], XdmfInformation))) + assert(len(information) == 1) + assert(isinstance(information[0], XdmfInformation)) + diff --git a/testdata/integration/regression/brain_2d/brain_2d.conf b/testdata/integration/regression/brain_2d/brain_2d.conf new file mode 100644 index 0000000..d2910c8 --- /dev/null +++ b/testdata/integration/regression/brain_2d/brain_2d.conf @@ -0,0 +1,10 @@ +fixed=fixed.image +moved=moved.image + +#map=registration_map.xmdf + +# use HDF5 filename without file groups +registered=registered.xdmf.h5 +map=map.xdmf +nodespacing=10 +save_intermediate_frames=false diff --git a/testdata/integration/regression/brain_2d/brain_2d.testconf b/testdata/integration/regression/brain_2d/brain_2d.testconf new file mode 100644 index 0000000..1a85da9 --- /dev/null +++ b/testdata/integration/regression/brain_2d/brain_2d.testconf @@ -0,0 +1,5 @@ +name = brain2d +type = regression +pfire_config = brain_2d.conf +accepted_image = moved.png # string: the image to compare with ("regression" mode only) +accepted_map = map.xdmf # string: the map to compare with ("regression" mode only) diff --git a/testdata/integration/regression/brain_2d/brain_2d_comparison.testconf b/testdata/integration/regression/brain_2d/brain_2d_comparison.testconf new file mode 100644 index 0000000..94ca871 --- /dev/null +++ b/testdata/integration/regression/brain_2d/brain_2d_comparison.testconf @@ -0,0 +1,4 @@ +name = brain2d +type = comparison +#pfire_config = /home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/testdata/integration/regression/brain_2d/brain_2d.conf +pfire_config = /home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/testdata/integration/regression/brain_2d/brain_2d.conf \ No newline at end of file diff --git a/testdata/integration/regression/brain_2d/brain_2d_regression.testconf b/testdata/integration/regression/brain_2d/brain_2d_regression.testconf new file mode 100644 index 0000000..69b91fe --- /dev/null +++ b/testdata/integration/regression/brain_2d/brain_2d_regression.testconf @@ -0,0 +1,7 @@ +name = brain2d_regression +type = regression +#/home/tartarini/DEVELOP/INSIGNEO/insigneo-pfire/pFIRE/testdata/integration/regression/brain_2d/brain_2d.conf + +pfire_config = brain_2d.conf +accepted_image = accepted.png # string: the image to compare with ("regression" mode only) +accepted_map = map.xdmf.h5 # string: the map to compare with ("regression" mode only) diff --git a/testdata/integration/regression/brain_2d/default_mask.mask b/testdata/integration/regression/brain_2d/default_mask.mask new file mode 100644 index 0000000..b395f71 Binary files /dev/null and b/testdata/integration/regression/brain_2d/default_mask.mask differ diff --git a/testdata/integration/regression/brain_2d/fixed.image b/testdata/integration/regression/brain_2d/fixed.image new file mode 100644 index 0000000..19ef18a Binary files /dev/null and b/testdata/integration/regression/brain_2d/fixed.image differ diff --git a/testdata/integration/regression/brain_2d/map.map b/testdata/integration/regression/brain_2d/map.map new file mode 100644 index 0000000..f742e9e Binary files /dev/null and b/testdata/integration/regression/brain_2d/map.map differ diff --git a/testdata/integration/regression/brain_2d/moved.image b/testdata/integration/regression/brain_2d/moved.image new file mode 100644 index 0000000..bb89593 Binary files /dev/null and b/testdata/integration/regression/brain_2d/moved.image differ diff --git a/testdata/integration/regression/brain_2d/shirt_brain2d_map.map b/testdata/integration/regression/brain_2d/shirt_brain2d_map.map new file mode 100644 index 0000000..f742e9e Binary files /dev/null and b/testdata/integration/regression/brain_2d/shirt_brain2d_map.map differ diff --git a/testdata/integration/regression/brain_2d/shirt_brain2d_registered.image b/testdata/integration/regression/brain_2d/shirt_brain2d_registered.image new file mode 100644 index 0000000..dc232c9 Binary files /dev/null and b/testdata/integration/regression/brain_2d/shirt_brain2d_registered.image differ diff --git a/testdata/integration/regression/brain_2d/test_python.py b/testdata/integration/regression/brain_2d/test_python.py new file mode 100644 index 0000000..df0ee8a --- /dev/null +++ b/testdata/integration/regression/brain_2d/test_python.py @@ -0,0 +1,94 @@ +from functools import partial +import numpy as np +import h5py +import sys +import skimage.io as skio + + +def __LINE__(): + return str(sys._getframe(1).f_lineno) + + + + +def load_pfire_image(imagepath): + """Load pFIRE image + """ + print("\nload_pfire_image filepath={}".format(imagepath) ) + imagepath+=":/registered" # FIXME data group should be read from xdmf file + + filename, group = [x.strip() for x in imagepath.split(':')] + print("\nload_pfire_image filename={}".format(filename) ) + print("\nload_pfire_image group={}".format(group )) + #print("load_pfire_image() line="+__LINE__()) + + if filename.endswith(".xdmf"): # FIXME read filename from xdmf + filename += ".h5" + + reader = XdmfReader.New() + domain = read(filename) + print (domain) + + print("\nload_pfire_image filename={}".format(filename) ) + print("load_pfire_image() line="+__LINE__()) + + with h5py.File(filename, 'r') as fh: + print(list(fh.keys())) + imgdata = np.asarray(fh['registered']) + + return imgdata + + + +filename="registered.xdmf.h5" +fh=h5py.File(filename, 'r') + + +print("\n open "+filename) +lista = list(fh.keys()) + +print(*lista) + +for element in lista: + print(element) + + + +print(list(fh.keys())) +imgdata = np.asarray(fh['registered']) +type(imgdata) +print("\n type of imagedata =" + str(type(imgdata))) + +print("\n shape of imagedata =" , imgdata.shape) +print("\n registered image\n =" , imgdata) + + +fh.close() + +filename="map.xdmf.h5" +print("\n open "+filename) +fh=h5py.File(filename, 'r') + + +for element in list(fh.keys()): + print(element) + +data = np.asarray(fh['/map/x']) +print("\n shape of data =" , data.shape) + + +########################## +imagepath="registered.xdmf" +image_data= load_pfire_image(imagepath) + +print ("\n image_data= \n " ) +print (image_data) + +#datax=map.xdmf.h5:/map/x + + + # + # + # map.xdmf.h5:/map/x + # map.xdmf.h5:/map/y + # \ No newline at end of file diff --git a/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.conf b/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.conf new file mode 100644 index 0000000..14e7773 --- /dev/null +++ b/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.conf @@ -0,0 +1,10 @@ +fixed=fixed_8000x10000.image +moved=moved_8000x10000.image + +#map=registration_map.xmdf +# use HDF5 filename without file groups +registered=registered.xdmf.h5 +map=map.xdmf +nodespacing=10 +save_intermediate_frames=false + diff --git a/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.testconf b/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.testconf new file mode 100644 index 0000000..146c6e8 --- /dev/null +++ b/testdata/integration/regression/brain_2d_8000x10000/brain2d_large.testconf @@ -0,0 +1,6 @@ +name = brain2d_regression +type = regression +pfire_config = brain2d_large.conf +accepted_image = registered_shirt.png # string: the image to compare with ("regression" mode only) +#accepted_map = map.xdmf.h5 # string: the map to compare with ("regression" mode only) + diff --git a/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.dcm b/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.dcm new file mode 100644 index 0000000..0d371fe Binary files /dev/null and b/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.dcm differ diff --git a/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.jpg b/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.jpg new file mode 100644 index 0000000..de7c2df Binary files /dev/null and b/testdata/integration/regression/brain_2d_8000x10000/fixed_8000x10000.jpg differ diff --git a/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.dcm b/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.dcm new file mode 100644 index 0000000..ed96c3f Binary files /dev/null and b/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.dcm differ diff --git a/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.jpg b/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.jpg new file mode 100644 index 0000000..0f3ccbe Binary files /dev/null and b/testdata/integration/regression/brain_2d_8000x10000/moved_8000x10000.jpg differ diff --git a/testdata/integration/regression/brain_2d_large/accepted_2000x2500.dcm b/testdata/integration/regression/brain_2d_large/accepted_2000x2500.dcm new file mode 100644 index 0000000..5d3e75e Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/accepted_2000x2500.dcm differ diff --git a/testdata/integration/regression/brain_2d_large/accepted_2000x2500.jpg b/testdata/integration/regression/brain_2d_large/accepted_2000x2500.jpg new file mode 100644 index 0000000..e679832 Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/accepted_2000x2500.jpg differ diff --git a/testdata/integration/regression/brain_2d_large/brain2d_large.conf b/testdata/integration/regression/brain_2d_large/brain2d_large.conf new file mode 100644 index 0000000..b1da588 --- /dev/null +++ b/testdata/integration/regression/brain_2d_large/brain2d_large.conf @@ -0,0 +1,10 @@ +fixed=moved_2000x2500.image +moved=moved_2000x2500.image + +#map=registration_map.xmdf +# use HDF5 filename without file groups +registered=registered.xdmf.h5 +map=map.xdmf +nodespacing=10 +save_intermediate_frames=false + diff --git a/testdata/integration/regression/brain_2d_large/brain2d_large.testconf b/testdata/integration/regression/brain_2d_large/brain2d_large.testconf new file mode 100644 index 0000000..146c6e8 --- /dev/null +++ b/testdata/integration/regression/brain_2d_large/brain2d_large.testconf @@ -0,0 +1,6 @@ +name = brain2d_regression +type = regression +pfire_config = brain2d_large.conf +accepted_image = registered_shirt.png # string: the image to compare with ("regression" mode only) +#accepted_map = map.xdmf.h5 # string: the map to compare with ("regression" mode only) + diff --git a/testdata/integration/regression/brain_2d_large/fixed_2000x2500.dcm b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.dcm new file mode 100644 index 0000000..c3be073 Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.dcm differ diff --git a/testdata/integration/regression/brain_2d_large/fixed_2000x2500.image b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.image new file mode 100644 index 0000000..2b96b9a Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.image differ diff --git a/testdata/integration/regression/brain_2d_large/fixed_2000x2500.jpg b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.jpg new file mode 100644 index 0000000..c5054e7 Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/fixed_2000x2500.jpg differ diff --git a/testdata/integration/regression/brain_2d_large/moved_2000x2500.dcm b/testdata/integration/regression/brain_2d_large/moved_2000x2500.dcm new file mode 100644 index 0000000..391611f Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/moved_2000x2500.dcm differ diff --git a/testdata/integration/regression/brain_2d_large/moved_2000x2500.image b/testdata/integration/regression/brain_2d_large/moved_2000x2500.image new file mode 100644 index 0000000..2b96b9a Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/moved_2000x2500.image differ diff --git a/testdata/integration/regression/brain_2d_large/moved_2000x2500.jpg b/testdata/integration/regression/brain_2d_large/moved_2000x2500.jpg new file mode 100644 index 0000000..9c59fe6 Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/moved_2000x2500.jpg differ diff --git a/testdata/integration/regression/brain_2d_large/registered_shirt.image b/testdata/integration/regression/brain_2d_large/registered_shirt.image new file mode 100644 index 0000000..2b96b9a Binary files /dev/null and b/testdata/integration/regression/brain_2d_large/registered_shirt.image differ diff --git a/testdata/integration/regression/mask_test_3d/default_mask.mask b/testdata/integration/regression/mask_test_3d/default_mask.mask new file mode 100644 index 0000000..df7b4b7 Binary files /dev/null and b/testdata/integration/regression/mask_test_3d/default_mask.mask differ diff --git a/testdata/integration/regression/mask_test_3d/fixed.image b/testdata/integration/regression/mask_test_3d/fixed.image new file mode 100644 index 0000000..2a55f51 Binary files /dev/null and b/testdata/integration/regression/mask_test_3d/fixed.image differ diff --git a/testdata/integration/regression/mask_test_3d/mask.mask b/testdata/integration/regression/mask_test_3d/mask.mask new file mode 100644 index 0000000..69b3277 Binary files /dev/null and b/testdata/integration/regression/mask_test_3d/mask.mask differ diff --git a/testdata/integration/regression/mask_test_3d/moved.image b/testdata/integration/regression/mask_test_3d/moved.image new file mode 100644 index 0000000..ae178b6 Binary files /dev/null and b/testdata/integration/regression/mask_test_3d/moved.image differ diff --git a/testdata/integration/regression/mask_test_3d/nomask.testconf b/testdata/integration/regression/mask_test_3d/nomask.testconf new file mode 100644 index 0000000..da626dd --- /dev/null +++ b/testdata/integration/regression/mask_test_3d/nomask.testconf @@ -0,0 +1,5 @@ +name = mask_test_3d_nomask +type = comparison +pfire_config = nomask.conf +accepted_image = accepted_nomask.image # string: the image to compare with ("regression" mode only) +accepted_map = map_nomask.xdmf # string: the map to compare with ("regression" mode only) diff --git a/testdata/integration/regression/mask_test_3d/requirements.txt b/testdata/integration/regression/mask_test_3d/requirements.txt new file mode 100644 index 0000000..931ad63 --- /dev/null +++ b/testdata/integration/regression/mask_test_3d/requirements.txt @@ -0,0 +1,16 @@ +cycler==0.10.0 +decorator==4.4.2 +../../../../utils/flannel/ +imageio==2.9.0 +kiwisolver==1.3.1 +matplotlib==3.3.4 +networkx==2.5.1 +numpy==1.19.5 +Pillow==8.3.2 +pyparsing==2.4.7 +python-dateutil==2.8.2 +PyWavelets==1.1.1 +scikit-image==0.17.2 +scipy==1.5.4 +six==1.16.0 +tifffile==2020.9.3 diff --git a/testdata/integration/regression/mask_test_3d/withmask.testconf b/testdata/integration/regression/mask_test_3d/withmask.testconf new file mode 100644 index 0000000..2ad239d --- /dev/null +++ b/testdata/integration/regression/mask_test_3d/withmask.testconf @@ -0,0 +1,5 @@ +name = mask_test_3d_withmask +type = comparison +pfire_config = withmask.conf +accepted_image = accepted_withmask.image # string: the image to compare with ("regression" mode only) +accepted_map = map_withmask.xdmf # string: the map to compare with ("regression" mode only) diff --git a/unit_test/test_hdfwriter.cpp b/unit_test/test_hdfwriter.cpp index 31dfa3d..d45c392 100644 --- a/unit_test/test_hdfwriter.cpp +++ b/unit_test/test_hdfwriter.cpp @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(test_write_hdf) for(integer iidx=0; iidx < map->num_owned_nodes(); iidx++) { - BOOST_REQUIRE(fcmp(dataspace[iidx], double(idof+1), 1e-20, 1e-20)); + BOOST_REQUIRE(fcmp((double) dataspace[iidx], double(idof+1), 1e-20, 1e-20)); } delete[] dataspace; diff --git a/unit_test/test_warp.cpp b/unit_test/test_warp.cpp index 5f7d51a..e2c1e5a 100644 --- a/unit_test/test_warp.cpp +++ b/unit_test/test_warp.cpp @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(uniform_warp) { for(integer xx=map_local_lo[0]; xx= block_offset && (curr_loc-warp_offset) < (block_offset + block_shape)) { - BOOST_REQUIRE(fcmp(image_data[zz][yy][xx], testdata, 1e-20, 1e-10)); + BOOST_REQUIRE(fcmp((double) image_data[zz][yy][xx], (double) testdata, 1e-20, 1e-10)); continue; } - BOOST_REQUIRE(fcmp(image_data[zz][yy][xx], 0., 1e-20, 1e-10)); + BOOST_REQUIRE(fcmp((double) image_data[zz][yy][xx], 0., 1e-20, 1e-10)); } } } diff --git a/utils/flannel/.gitignore b/utils/flannel/.gitignore new file mode 100644 index 0000000..8a8dd1b --- /dev/null +++ b/utils/flannel/.gitignore @@ -0,0 +1,104 @@ +# Temp files +.*.swp + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/utils/flannel/README.md b/utils/flannel/README.md new file mode 100644 index 0000000..11cdc78 --- /dev/null +++ b/utils/flannel/README.md @@ -0,0 +1,73 @@ +# Flannel + +It is a set of scripts made by Phil Tooley manipulate image data for ShIRT. + +## Installation + + +``` +# Clone the repository: +git clone + +# Create a python virtual environment to isolate installation (optional) +python3 -m venv +source venv/bin/activate + +# Install flannel from its source folder to avoid pip picks its own flannel package +cd flannel/ +pip3 install . + + +``` + +Please note that there exist other packages named Flannel tha are also included in +Pip packages. + +# Python API + +``` +image_to_shirt() +``` + +reads input from command line "/path/to/imagefile.ext" +Output: imagefile.image + + +``` +shirt_to_image() +``` +Reads input from command line "/path/to/imgname.image" +Output: imagename.png" + + +``` +image_to_mask() +``` +reads /path/to/imagefile.ext" +Output: imagefile.mask" + + + + +# Command line interface +Binaries converting image of supported formats to/from Shirt image, or generate mask. +Output files are named after the input file just replacing their extension. + + + +Convert image of supported extension to Shirt .image format +``` +image2shirt /path/to/imagefile.ext +``` + +Convert Shirt image to PNG format: +``` +shirt2image /path/to/imgname.image +``` + + +Generate mask for input Shirt image. Output file is named after input replacing extension to .mask +``` +image2shirtmask /path/to/imagefile.ext + +``` \ No newline at end of file diff --git a/utils/flannel/setup.py b/utils/flannel/setup.py new file mode 100644 index 0000000..4dc88b0 --- /dev/null +++ b/utils/flannel/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +setup(name='flannel', + version='0.1', + description='Routines for interacting with the original ShIRT', + author='Phil Tooley, INSIGNEO', + author_email='phil.tooley@sheffield.ac.uk', + url='insigneo.org', + package_dir={'':'src'}, + packages=find_packages('src'), + install_requires=['numpy', 'scikit-image'], + entry_points={ + 'console_scripts' : [ + 'shirt2image = flannel.helpers.image:shirt_to_image', + 'image2shirt = flannel.helpers.image:image_to_shirt', + 'image2shirtmask = flannel.helpers.image:image_to_mask', + ], + 'gui_scripts' : [] + } +) diff --git a/utils/flannel/src/flannel/__init__.py b/utils/flannel/src/flannel/__init__.py new file mode 100644 index 0000000..ea83579 --- /dev/null +++ b/utils/flannel/src/flannel/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from .io import save_image, load_image +from .io import save_mask, load_mask +from .io import save_map, load_map diff --git a/utils/flannel/src/flannel/helpers/__init__.py b/utils/flannel/src/flannel/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/flannel/src/flannel/helpers/image.py b/utils/flannel/src/flannel/helpers/image.py new file mode 100755 index 0000000..acd4a2d --- /dev/null +++ b/utils/flannel/src/flannel/helpers/image.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import sys +import os.path +import skimage.io as skio +import numpy as np +import pydicom + +from flannel.io import load_image, save_image, save_mask + +def image_to_shirt(): + if len(sys.argv) != 2: + print("Usage: {} /path/to/imagefile.ext".format(sys.argv[0])) + print("\tOutput: imagefile.image") + return(0) + + input_path = sys.argv[1] + input_dir, input_file = os.path.split(input_path) + output_file = os.path.splitext(input_file)[0] + '.image' + output_path = output_file + + try: + image = skio.imread(input_path, as_grey=True).astype(np.float32) + except Exception as err: + try: + dcm = pydicom.dcmread(input_path) + image = dcm.pixel_array + except Exception as err: + print("Error: Failed to open {}".format(input_path)) + print(err) + return(-1) + + save_image(image, output_path) + +def image_to_mask(): + if len(sys.argv) != 2: + print("Usage: {} /path/to/imagefile.ext".format(sys.argv[0])) + print("\tOutput: imagefile.mask") + return(0) + + input_path = sys.argv[1] + input_dir, input_file = os.path.split(input_path) + output_file = os.path.splitext(input_file)[0] + '.mask' + output_path = output_file + + try: + image = skio.imread(input_path, as_grey=True) + mask = np.ones(image.shape, dtype=np.int16) + save_mask(mask, output_path) + + except Exception as err: + try: + dcm = pydicom.dcmread(input_path) + image = dcm.pixel_array + mask = np.ones(image.shape, dtype=np.int16) + save_mask(mask, output_path) + except Exception as err: + print("Error: Failed to open {}".format(input_path)) + print(err) + return(-1) + + #save_mask(mask, output_path) + + +def shirt_to_image(): + if len(sys.argv) != 2: + print("Usage: {} /path/to/imgname.image".format(sys.argv[0])) + print("\tOutput: imagename.png") + return(0) + + input_path = sys.argv[1] + input_dir, input_file = os.path.split(input_path) + output_file = os.path.splitext(input_file)[0] + '.png' + output_path = output_file + + try: + image = load_image(input_path) + except Exception as err: + print("Error: Failed to open {}".format(input_path)) + print(err) + return(-1) + + print(image.shape) + print(image.min()) + print(image.max()) + + image = image - image.min() + image = image / image.max() + + skio.imsave(output_path, image) diff --git a/utils/flannel/src/flannel/io.py b/utils/flannel/src/flannel/io.py new file mode 100755 index 0000000..8d43071 --- /dev/null +++ b/utils/flannel/src/flannel/io.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import numpy as np + +def save_image(im, fname): + sizearr = np.ones(4, dtype=np.int32) + sizearr[0:len(im.shape)] = im.shape + im = im.astype(np.float32).flatten(order='F') + with open(fname, 'wb') as fh: + fh.write(sizearr.tobytes()) + fh.write(im.tobytes()) + + +def load_image(fname): + with open(fname, 'rb') as fh: + shape = np.frombuffer(fh.read(16), np.int32) + imsize = np.product(shape) * np.float32(0).itemsize + im = np.frombuffer(fh.read(imsize), np.float32) + im = np.squeeze(im.reshape(shape, order='F')) + return im + + +def save_mask(im, fname): + sizearr = np.ones(3,dtype=np.int32) + sizearr[0:len(im.shape)] = im.shape + im = im.astype(np.int16).flatten(order='F') + with open(fname, 'wb') as fh: + fh.write(sizearr.tobytes()) + fh.write(im.tobytes()) + + +def load_mask(fname): + with open(fname, 'rb') as fh: + shape = np.frombuffer(fh.read(12), np.int32) + imsize = np.product(shape) * np.float32(0).itemsize + im = np.frombuffer(fh.read(imsize), np.float32) + im = np.squeeze(im.reshape(shape, order='F')) + return im + + +def save_map(im, grid, fname, imgdims=None): + """ + Takes map as (N,X,..) matrix and creates shirt mapfile using + parameter grid to map node locations + """ + if grid.shape != im.shape[1:]: + raise ValueError("Dimensionality mismatch imgmap vs grid") + + if imgdims is not None: + if len(imgdims) != len(im.shape): + raise ValueError("Dimensionality mismatch imgdims vs imgmap.shape") + + shape3d = np.ones(3) + shape3d[0:len(grid.shape)] = grid.shape + gridmesh = np.mgrid[0:shape3d[0] + ,0:shape3d[1] + ,0:shape3d[2]] + + len1d = np.product(shape3d) + + mapheader = np.zeros(9, np.int32) + mapheader[0:3] = shape3d #map dimensions + mapheader[3] = len1d #total data size + mapheader[4] = gridmesh[0][1,0,0] - gridmesh[0][0,0,0] #node spacing + mapheader[5] = grid.shape # dimensionality + mapheader[6:9] = imgdims # image dimensions + + mapdata = np.zeros((7,len1d), np.int32) + + for i in range(3): + mapdata[i] = gridmesh[i].reshape(-1) + + if im.shape[-1] == 3: + mapdata[3] = im[0] + mapdata[6] = im[-2] + if im.shape[-1] == 4: + mapdata[3] = im[0] + mapdata[4] = im[1] + mapdata[6] = im[-2] + if im.shape[-1] == 5: + mapdata[3] = im[0] + mapdata[4] = im[1] + mapdata[5] = im[2] + mapdata[6] = im[-2] + + with open(fname, 'wb') as fh: + fh.write(mapheader.tobytes()) + fh.write(mapdata.tobytes()) + + +def load_map(fname): + """ + Loads shirt map file returning map as (N,X,..) matrix and grid as a list of N + vectors + """ + with open(fname, 'rb') as fh: + mapheader = np.frombuffer(fh.read(9*np.int32(4).itemsize), np.int32) + mapsize = tuple(mapheader[0:3]) + mapdatasize = int(np.float64(0).itemsize * np.product(mapsize)) + mapdata = [] + for i in range(7): + databuf = np.frombuffer(fh.read(mapdatasize), np.float64) + mapdata.append(databuf.reshape(mapsize, order='F')) + + griddata = [mapdata[0][:,0,0], mapdata[1][0,:,0], mapdata[2][0,0,:]] + + mapdata = np.stack(mapdata[3:]) + + return(mapdata, griddata)