diff --git a/CMakeLists.txt b/CMakeLists.txt index 826eb50..8414385 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,14 @@ if(NOT ITK_SOURCE_DIR) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() +project(PerformanceBenchmarking) + +option(ITK_USES_NUMBEROFTHREADS "If ITKv5 uses NumberOfThreads (not NumberOfWorkUnits" OFF) + +set(PerformanceBenchmarking_LIBRARIES PerformanceBenchmarking) + +include_directories(${CMAKE_BINARY_DIR}) +if(NOT ITK_SOURCE_DIR) include(ITKModuleExternal) else() set(ITK_DIR ${CMAKE_BINARY_DIR}) diff --git a/evaluate-itk-performance.py b/evaluate-itk-performance.py index f6a1b6b..b380375 100755 --- a/evaluate-itk-performance.py +++ b/evaluate-itk-performance.py @@ -7,6 +7,8 @@ import socket import json +import glob + def get_shell_output_as_string(commandlist): """ @@ -40,8 +42,8 @@ def __call__(self, parser, namespace, values, option_string=None): run_parser.add_argument('benchmark_bin', help='ITK performance benchmarks build directory', action = FullPaths) run_parser.add_argument('-r', '--rev-list', - help='Arguments for "git rev-list" to select the range of commits to benchmark, for example: "--no-merges v4.10.0..v5.0rc1"', - default='--no-merges HEAD~1..') + help='Arguments for "git rev-list" to select the range of commits to benchmark, for example: "--first-parent v4.10.0..v5.0rc1"', + default='--first-parent HEAD~1..') upload_parser = subparsers.add_parser('upload', help='upload the benchmarks to data.kitware.com') @@ -152,20 +154,41 @@ def check_for_build_information(itk_src): has_itkbuildinformation = True return has_itkbuildinformation +# 2075084be6f9988b1ae2231bca607830fe6d772b is sha1 that rename NumberOfThreads into NumberOfWorkUnits in filters +# Author: Dzenan Zukic +# Date: Tue Jul 17 19:30:02 2018 +# so all ancestors need to prevent the benchmarking from using +def check_for_NumberOfThreads(itk_src): + os.chdir(itk_src) + try: + cmd = ['git', 'merge-base', + '--is-ancestor', 'HEAD', + '2075084be6f9988b1ae2231bca607830fe6d772b'] + has_itkNumberOfThreads = not bool(subprocess.check_call( cmd ) ) + except subprocess.CalledProcessError: + has_itkNumberOfThreads = False + return has_itkNumberOfThreads + def build_benchmarks(benchmark_src, benchmark_bin, itk_bin, - itk_has_buildinformation): + itk_has_buildinformation, + itk_has_NumberOfThreads): os.chdir(benchmark_bin) if itk_has_buildinformation: build_information_arg = '-DITK_HAS_INFORMATION_H:BOOL=ON' else: build_information_arg = '-DITK_HAS_INFORMATION_H:BOOL=OFF' + if itk_has_NumberOfThreads: + NumberOfThreads_arg = '-DITK_USES_NUMBEROFTHREADS:BOOL=ON' + else: + NumberOfThreads_arg = '-DITK_USES_NUMBEROFTHREADS:BOOL=OFF' subprocess.check_call(['cmake', '-G', 'Ninja', '-DCMAKE_BUILD_TYPE:STRING=Release', '-DCMAKE_CXX_STANDARD:STRING=17', '-DITK_DIR:PATH=' + itk_bin, build_information_arg, + NumberOfThreads_arg, benchmark_src]) subprocess.check_call(['ninja']) @@ -309,15 +332,28 @@ def has_sha(filepath): os.environ['ITKPERFORMANCEBENCHMARK_AUX_JSON'] = \ json.dumps(itk_information) + ## Remove vcl_compiler.h in build dir that takes precidence over older non-generated + ## vcl_compiler.h in the source tree (older source code). + if not os.path.exists( os.path.join( args.src, 'Modules','ThirdParty','VNL','src','vxl','vcl','vcl_compiler.h.in') ): + generated_vcl_headers = glob.glob(os.path.join( args.bin, 'Modules','ThirdParty','VNL','src','vxl','vcl',"*.h")) + for vcl_header_file in generated_vcl_headers: + os.remove( vcl_header_file ) + + ## HDF generates and tries to build .c files from other builds, clean these out + hdf5_generated_files = os.path.join( args.bin, 'Modules','ThirdParty','HDF5','src','itkhdf5',"*.c") + hdf5_bld_src_files = glob.glob( hdf5_generated_files ) + for hdf5_file in hdf5_bld_src_files: + os.remove( hdf5_file ) print('\nBuilding ITK...') build_itk(args.src, args.bin) itk_has_buildinformation = check_for_build_information(args.src) + itk_has_NumberOfThreads = check_for_NumberOfThreads(args.src) print('\nBuilding benchmarks...') build_benchmarks(benchmark_src, args.benchmark_bin, args.bin, - itk_has_buildinformation) + itk_has_buildinformation, itk_has_NumberOfThreads) print('\nRunning benchmarks...') run_benchmarks(args.benchmark_bin, itk_information) diff --git a/include/PerformanceBenchmarkingUtilities.h b/include/PerformanceBenchmarkingUtilities.h index 480a870..33d579f 100644 --- a/include/PerformanceBenchmarkingUtilities.h +++ b/include/PerformanceBenchmarkingUtilities.h @@ -5,20 +5,20 @@ #ifndef PerformanceBenchmarkingUtilities_h #define PerformanceBenchmarkingUtilities_h -#include "PerformanceBenchmarkingExport.h" +#include "PerformanceBenchmarkingInformation.h" #include "jsonxx.h" #include //TODO: Move to utiliites #include "itkHighPriorityRealTimeProbesCollector.h" -#if ITK_VERSION_MAJOR >= 5 -# include "itkMultiThreaderBase.h" -using MultiThreaderName = itk::MultiThreaderBase; -# define SET_PARALLEL_UNITS(x) SetNumberOfWorkUnits(x) -#else -# include "itkMultiThreader.h" +#if ITK_VERSION_MAJOR < 5 || defined(ITK_USES_NUMBEROFTHREADS) +#include "itkMultiThreader.h" using MultiThreaderName = itk::MultiThreader; -# define SET_PARALLEL_UNITS(x) SetNumberOfThreads(x) +#define SET_PARALLEL_UNITS( x ) SetNumberOfThreads( x ) +#else +#include "itkMultiThreaderBase.h" +using MultiThreaderName = itk::MultiThreaderBase; +#define SET_PARALLEL_UNITS( x ) SetNumberOfWorkUnits( x ) #endif PerformanceBenchmarking_EXPORT std::string diff --git a/src/PerformanceBenchmarkingInformation.h.in b/src/PerformanceBenchmarkingInformation.h.in index af6f167..5c9905e 100644 --- a/src/PerformanceBenchmarkingInformation.h.in +++ b/src/PerformanceBenchmarkingInformation.h.in @@ -30,6 +30,8 @@ #include "PerformanceBenchmarkingExport.h" +#cmakedefine ITK_USES_NUMBEROFTHREADS + namespace itk {