Functional tests use GoogleTest now (#26)

This MR makes all LPF functional tests use GoogleTest.

The main changes for CMake are:
- reorganize CMake to use GTest CMake package for compilation, and gtest_add_tests clause to define Gtests;
- remove all use of the run.sh script;
- make explicit the list of tests and do not use file(GLOB, ...) clauses to look for all files in test directories, as recommended by CMakesource files: 

The main changes for source files are:
- every single test file needed to be modified to not include internal Test.h but gtest.h, and to use the assert/equal clauses of GoogleTest
- all death tests are slightly modified now to expect FAIL() after the expected fail condition
- some modernization of C to C++ was needed for a number of files, incl. use of new/delete instead of malloc/free, or string manipulation instead of C char * manipulation

A new Python wrapper script test_launcher.py is being employed now for more flexible checking of return codes, which is needed for death tests and normal tests alike. The expected return codes are still parsed from scanning the source files as before.

For the complete history, see GitHub PR #26 .

This MR also resolves bugs that have appeared on more modern cluster deployments compared to the previous main tag of LPF. This is hence a priority MR to base all further extensions and evolutions of LPF on. This MR was tested successfully for:
 - x86_64, CentOS Stream 9, GCC 11.5.0, MPICH 4.1.1 (caveat 2)
 - x86_64, CentOS Stream 9, GCC 11.5.0, OpenMPI 4.1.1 (caveats 1, 2, and 5)
 - x86_64, Fedora, GCC 9.3.1, MPICH 3.3.2 (caveat 2)
 - x86_64, Ubuntu 23.04, GCC 13.1.0, OpenMPI 4.1.6 (caveat 5)
 - x86_64, openEuler 20.03 LTS, GCC 7.3, MPICH 3.2.1
 - x86_64, Ubuntu 22.04 LTS, GCC 11.4.0, MPICH 4.3.0b1 (caveat 3, transient)
 - ARM, Ubuntu 22.04 LTS, GCC 11.4.0, MPICH 4.3.0b1 (caveats 3 & 4)

These are under different variations of calls to `bootstrap.sh` to test all the above-described changes.

The following caveats apply:
1. with OpenMPI and default arguments to its `mpirun`, oversubscription to larger process counts may be limited, causing some tests to fail. Issue #37 has been raised to pass the appropriate flags to the OpenMPI `mpirun` during testing;
2. `make install` post-install checks fail (as they should) when: 1) tests are disabled, 2) ibverbs dev libraries are present, but  3) no IB card is present. On the machines with IB we tested on, all post-install checks succeed.
3. The hybrid backend on MPICH 4.3.0b1 does not pass the debug layer tests due to a segfault in MPICH's MPI_Abort. Issue #41 has been raised to track this. This error presently only is consistently reproducible on our cluster's ARM nodes and transient on our cluster's x86_64 nodes. They have not been observed outside our cluster.
4. With MPICH 4.3.0b1, we furthermore hit issues #43 and #44. Also these issues are only reproducible on ARM nodes.
5. With OpenMPI 4.1.6 and OpenMPI 4.1.1 on x86_64 we hit issue #45
6. At present, we have no CI flow that can run all build variations relevant to this MR. Issues #40 has been raised to (partially) address as well as to track this issue.

Author: KADichev

CMakeLists.txt

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.29.0 FATAL_ERROR)
 project(LPF C CXX ASM)
 
 # Version info
@@ -52,12 +52,12 @@ set(CPACK_PACKAGE_VERSION_MAJOR "${VERSION_MAJOR}")
 set(CPACK_PACKAGE_VERSION_MINOR "${VERSION_MINOR}")
 set(CPACK_PACKAGE_VERSION_PATCH "${VERSION_PATCH}")
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
-	"A high performance BSP communications library" )
+  "A high performance BSP communications library" )
 
 set(CPACK_SOURCE_GENERATOR "TGZ" )
 set(CPACK_SOURCE_IGNORE_FILES "/\\\\.git/" "/\\\\.svn/" "\\\\.swp$" "/site/" "/build/" "/pclint/" "/junit/" "/ideas/" )
 set(CPACK_SOURCE_PACKAGE_FILE_NAME
-	"LPF-${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_PACKAGE}")
+  "LPF-${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_PACKAGE}")
 
 set(CPACK_GENERATOR "RPM")
 set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
@@ -105,18 +105,6 @@ set( INSTALL_HEADERS "${prefix}/include" CACHE PATH
         "Installation path for header files" )
 message( STATUS "Installation directory prefix is ${prefix}")
 
-# C++ standard
-find_file(TR1_ARRAY "tr1/array")
-if (TR1_ARRAY)
-    message(STATUS "Governing C++ standard is C++98/TR1")
-    set(CMAKE_CXX_STANDARD 98)
-    set(CMAKE_CXX_STANDARD_REQUIRED YES)
-else()
-    message(STATUS "Governing C++ standard is C++11")
-    set(CMAKE_CXX_STANDARD 11)
-    set(CMAKE_CXX_STANDARD_REQUIRED YES)
-endif()
-
 # Dependencies
 set(ENGINES)
 find_library( LIB_POSIX_THREADS
@@ -173,9 +161,6 @@ if ( MPI_FOUND )
    if ( NOT MPI_IS_THREAD_COMPAT OR NOT MPI_IS_NOT_OPENMPI1 )
       message( WARNING "MPI implementation does not tolerate any threading. Hybrid implementation will not be built")
    endif()
-   if ( NOT MPI_OPEN_PORT )
-      message( WARNING "MPI implementation does not support dynamically connecting separate MPI processes. Hence, lpf_mpi_initialize_over_tcp will always fail.")
-   endif()
    if ( NOT MPI_IBARRIER )
       message( WARNING "MPI implementation does not have MPI_Ibarrier, which is required to use the dense all-to-all algorithm on large (> 2 GB) meta-data exchanges")
    endif()
@@ -189,18 +174,37 @@ if ( LIB_MATH AND LIB_DL AND MPI_FOUND )
         list(APPEND ENGINES "mpirma")
     endif()
 
-    if (LIB_IBVERBS)
+    if (ENABLE_IBVERBS)
         list(APPEND ENGINES "ibverbs")
     endif()
 
 endif()
 
 #enable the hybrid engine
 if ( LIB_POSIX_THREADS AND LIB_MATH AND LIB_DL AND MPI_FOUND
-        AND MPI_IS_THREAD_COMPAT AND MPI_IS_NOT_OPENMPI1
-	AND LIB_IBVERBS )
-    list(APPEND ENGINES "hybrid")
-    set(HYBRID_ENGINE_ENABLED on)
+        AND MPI_IS_THREAD_COMPAT AND MPI_IS_NOT_OPENMPI1 )
+    if( ENABLE_IBVERBS )
+        set(LPFLIB_HYBRID_MPI_ENGINE "ibverbs" CACHE STRING
+            "Choice of MPI engine to use for inter-process communication")
+        list(APPEND ENGINES "hybrid")
+        set(HYBRID_ENGINE_ENABLED on)
+    elseif( MPI_RMA )
+        set(LPFLIB_HYBRID_MPI_ENGINE "mpirma" CACHE STRING
+            "Choice of MPI engine to use for inter-process communication")
+        list(APPEND ENGINES "hybrid")
+        set(HYBRID_ENGINE_ENABLED on)
+    elseif( LIB_MATH AND LIB_DL AND MPI_FOUND )
+        set(LPFLIB_HYBRID_MPI_ENGINE "mpimsg" CACHE STRING
+            "Choice of MPI engine to use for inter-process communication")
+        list(APPEND ENGINES "hybrid")
+        set(HYBRID_ENGINE_ENABLED on)
+    endif()
+    if( HYBRID_ENGINE_ENABLED )
+        message( "Hybrid engine will be built using the ${LPFLIB_HYBRID_MPI_ENGINE} engine" )
+    else()
+	message( "No suitable inter-node communication engine found; "
+	    "hybrid engine will not be built" )
+    endif()
 endif()
 
 message( STATUS "The following engines will be built: ${ENGINES}")
@@ -223,6 +227,7 @@ endif()
 
 # When system is not Linux, enable conditionally compiled blocks
 if (APPLE)
+    message( WARNING "LPF compilation on OS X is not regularly tested" )
     add_definitions(-DLPF_ON_MACOS=1)
 endif()
 
@@ -246,59 +251,19 @@ add_definitions(-DBSPLIB_DLL=1)
 option(LPF_ENABLE_TESTS
        "Enable unit and API tests. This uses Google Testing and Mocking Framework"
        OFF)
+option(GTEST_AGREE_TO_LICENSE
+      "Does the user agree to the GoogleTest license"
+      OFF)
+
+# C++ standard -- Google tests require newer C++ standard than C++11
 if (LPF_ENABLE_TESTS)
-message(STATUS "Unit and API tests will be built")
-
-# set testing timeout to 60 seconds
-set(CMAKE_TESTING_TIMEOUT 60)
-
-# import Google Testing Framework
-include(cmake/googletest.cmake)
-
-# Have directory to gather all the tests results
-file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/junit)
-set(test_output "${CMAKE_BINARY_DIR}/junit")
-
-# Have a macro to add a unit test
-function(add_gtest testName)
-    add_executable(${testName} ${ARGN})
-    target_link_libraries(${testName} gtest_main)
-    add_test(${testName} ${testName} --gtest_output=xml:${test_output}/ )
-endfunction(add_gtest)
-
-# Have a macro to add a unit test that should run with MPI
-if (MPI_FOUND)
-    function(add_gtest_mpi testName nprocs)
-        add_executable(${testName} ${ARGN})
-        target_link_libraries(${testName} ${MPI_C_LIBRARIES} gtest_main)
-        foreach( p ${nprocs})
-            set(mpmd)
-            foreach( i RANGE 1 ${p})
-                if (i GREATER 1)
-                  set(mpmd ${mpmd} ":")
-                endif()
-                set(mpmd ${mpmd} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS}
-                        ./${testName} --gtest_output=xml:${test_output}/${testName}_${i}of${p}.xml)
-            endforeach(i)
-            add_test(NAME ${testName}_${p}
-                     COMMAND ${MPIRUN} ${mpmd}
-                    )
-        endforeach(p)
-    endfunction(add_gtest_mpi)
-endif(MPI_FOUND)
-
-# Enable testing in CMake
-enable_testing()
-else(LPF_ENABLE_TESTS)
-    message(STATUS "Unit and API tests will *not* be built")
-    function(add_gtest testName)
-       # Do nothing because tests are disabled
-    endfunction(add_gtest)
+   set(CMAKE_CXX_STANDARD 17)
+   set(CMAKE_CXX_STANDARD_REQUIRED YES)
+else()
+   set(CMAKE_CXX_STANDARD 11)
+   set(CMAKE_CXX_STANDARD_REQUIRED YES)
+endif()
 
-    function(add_gtest_mpi testName nprocs)
-       # DO nothing because tests are disabled
-    endfunction(add_gtest_mpi)
-endif(LPF_ENABLE_TESTS)
 
 # Handling of compiler flags
 function(target_add_compilation_flags target visibility)
@@ -367,31 +332,185 @@ endfunction(target_compile_flags)
 set(lpf_cflags)
 set(lpf_lib_link_flags)
 set(lpf_exe_link_flags)
-include_directories(include)
-include_directories(src/common)
-add_subdirectory(src)
+
+# Populate lpf_cflags, lpf_lib_link_flags, lpf_exe_link_flags according to
+# (enabled) engine requirements
+#  - 0) PThreads engine needs nothing special
+#  - 1) MPI-based engines:
+if ( LIB_MATH AND LIB_DL AND MPI_FOUND )
+    # -fPIC and -rdynamic are necessary to ensure that symbols can be
+    # looked up by dlsym which is the mechanism lpf_exec uses to broadcast the
+    # function that should be executed
+    set(rdyn_lflag "-rdynamic")
+    if (APPLE)
+        # OS X does not support -rdynamic
+	set(rdyn_lflag "")
+    endif ()
+
+    # include flags:
+    set( mpi_include_flags )
+    string( REPLACE ";" " -I" mpi_include_flags "${MPI_C_INCLUDE_PATH}" )
+    set(lpf_cflags "${lpf_cflags} -I${mpi_include_flags} -fPIC")
+
+    # linker flags:
+    set(lib_lflags "${MPI_C_LINK_FLAGS}")  #Note: the core library is already linked with MPI_C_LIBRARIES.
+    string(REPLACE ";" " " lib_lflags "${lib_lflags}") # So, no need to also link executables with it.
+    set(lpf_lib_link_flags "${lpf_lib_link_flags} ${lib_lflags} ${rdyn_lflag}")
+
+    # executable linker flags:
+    set(lpf_exe_link_flags "${lpf_exe_link_flags} ${rdyn_lflag}")
+endif ()
+#  ...add requirements from other engines here...
 
 # Collating all compile & link flags
 set(LPF_CORE_COMPILE_FLAGS "${lpf_cflags}" CACHE STRING "Compilation flags for all user code" )
 set(LPF_CORE_LIB_LINK_FLAGS "${lpf_lib_link_flags}" CACHE STRING "Flags to link user libraries" )
 set(LPF_CORE_EXE_LINK_FLAGS "${lpf_exe_link_flags}" CACHE STRING "Flags to link user executables" )
 
-# Compiling LPF programmes in the build dir
+# Compiling LPF programs in the build dir
 function( target_link_exe_with_core target )
     set(engine "imp")
     if (ARGV1)
       set(engine "${ARGV1}")
     endif()
     set(corelib "lpf_core_univ_${engine}_${LPFLIB_CONFIG_NAME}")
 
-    target_link_libraries(${target} ${corelib})
+    if ("${engine}" STREQUAL "pthread")
+        target_link_libraries(${target} ${corelib})
+    else()
+        target_link_libraries(${target} ${corelib} ${MPI_C_LIBRARIES})
+    endif()
     target_compile_flags(${target} PRIVATE ${LPF_CORE_COMPILE_FLAGS})
     set_target_properties(${target} PROPERTIES
-            LINK_FLAGS "${LPF_CORE_LIB_LINK_FLAGS} ${LPF_CORE_EXE_LINK_FLAGS}"
-            LINKER_LANGUAGE CXX
-    )
+        LINK_FLAGS "${LPF_CORE_LIB_LINK_FLAGS} ${LPF_CORE_EXE_LINK_FLAGS}"
+        LINKER_LANGUAGE CXX
+        )
 endfunction()
 
+if (LPF_ENABLE_TESTS)
+    message(STATUS "Unit and API tests will be built. This requires CMake version 3.29 or higher, since we use recent features of the GoogleTest package in CMake.")
+
+    if (NOT GTEST_AGREE_TO_LICENSE)
+        message(FATAL_ERROR "The user needs to agree with the GoogleTest license to use tests (option GTEST_AGREE_TO_LICENSE=TRUE)")
+    endif()
+    # Enable testing in CMake
+    enable_testing()
+    include(ProcessorCount)
+    ProcessorCount(processorCount)
+    find_package(GTest)
+    include(GoogleTest)
+    if(NOT GTest_FOUND) # if not found, download it and pull it in
+        include(FetchContent)
+        FetchContent_Declare(
+            googletest
+            GIT_REPOSITORY https://github.com/google/googletest.git
+            # This tag corresponds to GoogleTest 1.15.0 release
+            GIT_TAG        e397860
+            )
+        FetchContent_MakeAvailable(googletest)
+
+    endif()
+    # set testing timeout to 60 seconds
+    set(CMAKE_TESTING_TIMEOUT 60)
+
+    # Have directory to gather all the tests results
+    file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/junit)
+    set(test_output "${CMAKE_BINARY_DIR}/junit")
+
+    find_package( Python3 REQUIRED COMPONENTS Interpreter)
+    set(MY_TEST_LAUNCHER ${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/test_launcher.py)
+    configure_file( ${CMAKE_SOURCE_DIR}/test_launcher.py.in ${CMAKE_BINARY_DIR}/test_launcher.py @ONLY FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ)
+
+    # Macro for adding a new GoogleTest test
+    function(add_gtest testName ENGINE debug testSource )
+        if ("{$ENGINE}" STREQUAL "")
+            message(FATAL_ERROR "engine cannot be empty, ever!")
+        endif()
+        add_executable(${testName} ${testSource} ${ARGN})
+        target_compile_definitions(${testName} PUBLIC LPF_CORE_IMPL_ID=${ENGINE})
+        target_compile_definitions(${testName} PUBLIC LPF_CORE_MPI_USES_${ENGINE})
+        if (debug)
+            target_include_directories( ${testName} BEFORE PRIVATE ${CMAKE_SOURCE_DIR}/include/debug )
+            target_link_libraries(${testName} lpf_debug lpf_hl_debug GTest::gtest GTest::gtest_main)
+        else(debug)
+            target_link_libraries(${testName} GTest::gtest GTest::gtest_main)
+        endif(debug)
+
+
+        # Extract test-specific information from comments of tests
+        file(READ ${testSource} fileContents)
+        string(REGEX MATCH "Exit code: ([0-9]+)" _ ${fileContents})
+        set(retCode ${CMAKE_MATCH_1})
+        string(REGEX MATCH "pre P >= ([0-9]+)"  _ ${fileContents})
+        set(minProcs ${CMAKE_MATCH_1})
+        string(REGEX MATCH "pre P <= ([0-9]+)" _ ${fileContents})
+        set(maxProcs ${CMAKE_MATCH_1})
+        string(REGEX MATCH "-probe ([0-9]+.[0-9]+)" _ ${fileContents})
+        set(lpfProbeSecs ${CMAKE_MATCH_1})
+
+        target_link_exe_with_core(${testName} ${ENGINE})
+
+        # The "\pre P <= max" comment in a test indicates the desired number of
+        # maximum LPF processes. If the test does not define a desired number of 
+        # maximum LPF processes, it will be set to 5.
+        #
+        # The "\pre P >= min" comment in a test indicates the desired number of
+        # minimum LPF processes. If the test does not define a desired minimum 
+        # number of LPF processes, it will be set to 1.
+        #
+        # Let 'processorCount' be the detected number of processors by the system.
+        # If this number is smaller than the desider minimum and/or maximum number
+        # of processes, it overwrites these
+        #
+        # Most tests only define a mininum number of desired processes, such as
+        # "\pre P >= 1". In those cases, the test will execute for the range 1,..,5
+        # (including)
+
+        if ("${minProcs}" STREQUAL "")
+            set(minProcs "1")
+        endif()
+        if ("${maxProcs}" STREQUAL "")
+            set(maxProcs "5")
+        endif()
+        # cap min with processorCount, if needed
+        if ("${minProcs}" GREATER "${processorCount}")
+            set(minProcs ${processorCount})
+        endif()
+        # cap max with processorCount, if needed
+        if ("${maxProcs}" GREATER "${processorCount}")
+            set(maxProcs ${processorCount})
+        endif()
+        if ("${lpfProbeSecs}" STREQUAL "")
+            set(lpfProbeSecs "0.0")
+        endif()
+        if ("${retCode}" STREQUAL "")
+            set(retCode "0")
+        endif()
+
+        # Most recent approach to Gtests, recommended!
+        set_property(TARGET ${testName} PROPERTY TEST_LAUNCHER ${MY_TEST_LAUNCHER};--engine;${ENGINE};--parallel_launcher;${CMAKE_BINARY_DIR}/lpfrun_build;--min_process_count;${minProcs};--max_process_count;${maxProcs};--lpf_probe_timer;${lpfProbeSecs};--expected_return_code;${retCode})
+        gtest_discover_tests(${testName}
+            TEST_PREFIX ${ENGINE}_
+            EXTRA_ARGS --gtest_output=xml:${test_output}/${ENGINE}_${testName}
+            DISCOVERY_MODE POST_BUILD
+            DISCOVERY_TIMEOUT 15
+        )
+
+    endfunction(add_gtest)
+
+else(LPF_ENABLE_TESTS)
+    message(STATUS "Unit and API tests will *not* be built")
+    function(add_gtest testName ENGINE debug testSource )
+        # Do nothing because tests are disabled
+    endfunction(add_gtest)
+
+endif(LPF_ENABLE_TESTS)
+
+# Main LPF library includes and sources
+include_directories(include)
+include_directories(src/common)
+add_subdirectory(src)
+
 # Apps
 add_subdirectory(src/utils)
 
@@ -432,7 +551,7 @@ set( lpf_proxy_dummy ${CMAKE_CURRENT_BINARY_DIR}/src/MPI/lpf_proxy_dummy)
 set( lpf_probe ${CMAKE_CURRENT_BINARY_DIR}/src/utils/lpfprobe)
 set( lpfrun ${CMAKE_CURRENT_BINARY_DIR}/lpfrun_build)
 set( lpfcore ${CMAKE_CURRENT_BINARY_DIR}/src/*/liblpf_core_univ_ENGINE_${LPFLIB_CONFIG_NAME}${SOSUFFIX} )
-configure_file( lpfrun.in lpfrun_build @ONLY)
+configure_file( lpfrun.in lpfrun_build FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE @ONLY)
 configure_file( lpfproxy.in lpfproxy_build @ONLY)
 configure_file( lpfprobe.in lpfprobe_build @ONLY)