diff --git a/.github/workflows/ruche-cuda.yml b/.github/workflows/ruche-cuda.yml new file mode 100644 index 0000000..b3000b5 --- /dev/null +++ b/.github/workflows/ruche-cuda.yml @@ -0,0 +1,183 @@ +# Portions of this workflow were adapted from Kokkos-FFT: +# https://github.com/kokkos/kokkos-fft +# +# SPDX-FileCopyrightText: (C) The Kokkos-FFT development team, see Kokkos-FFT COPYRIGHT.md +# SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception + +name: Ruche CUDA build and test + +on: + workflow_dispatch: + schedule: + - cron: "30 0 * * 1-5" # every weekday at midnight UTC + +permissions: + contents: read + +jobs: + build_and_test: + # only run on original repo + if: github.repository == 'CExA-project/kokkos-kernel-forge' + runs-on: [self-hosted, cuda] + timeout-minutes: 120 + env: + KOKKOS_VERSION: 5.1.1 + HDF5_VERSION: hdf5-1.14.6 + + strategy: + # allow all matrix jobs to perform even if errors occur + fail-fast: false + matrix: + build_type: + - Release + cxx_standard: + - 20 + backend: + - name: CUDA-A100-GCC/13.4.0-CUDA/12.8.1-Static-CUDA_CONSTEXPR-CUDA_RELOCATABLE_DEVICE_CODE + modules: >- + gcc/13.4.0/gcc-15.1.0 + cuda/12.8.1/none-none + cmake/3.31.9/gcc-15.1.0 + python/3.14.0/gcc-15.1.0 + test: -e gpu -g a100 + kokkos_configure: >- + -DKokkos_ENABLE_CUDA=ON + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ARCH_AMPERE80=ON + -DKokkos_ENABLE_CUDA_CONSTEXPR=ON + -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON + kkf_configure: >- + -DKKF_BUILD_EXAMPLES=ON + -DKKF_BUILD_TESTS=ON + -DKKF_ENABLE_WARNINGS=ON + -DKKF_ENABLE_ASAN=OFF + -DKKF_ENABLE_UBSAN=OFF + + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Compute Kokkos cache fingerprint + run: | + { + echo "RUNNER_OS=${RUNNER_OS:-unknown}" + echo "RUNNER_ARCH=${RUNNER_ARCH:-unknown}" + echo "KOKKOS_VERSION=$KOKKOS_VERSION" + echo "matrix.build_type=${{ matrix.build_type }}" + echo "matrix.cxx_standard=${{ matrix.cxx_standard }}" + echo "matrix.backend.name=${{ matrix.backend.name }}" + echo "matrix.backend.modules=${{ matrix.backend.modules }}" + echo "matrix.backend.kokkos_configure=${{ matrix.backend.kokkos_configure }}" + } > .kokkos-cache-fingerprint + cat .kokkos-cache-fingerprint + + - name: Cache Kokkos install + id: cache-kokkos + uses: actions/cache@v5 + with: + path: kokkos_install + key: kokkos-${{ runner.os }}-${{ runner.arch }}-${{ env.KOKKOS_VERSION }}-${{ hashFiles('.kokkos-cache-fingerprint') }} + + - name: Build and install Kokkos + if: steps.cache-kokkos.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch "$KOKKOS_VERSION" https://github.com/kokkos/kokkos.git + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake -S kokkos -B kokkos_build \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cxx_standard }} \ + -DCMAKE_INSTALL_PREFIX=kokkos_install \ + ${{ matrix.backend.kokkos_configure }} + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake --build kokkos_build --parallel + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake --install kokkos_build + + - name: Compute HDF5 cache fingerprint + run: | + { + echo "RUNNER_OS=${RUNNER_OS:-unknown}" + echo "RUNNER_ARCH=${RUNNER_ARCH:-unknown}" + echo "HDF5_VERSION=$HDF5_VERSION" + echo "matrix.build_type=${{ matrix.build_type }}" + echo "matrix.backend.name=${{ matrix.backend.name }}" + echo "matrix.backend.modules=${{ matrix.backend.modules }}" + echo "HDF5_BUILD_HL_LIB=ON" + echo "HDF5_BUILD_TOOLS=OFF" + echo "HDF5_BUILD_EXAMPLES=OFF" + echo "BUILD_TESTING=OFF" + } > .hdf5-cache-fingerprint + cat .hdf5-cache-fingerprint + + - name: Cache HDF5 install + id: cache-hdf5 + uses: actions/cache@v5 + with: + path: hdf5_install + key: hdf5-${{ runner.os }}-${{ runner.arch }}-${{ env.HDF5_VERSION }}-${{ hashFiles('.hdf5-cache-fingerprint') }} + + - name: Build and install HDF5 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch "$HDF5_VERSION" https://github.com/HDFGroup/hdf5.git + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake -S hdf5 -B hdf5_build \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DCMAKE_INSTALL_PREFIX=hdf5_install \ + -DHDF5_BUILD_HL_LIB=ON \ + -DHDF5_BUILD_TOOLS=OFF \ + -DHDF5_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake --build hdf5_build --parallel + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake --install hdf5_build + + - name: Initialize the build tree + run: | + run \ + -m "${{ matrix.backend.modules }}" \ + -e local \ + cmake \ + -G "Unix Makefiles" \ + -S . \ + -B build \ + -DBUILDNAME=${{ matrix.backend.name }}-C++${{ matrix.cxx_standard }}-${{ matrix.build_type }} \ + -DSITE=cea-ruche \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cxx_standard }} \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DCMAKE_CXX_FLAGS="-Wall -Wextra" \ + -DCMAKE_PREFIX_PATH="$PWD/hdf5_install" \ + -DKokkos_DIR=kokkos_install/lib64/cmake/Kokkos/ \ + ${{ matrix.backend.kkf_configure }} + + - name: Build + run: | + # Use make directly because cmake --build can complete the child build + # and then hang under Ruche's srun. + run \ + -m "${{ matrix.backend.modules }}" \ + -e cpu \ + make -C build -j40 + + - name: Test + run: | + # Use the Make replay target because ctest can complete launcher.sh + # and then hang under Ruche's srun. + run \ + -m "${{ matrix.backend.modules }}" \ + ${{ matrix.backend.test }} \ + make -C build -j1 kkf_run_replay_tests diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d49dc86..efcd784 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,21 +2,37 @@ enable_testing() set(TEST_LAUNCHER "${CMAKE_CURRENT_LIST_DIR}/launcher.sh") -function(kkf_set_test_working_directory TEST_NAME) +# CTest is unreliable through the Slurm launcher on Ruche. These targets run +# the same replay tests inside a single GPU allocation. +add_custom_target(kkf_run_replay_tests) + +function(add_replay_test TEST_NAME COMPARE_OUTPUT) set(TEST_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/work/${TEST_NAME}") file(MAKE_DIRECTORY "${TEST_WORKING_DIRECTORY}") - set_tests_properties("${TEST_NAME}" PROPERTIES WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}") -endfunction() -function(add_replay_test TEST_NAME COMPARE_OUTPUT) - add_test(NAME "${TEST_NAME}" COMMAND "${TEST_LAUNCHER}" $ $ - $ ${COMPARE_OUTPUT} ${ARGN} + if(COMPARE_OUTPUT) + set(COMPARE_OUTPUT TRUE) + else() + set(COMPARE_OUTPUT FALSE) + endif() + + set(TEST_COMMAND "${TEST_LAUNCHER}" "$" "$" + "$" "${COMPARE_OUTPUT}" ${ARGN} ) + add_test(NAME "${TEST_NAME}" COMMAND ${TEST_COMMAND} WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}") + # Replay tests with fixed-address host allocations link to the unsanitized # replayer, while scalar replay tests use the sanitized replayer variant. set_tests_properties("${TEST_NAME}" PROPERTIES LABELS "replay;sanitizer_compatible") - kkf_set_test_working_directory("${TEST_NAME}") + + add_custom_target( + "kkf_run_${TEST_NAME}" + COMMAND ${TEST_COMMAND} + WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}" + USES_TERMINAL VERBATIM + ) + add_dependencies(kkf_run_replay_tests "kkf_run_${TEST_NAME}") endfunction() add_subdirectory(no_capture) diff --git a/tests/launcher.sh b/tests/launcher.sh index 5fd829a..5bcbbef 100755 --- a/tests/launcher.sh +++ b/tests/launcher.sh @@ -34,9 +34,9 @@ clean_exit() { # TODO: Make output comparison less dependent on Kokkos/runtime log formatting. # Prefer comparing only stable test result lines instead of filtering known log noise. clean_output() { - grep -vF "[kkf-capture]" "$1" \ - | grep -vF "KokkosP:" \ - | grep -vF "-----------------------------------------------------------" + grep -vF -- "[kkf-capture]" "$1" \ + | grep -vF -- "KokkosP:" \ + | grep -vF -- "-----------------------------------------------------------" } rm -f *.h5 @@ -53,7 +53,7 @@ if [ $EXIT_CODE -ne 0 ]; then clean_exit $EXIT_CODE fi -if [ $COMPARE_OUTPUT -eq 0 ]; then +if (( COMPARE_OUTPUT == 0 )); then clean_exit 0 fi diff --git a/tests/multiple_views_capture/CMakeLists.txt b/tests/multiple_views_capture/CMakeLists.txt index 8e6d76c..9cd56e8 100644 --- a/tests/multiple_views_capture/CMakeLists.txt +++ b/tests/multiple_views_capture/CMakeLists.txt @@ -4,4 +4,4 @@ target_link_libraries(multiple_views_capture PRIVATE kkf_warnings kkf_sanitizers add_executable(multiple_views_capture_replay replay_main.cpp) target_link_libraries(multiple_views_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer) -add_replay_test(multiple_views_capture "") +add_replay_test(multiple_views_capture FALSE) diff --git a/tests/repeated_kernel_invocation/CMakeLists.txt b/tests/repeated_kernel_invocation/CMakeLists.txt index 59f9721..552c96b 100644 --- a/tests/repeated_kernel_invocation/CMakeLists.txt +++ b/tests/repeated_kernel_invocation/CMakeLists.txt @@ -4,4 +4,4 @@ target_link_libraries(repeated_kernel_invocation PRIVATE kkf_warnings kkf_saniti add_executable(repeated_kernel_invocation_replay replay_main.cpp) target_link_libraries(repeated_kernel_invocation_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer) -add_replay_test(repeated_kernel_invocation TRUE "--kkf-dump-kernel-invocation=2") +add_replay_test(repeated_kernel_invocation FALSE "--kkf-dump-kernel-invocation=2") diff --git a/tests/struct_functor/CMakeLists.txt b/tests/struct_functor/CMakeLists.txt index 8fa3eed..cda18ee 100644 --- a/tests/struct_functor/CMakeLists.txt +++ b/tests/struct_functor/CMakeLists.txt @@ -4,4 +4,4 @@ target_link_libraries(struct_functor PRIVATE kkf_warnings kkf_sanitizers Kokkos: add_executable(struct_functor_replay replay_main.cpp) target_link_libraries(struct_functor_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer) -add_replay_test(struct_functor "") +add_replay_test(struct_functor FALSE) diff --git a/tests/view_capture/CMakeLists.txt b/tests/view_capture/CMakeLists.txt index 0990931..6cd7f90 100644 --- a/tests/view_capture/CMakeLists.txt +++ b/tests/view_capture/CMakeLists.txt @@ -4,4 +4,4 @@ target_link_libraries(view_capture PRIVATE kkf_warnings kkf_sanitizers Kokkos::k add_executable(view_capture_replay replay_main.cpp) target_link_libraries(view_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer) -add_replay_test(view_capture "") +add_replay_test(view_capture FALSE) diff --git a/tests/view_scalar_capture/CMakeLists.txt b/tests/view_scalar_capture/CMakeLists.txt index bb6bdf9..462dfb1 100644 --- a/tests/view_scalar_capture/CMakeLists.txt +++ b/tests/view_scalar_capture/CMakeLists.txt @@ -4,4 +4,4 @@ target_link_libraries(view_scalar_capture PRIVATE kkf_warnings kkf_sanitizers Ko add_executable(view_scalar_capture_replay replay_main.cpp) target_link_libraries(view_scalar_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer) -add_replay_test(view_scalar_capture "") +add_replay_test(view_scalar_capture FALSE)