Add CI on Ruche for NVIDIA A100

.github/workflows/ruche-cuda.yml

@@ -0,0 +1,183 @@
+# Portions of this workflow were adapted from Kokkos-FFT:
+# https://github.com/kokkos/kokkos-fft
+#
+# SPDX-FileCopyrightText: (C) The Kokkos-FFT development team, see Kokkos-FFT COPYRIGHT.md
+# SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+
+name: Ruche CUDA build and test
+
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "30 0 * * 1-5" # every weekday at midnight UTC
+
+permissions:
+  contents: read
+
+jobs:
+  build_and_test:
+    # only run on original repo
+    if: github.repository == 'CExA-project/kokkos-kernel-forge'
+    runs-on: [self-hosted, cuda]
+    timeout-minutes: 120
+    env:
+      KOKKOS_VERSION: 5.1.1
+      HDF5_VERSION: hdf5-1.14.6
+
+    strategy:
+      # allow all matrix jobs to perform even if errors occur
+      fail-fast: false
+      matrix:
+        build_type:
+        - Release
+        cxx_standard:
+        - 20
+        backend:
+        - name: CUDA-A100-GCC/13.4.0-CUDA/12.8.1-Static-CUDA_CONSTEXPR-CUDA_RELOCATABLE_DEVICE_CODE
+          modules: >-
+            gcc/13.4.0/gcc-15.1.0
+            cuda/12.8.1/none-none
+            cmake/3.31.9/gcc-15.1.0
+            python/3.14.0/gcc-15.1.0
+          test: -e gpu -g a100
+          kokkos_configure: >-
+            -DKokkos_ENABLE_CUDA=ON
+            -DKokkos_ENABLE_SERIAL=ON
+            -DKokkos_ARCH_AMPERE80=ON
+            -DKokkos_ENABLE_CUDA_CONSTEXPR=ON
+            -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON
+          kkf_configure: >-
+            -DKKF_BUILD_EXAMPLES=ON
+            -DKKF_BUILD_TESTS=ON
+            -DKKF_ENABLE_WARNINGS=ON
+            -DKKF_ENABLE_ASAN=OFF
+            -DKKF_ENABLE_UBSAN=OFF
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v7
+
+    - name: Compute Kokkos cache fingerprint
+      run: |
+        {
+          echo "RUNNER_OS=${RUNNER_OS:-unknown}"
+          echo "RUNNER_ARCH=${RUNNER_ARCH:-unknown}"
+          echo "KOKKOS_VERSION=$KOKKOS_VERSION"
+          echo "matrix.build_type=${{ matrix.build_type }}"
+          echo "matrix.cxx_standard=${{ matrix.cxx_standard }}"
+          echo "matrix.backend.name=${{ matrix.backend.name }}"
+          echo "matrix.backend.modules=${{ matrix.backend.modules }}"
+          echo "matrix.backend.kokkos_configure=${{ matrix.backend.kokkos_configure }}"
+        } > .kokkos-cache-fingerprint
+        cat .kokkos-cache-fingerprint
+
+    - name: Cache Kokkos install
+      id: cache-kokkos
+      uses: actions/cache@v5
+      with:
+        path: kokkos_install
+        key: kokkos-${{ runner.os }}-${{ runner.arch }}-${{ env.KOKKOS_VERSION }}-${{ hashFiles('.kokkos-cache-fingerprint') }}
+
+    - name: Build and install Kokkos
+      if: steps.cache-kokkos.outputs.cache-hit != 'true'
+      run: |
+        git clone --depth 1 --branch "$KOKKOS_VERSION" https://github.com/kokkos/kokkos.git
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake -S kokkos -B kokkos_build \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_STANDARD=${{ matrix.cxx_standard }} \
+            -DCMAKE_INSTALL_PREFIX=kokkos_install \
+            ${{ matrix.backend.kokkos_configure }}
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake --build kokkos_build --parallel
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake --install kokkos_build
+
+    - name: Compute HDF5 cache fingerprint
+      run: |
+        {
+          echo "RUNNER_OS=${RUNNER_OS:-unknown}"
+          echo "RUNNER_ARCH=${RUNNER_ARCH:-unknown}"
+          echo "HDF5_VERSION=$HDF5_VERSION"
+          echo "matrix.build_type=${{ matrix.build_type }}"
+          echo "matrix.backend.name=${{ matrix.backend.name }}"
+          echo "matrix.backend.modules=${{ matrix.backend.modules }}"
+          echo "HDF5_BUILD_HL_LIB=ON"
+          echo "HDF5_BUILD_TOOLS=OFF"
+          echo "HDF5_BUILD_EXAMPLES=OFF"
+          echo "BUILD_TESTING=OFF"
+        } > .hdf5-cache-fingerprint
+        cat .hdf5-cache-fingerprint
+
+    - name: Cache HDF5 install
+      id: cache-hdf5
+      uses: actions/cache@v5
+      with:
+        path: hdf5_install
+        key: hdf5-${{ runner.os }}-${{ runner.arch }}-${{ env.HDF5_VERSION }}-${{ hashFiles('.hdf5-cache-fingerprint') }}
+
+    - name: Build and install HDF5
+      if: steps.cache-hdf5.outputs.cache-hit != 'true'
+      run: |
+        git clone --depth 1 --branch "$HDF5_VERSION" https://github.com/HDFGroup/hdf5.git
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake -S hdf5 -B hdf5_build \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_INSTALL_PREFIX=hdf5_install \
+            -DHDF5_BUILD_HL_LIB=ON \
+            -DHDF5_BUILD_TOOLS=OFF \
+            -DHDF5_BUILD_EXAMPLES=OFF \
+            -DBUILD_TESTING=OFF
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake --build hdf5_build --parallel
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake --install hdf5_build
+
+    - name: Initialize the build tree
+      run: |
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e local \
+          cmake \
+            -G "Unix Makefiles" \
+            -S . \
+            -B build \
+            -DBUILDNAME=${{ matrix.backend.name }}-C++${{ matrix.cxx_standard }}-${{ matrix.build_type }} \
+            -DSITE=cea-ruche \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_STANDARD=${{ matrix.cxx_standard }} \
+            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+            -DCMAKE_CXX_FLAGS="-Wall -Wextra" \
+            -DCMAKE_PREFIX_PATH="$PWD/hdf5_install" \
+            -DKokkos_DIR=kokkos_install/lib64/cmake/Kokkos/ \
+            ${{ matrix.backend.kkf_configure }}
+
+    - name: Build
+      run: |
+        # Use make directly because cmake --build can complete the child build
+        # and then hang under Ruche's srun.
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          -e cpu \
+          make -C build -j40
+
+    - name: Test
+      run: |
+        # Use the Make replay target because ctest can complete launcher.sh
+        # and then hang under Ruche's srun.
+        run \
+          -m "${{ matrix.backend.modules }}" \
+          ${{ matrix.backend.test }} \
+          make -C build -j1 kkf_run_replay_tests

tests/CMakeLists.txt

@@ -2,21 +2,37 @@ enable_testing()
 
 set(TEST_LAUNCHER "${CMAKE_CURRENT_LIST_DIR}/launcher.sh")
 
-function(kkf_set_test_working_directory TEST_NAME)
+# CTest is unreliable through the Slurm launcher on Ruche. These targets run
+# the same replay tests inside a single GPU allocation.
+add_custom_target(kkf_run_replay_tests)
+
+function(add_replay_test TEST_NAME COMPARE_OUTPUT)
   set(TEST_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/work/${TEST_NAME}")
   file(MAKE_DIRECTORY "${TEST_WORKING_DIRECTORY}")
-  set_tests_properties("${TEST_NAME}" PROPERTIES WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}")
-endfunction()
 
-function(add_replay_test TEST_NAME COMPARE_OUTPUT)
-  add_test(NAME "${TEST_NAME}" COMMAND "${TEST_LAUNCHER}" $<TARGET_FILE:kkf_kokkos_hooks> $<TARGET_FILE:${TEST_NAME}>
-                                       $<TARGET_FILE:${TEST_NAME}_replay> ${COMPARE_OUTPUT} ${ARGN}
+  if(COMPARE_OUTPUT)
+    set(COMPARE_OUTPUT TRUE)
+  else()
+    set(COMPARE_OUTPUT FALSE)
+  endif()
+
+  set(TEST_COMMAND "${TEST_LAUNCHER}" "$<TARGET_FILE:kkf_kokkos_hooks>" "$<TARGET_FILE:${TEST_NAME}>"
+                   "$<TARGET_FILE:${TEST_NAME}_replay>" "${COMPARE_OUTPUT}" ${ARGN}
   )
 
+  add_test(NAME "${TEST_NAME}" COMMAND ${TEST_COMMAND} WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}")
+
   # Replay tests with fixed-address host allocations link to the unsanitized
   # replayer, while scalar replay tests use the sanitized replayer variant.
   set_tests_properties("${TEST_NAME}" PROPERTIES LABELS "replay;sanitizer_compatible")
-  kkf_set_test_working_directory("${TEST_NAME}")
+
+  add_custom_target(
+    "kkf_run_${TEST_NAME}"
+    COMMAND ${TEST_COMMAND}
+    WORKING_DIRECTORY "${TEST_WORKING_DIRECTORY}"
+    USES_TERMINAL VERBATIM
+  )
+  add_dependencies(kkf_run_replay_tests "kkf_run_${TEST_NAME}")
 endfunction()
 
 add_subdirectory(no_capture)

tests/launcher.sh

@@ -34,9 +34,9 @@ clean_exit() {
 # TODO: Make output comparison less dependent on Kokkos/runtime log formatting.
 # Prefer comparing only stable test result lines instead of filtering known log noise.
 clean_output() {
-  grep -vF "[kkf-capture]" "$1" \
-    | grep -vF "KokkosP:" \
-    | grep -vF "-----------------------------------------------------------"
+  grep -vF -- "[kkf-capture]" "$1" \
+    | grep -vF -- "KokkosP:" \
+    | grep -vF -- "-----------------------------------------------------------"
 }
 
 rm -f *.h5
@@ -53,7 +53,7 @@ if [ $EXIT_CODE -ne 0 ]; then
   clean_exit $EXIT_CODE
 fi
 
-if [ $COMPARE_OUTPUT -eq 0 ]; then
+if (( COMPARE_OUTPUT == 0 )); then
   clean_exit 0
 fi
 

tests/multiple_views_capture/CMakeLists.txt

@@ -4,4 +4,4 @@ target_link_libraries(multiple_views_capture PRIVATE kkf_warnings kkf_sanitizers
 add_executable(multiple_views_capture_replay replay_main.cpp)
 target_link_libraries(multiple_views_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer)
 
-add_replay_test(multiple_views_capture "")
+add_replay_test(multiple_views_capture FALSE)

tests/repeated_kernel_invocation/CMakeLists.txt

@@ -4,4 +4,4 @@ target_link_libraries(repeated_kernel_invocation PRIVATE kkf_warnings kkf_saniti
 add_executable(repeated_kernel_invocation_replay replay_main.cpp)
 target_link_libraries(repeated_kernel_invocation_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer)
 
-add_replay_test(repeated_kernel_invocation TRUE "--kkf-dump-kernel-invocation=2")
+add_replay_test(repeated_kernel_invocation FALSE "--kkf-dump-kernel-invocation=2")

tests/struct_functor/CMakeLists.txt

@@ -4,4 +4,4 @@ target_link_libraries(struct_functor PRIVATE kkf_warnings kkf_sanitizers Kokkos:
 add_executable(struct_functor_replay replay_main.cpp)
 target_link_libraries(struct_functor_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer)
 
-add_replay_test(struct_functor "")
+add_replay_test(struct_functor FALSE)

tests/view_capture/CMakeLists.txt

@@ -4,4 +4,4 @@ target_link_libraries(view_capture PRIVATE kkf_warnings kkf_sanitizers Kokkos::k
 add_executable(view_capture_replay replay_main.cpp)
 target_link_libraries(view_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer)
 
-add_replay_test(view_capture "")
+add_replay_test(view_capture FALSE)

tests/view_scalar_capture/CMakeLists.txt

@@ -4,4 +4,4 @@ target_link_libraries(view_scalar_capture PRIVATE kkf_warnings kkf_sanitizers Ko
 add_executable(view_scalar_capture_replay replay_main.cpp)
 target_link_libraries(view_scalar_capture_replay PRIVATE kkf_warnings Kokkos::kokkos kernel_replayer)
 
-add_replay_test(view_scalar_capture "")
+add_replay_test(view_scalar_capture FALSE)