GPU: Enforce some GPU-kernels are RTC-compiled with correct WARP_SIZE as launch-bounds

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstructionProcessing.h

@@ -162,9 +162,10 @@ class GPUReconstructionProcessing : public GPUReconstruction
   // Interface to query name of a kernel
   template <class T, int32_t I>
   static const char* GetKernelName();
-  const std::string& GetKernelName(int32_t i) const { return mKernelNames[i]; }
+  static const std::string& GetKernelName(int32_t i) { return mKernelNames[i]; }
   template <class T, int32_t I = 0>
   static uint32_t GetKernelNum();
+  static uint32_t GetNKernels() { return mKernelNames.size(); }
 
   // Public queries for timers
   auto& getRecoStepTimer(RecoStep step) { return mTimersRecoSteps[getRecoStepNum(step)]; }

GPU/GPUTracking/Base/cuda/GPUReconstructionCUDAGenRTC.cxx

@@ -17,11 +17,13 @@
 #include "GPUReconstructionCUDA.h"
 #include "GPUParamRTC.h"
 #include "GPUDefParametersLoad.inc"
+#include "GPUKernelsWith1Warp.inc"
 #include <unistd.h>
 #include "Framework/SHA1.h"
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <filesystem>
+#include <algorithm>
 
 #include <oneapi/tbb.h>
 using namespace o2::gpu;
@@ -81,6 +83,11 @@ int32_t GPUReconstructionCUDA::genRTC(std::string& filename, uint32_t& nCompile)
       GPUFatal("AMD_EUS_PER_CU not set in the parameters provided for the AMD GPU, you can override this via --PROChipOverrideAMDEUSperCU [n]");
     }
   }
+  for (uint32_t i = 0; i < GetNKernels(); i++) {
+    if (std::find(gpuKernelsWith1Warp.begin(), gpuKernelsWith1Warp.end(), GetKernelName(i)) != gpuKernelsWith1Warp.end()) {
+      mParDevice->par_LB_maxThreads[i] = mWarpSize;
+    }
+  }
   const std::string launchBounds = o2::gpu::internal::GPUDefParametersExport(*mParDevice, true, mParDevice->par_AMD_EUS_PER_CU ? (mParDevice->par_AMD_EUS_PER_CU * mWarpSize) : 0) +
                                    "#define GPUCA_WARP_SIZE " + std::to_string(mWarpSize) + "\n";
   if (GetProcessingSettings().rtctech.printLaunchBounds || GetProcessingSettings().debugLevel >= 3) {

GPU/GPUTracking/CMakeLists.txt

@@ -299,6 +299,11 @@ set(GPU_DEFAULT_PARAMS_HEADER_DEVICE ${ON_THE_FLY_DIR}/GPUDefParametersDefaultsD
 generate_gpu_param_header("${GPU_PARAM_JSON_FILES}" "ALL" "${GPU_DEFAULT_PARAMS_HEADER}" "${GPU_DEFAULT_PARAMS_HEADER_DEVICE}" GPU_CONST_PARAM_ARCHITECTUES) # generate header with default GPU parameters for all architectures
 list(APPEND GENERATED_HEADERS_LIST ${GPU_DEFAULT_PARAMS_HEADER} ${GPU_DEFAULT_PARAMS_HEADER_DEVICE})
 
+file(GENERATE
+    OUTPUT "${ON_THE_FLY_DIR}/GPUKernelsWith1Warp.inc"
+    CONTENT "namespace o2::gpu { static const std::vector<std::string> gpuKernelsWith1Warp = {\"$<JOIN:$<TARGET_PROPERTY:O2_GPU_KERNELS,O2_GPU_KERNELS_FORCE_1_WARP>,\"$<COMMA> \">\"}; }")
+list(APPEND GENERATED_HEADERS_LIST ${GPU_DEFAULT_PARAMS_HEADER} ${ON_THE_FLY_DIR}/GPUKernelsWith1Warp.inc)
+
 set(HDRS_INSTALL ${HDRS_INSTALL} ${GENERATED_HEADERS_LIST})
 include(kernels.cmake)
 

GPU/GPUTracking/cmake/kernel_helpers.cmake

@@ -20,6 +20,7 @@ define_property(TARGET PROPERTY O2_GPU_KERNEL_FILES)
 define_property(TARGET PROPERTY O2_GPU_KERNEL_NO_FAST_MATH)
 define_property(TARGET PROPERTY O2_GPU_KERNEL_PARAMS)
 define_property(TARGET PROPERTY O2_GPU_KERNEL_STRING_PARAMS)
+define_property(TARGET PROPERTY O2_GPU_KERNELS_FORCE_1_WARP)
 set(O2_GPU_KERNEL_WRAPPER_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/GPU/include_gpu_onthefly")
 file(MAKE_DIRECTORY ${O2_GPU_KERNEL_WRAPPER_FOLDER})
 set(O2_GPU_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}/../")
@@ -184,3 +185,7 @@ function(o2_gpu_kernel_add_string_parameter)
     set_property(TARGET O2_GPU_KERNELS APPEND PROPERTY O2_GPU_KERNEL_STRING_PARAMS "${ARGV${i}}")
   endforeach()
 endfunction()
+
+function(o2_gpu_kernel_requires_1_warp kernel)
+  set_property(TARGET O2_GPU_KERNELS APPEND PROPERTY O2_GPU_KERNELS_FORCE_1_WARP "${kernel}")
+endfunction()

GPU/GPUTracking/kernels.cmake

@@ -150,3 +150,6 @@ o2_gpu_kernel_add_parameter(NEIGHBOURS_FINDER_MAX_NNEIGHUP  # Number of neighhbo
 
 o2_gpu_kernel_add_string_parameter(DEDX_STORAGE_TYPE                # Data type to use for intermediate storage of dEdx truncated mean inputs
                                    MERGER_INTERPOLATION_ERROR_TYPE) # Data type for storing intermediate track residuals for interpolation
+
+o2_gpu_kernel_requires_1_warp("GPUTPCCFDecodeZSLink")
+o2_gpu_kernel_requires_1_warp("GPUTPCCFDecodeZSDenseLink")