Add nsys profiler

Author: OliverRietmann

.github/scripts/profiler_rocm.py .github/scripts/profiler_amd.py.github/scripts/profiler_rocm.py renamed to .github/scripts/profiler_amd.py

@@ -0,0 +1,37 @@
+import argparse
+import csv
+import statistics
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
+
+ntsi_list = []
+with open(args.input) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  next(csv_reader)
+  next(csv_reader)
+  for row in csv_reader:
+    if row:
+      full_name = row[8]
+      instances = int(row[2])
+      time = int(row[1]) / 1000.0
+      sigma = float(row[7]) / 1000.0
+      if len(full_name) > 5 and full_name[:5] == "krnl_":
+        name = full_name[5:]
+        ntsi_list.append([name, time, sigma, instances])
+
+data = [["name", "time", "stdev"]]
+for name, time, sigma, instances in ntsi_list:
+  count = instances / args.runs
+  mean = time * count
+  stdev = sigma * count
+  data.append([name, mean, stdev])
+
+with open(args.output, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)
+

.github/scripts/profiler_nvidia.py

@@ -17,21 +17,30 @@ jobs:
         name: [nvidia-h100, nvidia-l40s, amd-mi300x, amd-w7900]
         include:
           - name: nvidia-h100
+            vendor: nvidia
             runner: cern-nextgen-h100
             cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=90
-            profiler: ""
+            vendor: nvidia
+            profiler: nsys profile -o nvidia-h100
+            profiler_post: nsys stats --report cuda_gpu_kern_sum --force-export=true --format csv nvidia-h100.nsys-rep >
           - name: nvidia-l40s
+            vendor: nvidia
             runner: cern-nextgen-l40s
             cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=89
-            profiler: ""
+            profiler: nsys profile -o nvidia-l40s
+            profiler_post: nsys stats --report cuda_gpu_kern_sum --force-export=true --format csv nvidia-l40s.nsys-rep >
           - name: amd-mi300x
+            vendor: amd
             runner: cern-nextgen-mi300x
             cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx942
             profiler: rocprofv2 --basenames --output-directory /root --output-file-name amd-mi300x
+            profiler_post: touch
           - name: amd-w7900
+            vendor: amd
             runner: cern-nextgen-w7900
             cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx1100
             profiler: rocprofv2 --basenames --output-directory /root --output-file-name amd-w7900
+            profiler_post: touch
     env:
       WORK_DIR: /cvmfs/alice.cern.ch
       ALIBUILD_ARCH_PREFIX: el9-x86_64/Packages
@@ -50,6 +59,10 @@ jobs:
         run: |
           mkdir -p ${STANDALONE_DIR}
 
+          curl -fL --retry 3 -o ${STANDALONE_DIR}/nsys.rpm https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2026_2/NsightSystems-linux-cli-public-2026.2.1.210-3763964.rpm
+          dnf install -y nsys.rpm
+          rm -f ${STANDALONE_DIR}/nsys.rpm
+
           curl -fL --retry 3 -o ${STANDALONE_DIR}/o2-simple-GPU.out https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/o2-simple-GPU.out
 
           mkdir -p ${STANDALONE_DIR}/baseline
@@ -90,17 +103,20 @@ jobs:
 
       - name: Benchmark GPU Track Reconstruction
         run: |
+          dnf install -y 
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
           ${{ matrix.profiler }} ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --debug 1 --runs 12 --runsInit 2 --PROCresetTimers 1 --PROCtimingCSV /root/${BENCHMARK_CSV}
+          ${{ matrix.profiler_post }} /root/${PROFILER_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
 
       - name: Display table on GitHub web
         run: |
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
-          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_rocm.py --runs 12 --input /root/${PROFILER_CSV} --output /root/${PROFILER_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_${{ matrix.vendor }}.py --runs 12 --input /root/${PROFILER_CSV} --output /root/${PROFILER_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_${{ matrix.vendor }}.py --runs 12 --input /root/${PROFILER_CSV} --output /root/${PROFILER_CSV}
           python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input /root/${BENCHMARK_CSV} --output /root/${BENCHMARK_CSV}
           python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --baseline ${STANDALONE_DIR}/baseline/${PROFILER_CSV} --current /root/${PROFILER_CSV} >> ${GITHUB_STEP_SUMMARY}
           echo -e "\n\n" >> ${GITHUB_STEP_SUMMARY}