Merge branch ngt-ci-pipeline into template

Author: OliverRietmann

.github/scripts/csv_to_md.py

@@ -1,8 +1,10 @@
 import argparse
 import csv
+import math
 import tabulate as tab
 
 parser = argparse.ArgumentParser()
+parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
 parser.add_argument('-b', '--baseline', required=True, help='Baseline CSV file')
 parser.add_argument('-c', '--current', required=True, help='Current CSV file')
 args = parser.parse_args()
@@ -11,29 +13,35 @@ def get_2d_list(csv_filename):
   with open(csv_filename) as csv_file:
     csv_reader = csv.reader(csv_file)
     next(csv_reader)
-    return [[str(name), float(mean), float(stdev)] for name, mean, stdev in csv_reader]
+    return [[str(name), float(mean), float(stdev), int(count)] for name, mean, stdev, count in csv_reader]
 
 table_baseline = get_2d_list(args.baseline)
 table_current = get_2d_list(args.current)
 
-def get_emoji(d, stdev):
-  z = 1.96 # 95% confidence interval
-  if d < -z * stdev:
+def student(x, sx, m, y, sy, n):
+  s = 0.0 if m < 2 or n < 2 else math.sqrt(((m - 1) * sx**2 + (n - 1) * sy**2) / (m + n - 2))
+  d = x - y
+  t = 0.0 if s == 0.0 else math.sqrt((n * m) / (n + m)) * d / s
+  return d, s, t
+
+def get_emoji(t):
+  quantile = 2.0 # 95% confidence interval
+  if t < -quantile:
     return ':green_circle:'
-  elif d > z * stdev:
+  elif t > quantile:
     return ':red_circle:'
   else:
     return ':white_circle:'
 
 table = []
 for baseline, current in zip(table_baseline, table_current):
-  baseline_name, baseline_mean, _ = baseline
-  name, mean, stdev = current
+  baseline_name, baseline_mean, baseline_stdev, count_baseline = baseline
+  name, mean, stdev, count = current
   assert(baseline_name == name)
-  diff = baseline_mean - mean
-  impact = 0.0 if stdev == 0.0 else diff / stdev
-  emoji = get_emoji(diff, stdev)
-  table.append([name, int(mean), f'{stdev:.2f}', int(diff), f'{impact:.2f}', emoji])
+  total_time = mean * (count // args.runs)
+  d, s, t = student(baseline_mean, baseline_stdev, args.runs, mean, stdev, args.runs)
+  emoji = get_emoji(t)
+  table.append([name, int(total_time), int(mean), f'{stdev:.2f}', f'{d:.2f}', f'{t:.2f}', emoji])
 
-header = ['name', 'mean (ms)', 'stdev \u03C3', 'diff \u0394', '\u0394 / \u03C3', '']
+header = ['name', 'total time (\u03BCs)', 'mean (\u03BCs)', 'stdev \u03C3', 'diff \u0394', 't', '']
 print(tab.tabulate(table, header, tablefmt="github"))

.github/scripts/profiler_ncu.py

@@ -0,0 +1,33 @@
+import argparse
+import csv
+import statistics
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
+
+kernel_dict = {}
+with open(args.input) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  for row in csv_reader:
+    full_name = row[4]
+    time = int(row[14]) / 1000.0
+    if len(full_name) > 5 and full_name[:5] == "krnl_":
+      name = full_name[5:]
+      if name in kernel_dict.keys():
+        kernel_dict[name].append(time)
+      else:
+        kernel_dict[name] = [time]
+
+data = [["name", "time", "stdev", "count"]]
+for name, time_list in kernel_dict.items():
+  count = len(time_list)
+  mean = statistics.mean(time_list)
+  stdev = 0 if count == 1 else statistics.stdev(time_list)
+  data.append([name, mean, stdev, count])
+
+with open(args.output, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)

.github/scripts/profiler_nsys.py

@@ -0,0 +1,33 @@
+import argparse
+import csv
+import statistics
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
+
+kernel_list = []
+with open(args.input) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  next(csv_reader)
+  next(csv_reader)
+  for row in csv_reader:
+    if row:
+      full_name = row[8]
+      count = int(row[2])
+      mean = float(row[3])
+      stdev = float(row[7])
+      if len(full_name) > 5 and full_name[:5] == "krnl_":
+        name = full_name[5:]
+        kernel_list.append([name, mean, stdev, count])
+
+kernel_list.sort(key = lambda row: row[0])
+
+data = [["name", "mean", "stdev", "count"]]
+data += kernel_list
+
+with open(args.output, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)

.github/scripts/profiler_rocprofv2.py

@@ -0,0 +1,33 @@
+import argparse
+import csv
+import statistics
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
+
+kernel_dict = dict({})
+with open(args.input) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  for row in csv_reader:
+    full_name = row[13]
+    time = (int(row[15]) - int(row[14])) / 1000.0
+    if len(full_name) > 5 and full_name[:5] == "krnl_":
+      name = full_name[5:-3]
+      if name in kernel_dict.keys():
+        kernel_dict[name].append(time)
+      else:
+        kernel_dict[name] = [time]
+
+data = [["name", "mean", "stdev", "count"]]
+for name, time_list in kernel_dict.items():
+  count = len(time_list)
+  mean = statistics.mean(time_list)
+  stdev = 0 if count == 1 else statistics.stdev(time_list)
+  data.append([name, mean, stdev, count])
+
+with open(args.output, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)

.github/scripts/profiler_standalone.py

@@ -0,0 +1,37 @@
+import argparse
+import csv
+import math
+import statistics
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-d', '--discard', type=int, default=0, help='Number of initial measurements to discard')
+parser.add_argument('-i', '--input', required=True, help='Input CSV file')
+parser.add_argument('-o', '--output', required=True, help='Output CSV file')
+args = parser.parse_args()
+
+time_dict = dict({})
+count_dict = dict({})
+with open(args.input) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  for row in csv_reader:
+    name = row[2]
+    time = float(row[3])
+    count = row[1]
+    if name in time_dict.keys():
+      time_dict[name].append(time)
+    else:
+      time_dict[name] = [time]
+      count_dict[name] = 1 if count == '' else int(count)
+
+data = [["name", "mean", "stdev", "count"]]
+for name, time_list in time_dict.items():
+  count = count_dict[name]
+  mean = statistics.mean(time_list[args.discard:]) / count
+  runs = len(time_list[args.discard:])
+  stdev = 0.0 if runs == 1 else statistics.stdev(time_list[args.discard:]) / math.sqrt(count)
+  data.append([name, mean, stdev, runs * count])
+
+with open(args.output, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)

.github/workflows/clean-test.yml

@@ -19,10 +19,6 @@ name: Clean PR checks
       # Warning: the check_* keys are magic and must consist of the string
       # "check_" followed by the applicable check name exactly. The
       # "description" field is only the human-readable label for the input.
-      'check_build/AliceO2/O2/o2/macOS':
-        description: build/AliceO2/O2/o2/macOS
-        type: boolean
-        default: true
       'check_build/AliceO2/O2/o2/macOS-arm':
         description: build/AliceO2/O2/o2/macOS-arm
         type: boolean
@@ -31,8 +27,8 @@ name: Clean PR checks
         description: build/O2/fullCI
         type: boolean
         default: true
-      'check_build/O2/o2-dataflow-cs8':
-        description: build/O2/o2-dataflow-cs8
+      'check_build/O2/o2-dataflow-slc9':
+        description: build/O2/o2-dataflow-slc9
         type: boolean
         default: true
       'check_build/O2/o2/aarch64':

.github/workflows/standalone-benchmark.yml

@@ -14,26 +14,46 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [nvidia-h100, nvidia-l40s, amd-mi300x, amd-w7900]
+        name: [cpu, nvidia-h100, nvidia-l40s, amd-mi300x, amd-w7900]
         include:
+          - name: cpu
+            runner: cern-nextgen-h100
+            cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=0
+            profiler_runs: 42
+            standalone_runs: 42
+            cpu_gpu: "-c"
           - name: nvidia-h100
             runner: cern-nextgen-h100
             cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=90
+            profiler_runs: 21
+            standalone_runs: 42
+            cpu_gpu: "-g --memSize 20000000000"
           - name: nvidia-l40s
             runner: cern-nextgen-l40s
             cmake_args: -DENABLE_CUDA=1 -DENABLE_HIP=0 -DCUDA_COMPUTETARGET=89
+            profiler_runs: 42
+            standalone_runs: 42
+            cpu_gpu: "-g --memSize 20000000000"
           - name: amd-mi300x
             runner: cern-nextgen-mi300x
             cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx942
+            profiler_runs: 42
+            standalone_runs: 42
+            cpu_gpu: "-g --memSize 20000000000"
           - name: amd-w7900
             runner: cern-nextgen-w7900
             cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=1 -DHIP_AMDGPUTARGET=gfx1100
+            profiler_runs: 42
+            standalone_runs: 42
+            cpu_gpu: "-g --memSize 20000000000"
     env:
       WORK_DIR: /cvmfs/alice.cern.ch
       ALIBUILD_ARCH_PREFIX: el9-x86_64/Packages
       MODULEPATH: /cvmfs/alice.cern.ch/etc/toolchain/modulefiles/el9-x86_64:/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles
       STANDALONE_DIR: /root/standalone
-      BENCHMARK_CSV: /root/${{ matrix.name }}.csv
+      BENCHMARK_CSV: standalone_${{ matrix.name }}.csv
+      PROFILER_CSV: profiler_${{ matrix.name }}.csv
+      TIMING_CA: ./ca -e 50kHz ${{ matrix.cpu_gpu }} --seed 0 --sync --debug 1 # Add --PROCdebugMarkdown 1 --runs 42 --runsInit 2 --PROCresetTimers 1 for benchmark runs
       LD_LIBRARY_PATH: /usr/local/cuda-13.0/compat
 
     name: ${{ matrix.name }}
@@ -47,9 +67,6 @@ jobs:
 
           curl -fL --retry 3 -o ${STANDALONE_DIR}/o2-simple-GPU.out https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/o2-simple-GPU.out
 
-          mkdir -p ${STANDALONE_DIR}/baseline
-          curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/${{ matrix.name }}.csv
-
           mkdir -p ${STANDALONE_DIR}/events
           curl -fL --retry 3 -o ${STANDALONE_DIR}/events/o2-simple.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/o2-simple.tar.xz
           tar -xf ${STANDALONE_DIR}/events/o2-simple.tar.xz -C ${STANDALONE_DIR}/events
@@ -68,38 +85,79 @@ jobs:
         env:
           DETERMINISTIC_MODE: GPU
 
-      - name: Test GPU Track Reconstruction
+      - name: Test Track Reconstruction
         run: |
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${STANDALONE_DIR}/ca -e o2-simple -g --seed 0 --memSize 20000000000 --sync --runs 1 --RTCenable --PROCdeterministicGPUReconstruction 1 --RTCoptConstexpr 1 --RTCoptSpecialCode 1 --debug 6
-          cmp ${STANDALONE_DIR}/GPU.out ${STANDALONE_DIR}/o2-simple-GPU.out
-          rm -rf ${STANDALONE_DIR}/GPU.out ${STANDALONE_DIR}/o2-simple-GPU.out ${STANDALONE_DIR}/events/o2-simple ${STANDALONE_DIR}/build
+          ${STANDALONE_DIR}/ca -e o2-simple ${{ matrix.cpu_gpu }} --seed 0 --sync --runs 1 --RTCenable --PROCdeterministicGPUReconstruction 1 --RTCoptConstexpr 1 --RTCoptSpecialCode 1 --debug 6
+          cmp ${STANDALONE_DIR}/*.out
+          rm -rf ${STANDALONE_DIR}/*.out ${STANDALONE_DIR}/events/o2-simple ${STANDALONE_DIR}/build
 
       - name: Build Non-Deterministic
         run: *build
         env:
           DETERMINISTIC_MODE: OFF
 
-      - name: Benchmark GPU Track Reconstruction
+      - name: Benchmark Track Reconstruction
+        run: |
+          source /etc/profile.d/modules.sh
+          module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
+          cd ${STANDALONE_DIR}
+          ${TIMING_CA} --debug 1 --runs ${{ matrix.standalone_runs }} --runsInit 0 --PROCdebugMarkdown 1 --PROCresetTimers 1 --PROCdebugCSV /root/${BENCHMARK_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_standalone.py --discard 0 --input /root/${BENCHMARK_CSV} --output /root/summary_${BENCHMARK_CSV}
+      
+      - name: Profiler - Nsight Compute
+        if: ${{ matrix.name == 'nvidia-h100' }}
         run: |
+          dnf install -y cuda-nsight-compute-13-1
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 12 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
+          ncu --set none --metrics gpu__time_duration.avg --export ${{ matrix.name }} --clock-control none --force-overwrite ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates ${{ matrix.name }}.ncu-rep
+          ncu --import ${STANDALONE_DIR}/${{ matrix.name }}.ncu-rep --print-units base --csv > /root/${PROFILER_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_ncu.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
 
-      - name: Display table on GitHub web
+      - name: Profiler - Nsight Systems
+        if: ${{ matrix.name == 'nvidia-l40s' }}
         run: |
+          dnf config-manager --add-repo "https://developer.download.nvidia.com/devtools/repos/rhel$(source /etc/os-release; echo ${VERSION_ID%%.*})/$(rpm --eval '%{_arch}' | sed s/aarch/arm/)/"
+          dnf install --nogpgcheck -y nsight-systems-cli-2026.2.1
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
-          python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input ${BENCHMARK_CSV} --output ${BENCHMARK_CSV}
-          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --baseline ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv --current ${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
-          rm -rf ${STANDALONE_DIR}/baseline
+          cd ${STANDALONE_DIR}
+          nsys profile -o ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates ${{ matrix.name }}.nsys-rep
+          nsys stats --report cuda_gpu_kern_sum --timeunit usec --force-export=true --format csv ${{ matrix.name }}.nsys-rep > /root/${PROFILER_CSV}
+          rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_nsys.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
+
+      - name: Profiler - rocprofv2
+        if: ${{ matrix.name == 'amd-mi300x' || matrix.name == 'amd-w7900' }}
+        run: |
+          source /etc/profile.d/modules.sh
+          module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
+          cd ${STANDALONE_DIR}
+          rocprofv2 --output-directory /root --output-file-name ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.standalone_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates results_${{ matrix.name }}.csv
+          rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
+          mv /root/results_${{ matrix.name }}.csv /root/${PROFILER_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_rocprofv2.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v6
         with:
           name: ${{ matrix.name }}-artifact
-          path: /root/${{ matrix.name }}.csv
+          path: "/root/*.csv"
+
+      - name: Display table on GitHub web
+        run: |
+          source /etc/profile.d/modules.sh
+          module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
+          mkdir -p ${STANDALONE_DIR}/baseline
+          curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${PROFILER_CSV}
+          curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${BENCHMARK_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.profiler_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} --current /root/summary_${PROFILER_CSV} >> ${GITHUB_STEP_SUMMARY}
+          echo -e "\n\n" >> ${GITHUB_STEP_SUMMARY}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.standalone_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} --current /root/summary_${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
+          rm -rf ${STANDALONE_DIR}/baseline
+        if: ${{ matrix.name != 'cpu' }}

.skills/build-software-stack/SKILL.md

@@ -0,0 +1 @@
+

.skills/create-a-new-file/SKILL.md

@@ -0,0 +1,20 @@
+---
+name: create-a-new-file
+description: describes how to create a new file
+---
+
+## Copyright statements
+
+The copyright statement for ALICE / O2 is found in ./o2-copyright-statement.md. It should be at the beginning of
+the new file using the proper commenting syntax for the given programming language. For example in C++ it should be commented via
+multiline comments:
+
+``` C++
+// Copyright 2019-<current-year> CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// ...
+```
+
+The only part which needs to be adapted by you is the `<current-year>` which you need to replace with the actual current year.
+
+