.gitlab-ci.yml

# Only spawn workflows for MRs or protected branches
workflow:
  auto_cancel:
    on_job_failure: all
    on_new_commit: interruptible
  rules:
  - if: $CI_COMMIT_REF_PROTECTED == "true"
    auto_cancel:
      on_job_failure: none
      on_new_commit: none
    when: always
  - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED != "true"
    when: never
  - when: always


stages:
- individual tests
- semi-fresh tests
- configuration tests

variables:
  # Most jobs require the submodules, those that don't will override this
  GIT_SUBMODULE_STRATEGY: recursive

  # Add a transfer progress bar for artifacts
  TRANSFER_METER_FREQUENCY: 2s

  # Use fastzip to package caches and artifacts
  FF_USE_FASTZIP: 'true'
  ARTIFACT_COMPRESSION_LEVEL: default
  CACHE_COMPRESSION_LEVEL: fastest

  # Retry various preliminary job stages (network errors)
  GET_SOURCES_ATTEMPTS: 3
  ARTIFACTS_DOWNLOAD_ATTEMPTS: 3
  EXECUTOR_JOB_SECTION_ATTEMPTS: 3


default:
  retry:
    max: 2
    when:
    - unknown_failure
    - api_failure
    - stuck_or_timeout_failure
    - runner_system_failure

.not external mr:
  # Some jobs require specialized hardware that isn't generally available to the average GitLab user,
  # this includes any of the `rice-*` tags as well as `saas-linux-*large-*` and `saas-linux-*-gpu-*`.
  # These jobs should never run in scenarios where said runners may not be available.
  rules: &only_runner_privileged
  - &never_if_runner_unprivileged
    # Never run in merge request (but not merged results) pipelines from external projects (forks)
    if: >-
      ($ENABLE_HPCTOOLKIT_CUSTOM_RUNNERS == "" || $ENABLE_HPCTOOLKIT_CUSTOM_RUNNERS == null)
      && $CI_COMMIT_REF_PROTECTED == "false"
      && $CI_MERGE_REQUEST_IID
      && ($CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == "" || $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == null)
      && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
    when: never
  - when: on_success


include:
# All code should pass pre-commit without error.
- component: gitlab.com/blue42u/ci.pre-commit/lite@0.2.0

# These images are based on classic distributions and contain no vendor software
- component: &ci_predeps gitlab.com/blue42u/ci.predeps/buildah@0.4.1
  inputs:
    <<: &ci_predeps_shared
      fallback_registry: registry.gitlab.com/hpctoolkit/hpctoolkit/ci.predeps
    name: rhel8-amd64
    containerfile: .ci.predeps/Containerfile.rhel8
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rhel9-amd64
    containerfile: .ci.predeps/Containerfile.rhel9
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rhel9-arm64
    containerfile: .ci.predeps/Containerfile.rhel9
    platform: linux/arm64
    job_tag: saas-linux-medium-arm64
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rhel9-ppc64le
    containerfile: .ci.predeps/Containerfile.rhel9
    platform: linux/ppc64le
    job_tag: rice-linux-medium-ppc64le
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: ubuntu20.04-amd64
    containerfile: .ci.predeps/Containerfile.ubuntu20.04
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: ubuntu22.04-amd64
    containerfile: .ci.predeps/Containerfile.ubuntu22.04
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: ubuntu22.04-arm64
    containerfile: .ci.predeps/Containerfile.ubuntu22.04
    platform: linux/arm64
    job_tag: saas-linux-medium-arm64
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: ubuntu22.04-ppc64le
    containerfile: .ci.predeps/Containerfile.ubuntu22.04
    platform: linux/ppc64le
    job_tag: rice-linux-medium-ppc64le
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: fedora39-amd64
    containerfile: .ci.predeps/Containerfile.fedora39
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: leap15-amd64
    containerfile: .ci.predeps/Containerfile.leap15
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: bare-amd64
    containerfile: .ci.predeps/Containerfile.bare
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: bare-arm64
    containerfile: .ci.predeps/Containerfile.bare
    platform: linux/arm64
    job_tag: saas-linux-medium-arm64
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: bare-ppc64le
    containerfile: .ci.predeps/Containerfile.bare
    platform: linux/ppc64le
    job_tag: rice-linux-medium-ppc64le

# Images based on (containing) Nvidia's CUDA Toolkit
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: cuda11.8-amd64
    containerfile: .ci.predeps/Containerfile.cuda
    build_args: CUDA_VERSION=11.8.0
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: cuda12.0-amd64
    containerfile: .ci.predeps/Containerfile.cuda
    build_args: CUDA_VERSION=12.0.1
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: cuda12.3-amd64
    containerfile: .ci.predeps/Containerfile.cuda
    build_args: CUDA_VERSION=12.3.2

# Images based on (containing) AMD's ROCm
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm5.3-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=5.3.3
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm5.4-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=5.4.2
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm5.5-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=5.5.1
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm5.6-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=5.6.1
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm5.7-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=5.7.1
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm6.0-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=6.0.2
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: rocm6.1-amd64
    containerfile: .ci.predeps/Containerfile.rocm
    build_args: ROCM_VERSION=6.1

# Image(s) based on (containing) Intel's Level Zero, IGC, GTPin, etc.
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: intel-amd64
    containerfile: .ci.predeps/Containerfile.intel
    build_args: ONEAPI_VERSION=2024.0.1

# Image(s) containing system software in uncommon arrangements
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: libintl-amd64
    containerfile: .ci.predeps/Containerfile.libintl

# Image(s) containing Spack-installed software for Spack-based verification
- component: *ci_predeps
  inputs:
    <<: *ci_predeps_shared
    name: spack
    containerfile: .ci.predeps/Containerfile.spack
    ignorefile: .ci.predeps/ignore.spack

'predeps: [spack]':
  before_script:
  - export PREDEPS_SECRET_spack_mirrors=/tmp/mirrors.yaml
  - >-
    printf "mirrors: {%s: {url: '%s', access_pair: ['%s', '%s'], binary: true, source: false}}"
    __ci_remote_cache_push_mirror oci://"$CI_REGISTRY_IMAGE"/spack-buildcache/v@SPACK_CURRENT_RELEASE@
    "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD"
    > "$PREDEPS_SECRET_spack_mirrors"

'predeps: [ubuntu22.04-ppc64le]':
  rules: *only_runner_privileged
'predeps: [rhel9-ppc64le]':
  rules: *only_runner_privileged
'predeps: [bare-ppc64le]':
  rules: *only_runner_privileged


# The vast majority of jobs in this sequence do a build and then run some tests. This is the base
# job spec for such jobs, configurable via a small number of variables.
# NB: We use --repeat in these jobs to detect bugs that cause sporadic test failures.
# We chose --repeat 7, since this statistically provides:
# - 99.2% confidence that this MR does not introduce a "blocking" bug that would prevent others'
#   work. We are very confident that a run will succeed >1/2 the time, and so require minimal
#   repeated commands to "push past" the issue. (`1 - pbinom(0, 7, 1 - 1/2) -> 0.992`)
# - 79.0% confidence that this MR does not introduce a bug that would "annoy" others during their
#   work. We are modestly confident that a run will succeed >4/5 of the time, in which case the
#   bug may not be especially noticable. (`1 - pbinom(0, 7, 1 - 4/5) -> 0.790`)
# Also, --repeat 7 is low enough that it doesn't slow down the CI too much.
#
# For the curious, the key formula to solve for the --repeat value is:
#     {# of repeats} > log(1 - {confidence}) / log({min prob of success})
# So a 99% confidence of 90% success rate requires a --repeat of at least 44.
.test job:
  interruptible: true
  cache:
  - key: meson-packagecache
    when: always
    paths:
    - subprojects/packagecache
  - key: ccache-$CI_JOB_NAME
    when: always
    paths:
    - .ccache/
  variables:
    CCACHE_DIR: '$CI_PROJECT_DIR/.ccache'
    SETUP_ARGS: '--native-file hpctoolkit-ci.ini'
    TEST_ARGS: ''
    # OpenMPI refuses to run as user 0 without these options set.
    OMPI_ALLOW_RUN_AS_ROOT: 1
    OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
  script:
  - &test_job_setup
    - mkdir -p logs
    # If we don't have sufficient perf events support, try to enable it.
    - if [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt 1 ]; then echo 1 > /proc/sys/kernel/perf_event_paranoid; fi
    # We also want kernel symbols available (kallsyms), so enable that too.
    - if [ "$(cat /proc/sys/kernel/kptr_restrict)" -gt 0 ]; then echo 0 > /proc/sys/kernel/kptr_restrict; fi
    # Usual sequence from here: setup, build and test
    - meson setup $SETUP_ARGS builddir
  - meson compile -C builddir 2>&1 | tee compile.log
  - meson test -C builddir -j $(nproc) --maxfail 3 --repeat 7 --print-errorlogs $TEST_ARGS
  after_script:
  - |
    echo "To reproduce this build, run:"
    echo "    podman run --rm -it -v ./:/hpctoolkit:ro --workdir /hpctoolkit \\"
    echo "      -e OMPI_ALLOW_RUN_AS_ROOT=1 -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \\"
    echo "      -e SETUP_ARGS='$SETUP_ARGS' -e TEST_ARGS='$TEST_ARGS'$EXTRA_ENVS \\"
    echo "      $CI_JOB_IMAGE"
    echo "and repeat the commands listed in the job log above, with builddir replaced with /tmp/builddir."
    echo "If using VSCode, you may also try using the appropriate devcontainer."
  - if [ -f compile.log ]; then ./ci/cc-diagnostics.py cq.json compile.log; fi
  artifacts:
    reports:
      codequality: cq.json
      junit: builddir/meson-logs/testlog.junit.xml
    when: always


# Individual test runs, each on a single known software stack
.individual test job:
  extends: .test job
  stage: individual tests
.individual test job std:
  extends: .individual test job
  tags: [saas-linux-medium-amd64]
ubuntu20.04 amd64:
  extends: .individual test job std
  needs: ['predeps: [ubuntu20.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU20_04_AMD64
ubuntu22.04 amd64:
  extends: .individual test job std
  needs: ['predeps: [ubuntu22.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU22_04_AMD64
ubuntu22.04 arm64:
  extends: .individual test job std
  needs: ['predeps: [ubuntu22.04-arm64]']
  image: $PREDEPS_IMAGE_UBUNTU22_04_ARM64
  tags: [saas-linux-medium-arm64]
ubuntu22.04 ppc64le:
  extends: .individual test job std
  needs: ['predeps: [ubuntu22.04-ppc64le]']
  image: $PREDEPS_IMAGE_UBUNTU22_04_PPC64LE
  tags: [rice-linux-medium-ppc64le]
  rules: *only_runner_privileged
leap15 amd64:
  extends: .individual test job std
  needs: ['predeps: [leap15-amd64]']
  image: $PREDEPS_IMAGE_LEAP15_AMD64
rhel8 amd64:
  extends: .individual test job std
  needs: ['predeps: [rhel8-amd64]']
  image: $PREDEPS_IMAGE_RHEL8_AMD64
  variables:
    # FIXME: MVAPICH2 reacts badly in VMs, it claims to have no threading support. The same effect
    # can't be reproduced in a local container, for whatever reason. Disable the MPI tests on this
    # MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
rhel9 amd64:
  extends: .individual test job std
  needs: ['predeps: [rhel9-amd64]']
  image: $PREDEPS_IMAGE_RHEL9_AMD64
  variables:
    # FIXME: This particular combination of PAPI et al. seems to cause hpcrun to hang, always.
    # Disable PAPI in the build until a proper investigation can be carried out.
    SETUP_ARGS: -Dpapi=disabled
    # FIXME: MVAPICH2 reacts badly in VMs, it claims to have no threading support. The same effect
    # can't be reproduced in a local container, for whatever reason. Disable the MPI tests on this
    # MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
rhel9 arm64:
  extends: .individual test job std
  needs: ['predeps: [rhel9-arm64]']
  image: $PREDEPS_IMAGE_RHEL9_ARM64
  tags: [saas-linux-medium-arm64]
  variables:
    # FIXME: This particular combination of PAPI et al. seems to cause hpcrun to hang, always.
    # Disable PAPI in the build until a proper investigation can be carried out.
    SETUP_ARGS: -Dpapi=disabled
    # FIXME: MVAPICH2 reacts badly in VMs, it claims to have no threading support. The same effect
    # can't be reproduced in a local container, for whatever reason. Disable the MPI tests on this
    # MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
rhel9 ppc64le:
  extends: .individual test job std
  needs: ['predeps: [rhel9-ppc64le]']
  image: $PREDEPS_IMAGE_RHEL9_PPC64LE
  tags: [rice-linux-medium-ppc64le]
  rules: *only_runner_privileged
  variables:
    # FIXME: This particular combination of PAPI et al. seems to cause hpcrun to hang, always.
    # Disable PAPI in the build until a proper investigation can be carried out.
    SETUP_ARGS: -Dpapi=disabled
    # FIXME: MVAPICH2 reacts badly in VMs, it claims to have no threading support. The same effect
    # can't be reproduced in a local container, for whatever reason. Disable the MPI tests on this
    # MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
fedora39 amd64:
  extends: .individual test job std
  needs: ['predeps: [fedora39-amd64]']
  image: $PREDEPS_IMAGE_FEDORA39_AMD64
  variables:
    # FIXME: This particular version of OpenMPI reacts badly in VMs, hpcprof-mpi hangs. The same
    # effect can't be reproduced in a local container, for whatever reason. Disable the MPI tests
    # on this MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi

.individual test job bare:
  extends: .individual test job
  tags: [saas-linux-medium-amd64]
bare amd64:
  extends: .individual test job bare
  needs: ['predeps: [bare-amd64]']
  image: $PREDEPS_IMAGE_BARE_AMD64
bare arm64:
  extends: .individual test job bare
  needs: ['predeps: [bare-arm64]']
  image: $PREDEPS_IMAGE_BARE_ARM64
  tags: [saas-linux-medium-arm64]
bare ppc64le:
  extends: .individual test job bare
  needs: ['predeps: [bare-ppc64le]']
  image: $PREDEPS_IMAGE_BARE_PPC64LE
  tags: [rice-linux-medium-ppc64le]
  rules: *only_runner_privileged

.individual test job cuda:
  extends: .individual test job
  tags: [rice-linux-large-amd64-gpu-p100]
  rules: *only_runner_privileged
  variables:
    TEST_ARGS: >-
      --suite opencl
      --suite cuda
      --suite hpcstruct
cuda11.8 amd64:
  extends: .individual test job cuda
  needs: ['predeps: [cuda11.8-amd64]']
  image: $PREDEPS_IMAGE_CUDA11_8_AMD64
cuda12.0 amd64:
  extends: .individual test job cuda
  needs: ['predeps: [cuda12.0-amd64]']
  image: $PREDEPS_IMAGE_CUDA12_0_AMD64
cuda12.3 amd64:
  extends: .individual test job cuda
  needs: ['predeps: [cuda12.3-amd64]']
  image: $PREDEPS_IMAGE_CUDA12_3_AMD64

.individual test job rocm:
  extends: .individual test job
  tags: [rice-linux-large-amd64-gpu-amd]
  rules: *only_runner_privileged
  variables:
    TEST_ARGS: >-
      --suite opencl
      --suite rocm
rocm5.3 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm5.3-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_3_AMD64
rocm5.4 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm5.4-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_4_AMD64
rocm5.5 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm5.5-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_5_AMD64
rocm5.6 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm5.6-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_6_AMD64
rocm5.7 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm5.7-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_7_AMD64
rocm6.0 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm6.0-amd64]']
  image: $PREDEPS_IMAGE_ROCM6_0_AMD64
rocm6.1 amd64:
  extends: .individual test job rocm
  needs: ['predeps: [rocm6.1-amd64]']
  image: $PREDEPS_IMAGE_ROCM6_1_AMD64

intel amd64:
  extends: .individual test job std
  tags: [rice-linux-large-amd64-gpu-intel]
  rules: *only_runner_privileged
  needs: ['predeps: [intel-amd64]']
  image: $PREDEPS_IMAGE_INTEL_AMD64
  variables:
    # FIXME: Intel MPI doesn't work on a single node, it tries to ssh to localhost and doesn't get
    # very far as a result. Don't try to run the MPI tests here.
    TEST_ARGS: >-
      --suite opencl
      --suite level0
      --no-suite mpi


# Selected semi-fresh test runs. These regenerate suitable parts of the test data
# before running the tests, thus testing that "freshening" the test data doesn't
# break any tests. For example, an uncaught change to the output format.
.semifresh test job:
  extends: .test job
  stage: semi-fresh tests
  rules: *only_runner_privileged
  variables:
    SETUP_ARGS: ''
    REGEN_SUITES: ''
  script:
  - *test_job_setup
  - meson compile -C builddir $(printf 'regen-testdata-%s\n' $REGEN_SUITES)
  - git add --intent-to-add tests/data/
  - meson compile -C builddir $(printf 'patches-testdata-%s\n' $REGEN_SUITES)
  - mkdir testdata-patches/
  - cp -t testdata-patches/ builddir/tests/data/*.patch
  - find testdata-patches/ -type f -size 0 -delete
  # Make sure we run the tests once in the same build directory that generated them
  - meson test -C builddir -j $(nproc) --maxfail 3 --print-errorlogs $TEST_ARGS
  # Then wipe the build directory and re-test against a "sterilized" build
  - meson setup --wipe builddir
  - meson test -C builddir -j $(nproc) --maxfail 3 --repeat 6 --print-errorlogs $TEST_ARGS
  after_script:
  - EXTRA_ENVS=" -e REGEN_SUITES='$REGEN_SUITES'"
  - !reference [.test job, after_script]
  artifacts:
    expire_in: 1 day
    reports:
      junit: builddir/meson-logs/testlog.junit.xml
    paths:
    - testdata-patches/
    when: always
semifresh none cpu:
  extends: .semifresh test job
  needs: ['predeps: [ubuntu20.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU20_04_AMD64
  tags: [saas-linux-medium-amd64]
  variables:
    REGEN_SUITES: none cpu
semifresh nvidia sw-cuda:
  extends: .semifresh test job
  needs: ['predeps: [cuda11.8-amd64]']
  image: $PREDEPS_IMAGE_CUDA11_8_AMD64
  tags: [rice-linux-large-amd64-gpu-p100]
  variables:
    REGEN_SUITES: nvidia sw-cuda
semifresh amd:
  extends: .semifresh test job
  needs: ['predeps: [rocm5.7-amd64]']
  image: $PREDEPS_IMAGE_ROCM5_7_AMD64
  tags: [rice-linux-large-amd64-gpu-amd]
  variables:
    REGEN_SUITES: amd

# For easy access, the regenerated test data is made available as a single job artifact archive
fresh testdata:
  stage: semi-fresh tests
  image: docker.io/alpine
  needs:
  - semifresh none cpu
  - semifresh nvidia sw-cuda
  - semifresh amd
  rules: *only_runner_privileged
  when: always
  script:
  - apk add git
  - ls -l testdata-patches/
  - git apply --index --check -- testdata-patches/*.patch
  artifacts:
    expose_as: "Test data refresh patches"
    expire_in: 3 days
    paths:
    - testdata-patches/


# Compiler compatibility checks. There are a lot of different breeds of compilers out there,
# and we want to be widely compatible with a wide range of them. Test the ones that aren't
# the defaults in the individual tests, and ensure the wraps are tested here as well.
.compiler compat test job:
  extends: .test job
  stage: configuration tests
  tags: [saas-linux-medium-amd64]
  variables:
    SETUP_ARGS: >-
      --native-file hpctoolkit-ci.ini
      --native-file $JOB_CC.ini
      --wrap-mode=forcefallback
compiler gcc8:
  extends: .compiler compat test job
  needs: ['predeps: [rhel8-amd64]']
  image: $PREDEPS_IMAGE_RHEL8_AMD64
  variables:
    JOB_CC: gcc
    # FIXME: MVAPICH2 reacts badly in VMs, it claims to have no threading support. The same effect
    # can't be reproduced in a local container, for whatever reason. Disable the MPI tests on this
    # MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
compiler clang10:
  extends: .compiler compat test job
  needs: ['predeps: [ubuntu20.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU20_04_AMD64
  variables:
    JOB_CC: clang10
compiler clang16:
  extends: .compiler compat test job
  needs: ['predeps: [fedora39-amd64]']
  image: $PREDEPS_IMAGE_FEDORA39_AMD64
  variables:
    JOB_CC: clang
    # FIXME: This particular version of OpenMPI reacts badly in VMs, hpcprof-mpi hangs. The same
    # effect can't be reproduced in a local container, for whatever reason. Disable the MPI tests
    # on this MPI for now until a proper fix can be figured out.
    TEST_ARGS: --no-suite mpi
compiler icx:
  extends: .compiler compat test job
  tags: [rice-linux-large-amd64-gpu-intel]
  rules: *only_runner_privileged
  needs: ['predeps: [intel-amd64]']
  image: $PREDEPS_IMAGE_INTEL_AMD64
  variables:
    JOB_CC: icx
    # FIXME: Intel MPI doesn't work on a single node, it tries to ssh to localhost and doesn't get
    # very far as a result. Don't try to run the MPI tests here.
    TEST_ARGS: --no-suite mpi


# Non-default option checks. There are a small number of options that users/packagers can
# configure to their whims. These tests check that changing them from the defaults doesn't
# break anything along the way.
.option test job:
  extends: .test job
  stage: configuration tests
  tags: [saas-linux-medium-amd64]
'option: [ubuntu22.04 amd64]':
  extends: .option test job
  needs: ['predeps: [ubuntu22.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU22_04_AMD64
  parallel:
    matrix:
    - SETUP_ARGS:
      - --buildtype=release
      - -Dvalgrind_annotations=true
    - SETUP_ARGS:
      - -Dpapi=disabled
      - -Dpython=disabled
      - -Dopencl=disabled
      TEST_ARGS: --suite hpcrun
    - SETUP_ARGS:
      - -Dmanpages=disabled
      - -Dextended_tests=disabled
      - -Dhpcprof_mpi=disabled
      TEST_ARGS: --suite none
'option: [bare amd64]':
  extends: .option test job
  needs: ['predeps: [bare-amd64]']
  image: $PREDEPS_IMAGE_BARE_AMD64
  parallel:
    matrix:
    - SETUP_ARGS:
      - --buildtype=release
      - -Dvalgrind_annotations=true
    - SETUP_ARGS:
      - -Dopencl=disabled
      TEST_ARGS: --suite hpcrun
    - SETUP_ARGS:
      - -Dextended_tests=disabled
      TEST_ARGS: --suite none
'option: [cuda amd64]':
  extends: .option test job
  needs: ['predeps: [cuda12.0-amd64]']
  image: $PREDEPS_IMAGE_CUDA12_0_AMD64
  parallel:
    matrix:
    - SETUP_ARGS:
      - -Dcuda=disabled
      TEST_ARGS: --suite hpcrun
    - SETUP_ARGS:
      - -Dextended_tests=disabled
      TEST_ARGS: --suite none
'option: [rocm amd64]':
  extends: .option test job
  needs: ['predeps: [rocm6.1-amd64]']
  image: $PREDEPS_IMAGE_ROCM6_1_AMD64
  parallel:
    matrix:
    - SETUP_ARGS:
      - -Drocm=disabled
      TEST_ARGS: --suite hpcrun
    - SETUP_ARGS:
      - -Dextended_tests=disabled
      TEST_ARGS: --suite none
'option: [intel amd64]':
  extends: .option test job
  tags: [rice-linux-large-amd64-gpu-intel]
  rules: *only_runner_privileged
  needs: ['predeps: [intel-amd64]']
  image: $PREDEPS_IMAGE_INTEL_AMD64
  parallel:
    matrix:
    - SETUP_ARGS:
      - -Dlevel0=disabled
      - -Dgtpin=disabled
      TEST_ARGS: --suite hpcrun
    - SETUP_ARGS:
      - -Dextended_tests=disabled
      TEST_ARGS: --suite none


# Check that (parts of) the build pass under specific off-configurations.
'setup: [no libdw]':  # Libelf but not libdw
  extends: .test job
  stage: configuration tests
  needs: ['predeps: [ubuntu22.04-amd64]']
  image: $PREDEPS_IMAGE_UBUNTU22_04_AMD64
  variables:
    SETUP_ARGS: ''
  before_script:
  - apt-get purge -yqq libdw-dev libdw1
  script: *test_job_setup
'setup: [libintl]':  # gettext from libintl instead of glibc
  extends: .test job
  stage: configuration tests
  needs: ['predeps: [libintl-amd64]']
  image: $PREDEPS_IMAGE_LIBINTL_AMD64
  variables:
    SETUP_ARGS: >-
      --native-file hpctoolkit-ci.ini
      --wrap-mode=forcefallback


# Check that Spack develop can properly build HPCToolkit in a small matrix of configurations.
'spack install':
  interruptible: true
  stage: configuration tests
  image:
    name: $PREDEPS_IMAGE_SPACK
    entrypoint: ['spack-env']
  needs: ['predeps: [spack]']
  tags: [saas-linux-medium-amd64]
  rules:
  - *never_if_runner_unprivileged  # Expensive and not necessary for review
  - if: $CI_COMMIT_REF_PROTECTED == "true"
    allow_failure: false
  - allow_failure: true
  cache:
  - key: spack-src
    paths:
    - .cache/spack-src/
  before_script:
  - mkdir -p .cache/spack-src
  - spack config add config:source_cache:"$CI_PROJECT_DIR"/.cache/spack-src
  script:
  # Update to the latest Spack develop, from whatever we have here
  - git -C $SPACK_ROOT pull
  # Install the environment, running the tests along the way
  - spack env create -d /tmp/env ci/tests.spack.yaml
  # FIXME: Apparently the reuse concretization allows deprecated versions to appear in the final
  # spec? For now allow installing deprecated versions as needed until this can be properly fixed.
  - spack -D /tmp/env install --fail-fast --test=root --deprecated
  - spack -D /tmp/env test run --log-file spack.junit.xml --log-format junit hpctoolkit
  after_script:
  - |
    echo "To reproduce:"
    echo "    cd path/to/hpctoolkit"
    echo "    podman run --rm -it -v ./:/hpctoolkit:O --workdir /hpctoolkit $CI_JOB_IMAGE"
  artifacts:
    reports:
      junit: spack.junit.xml