codspeed.yml

name: Codspeed Benchmarking

# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks
#   run on every commit for our benchmarks website.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
  push:
    branches: [develop]
  pull_request: { }
  workflow_dispatch: { }

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  NIGHTLY_TOOLCHAIN: nightly-2026-02-05

jobs:
  bench-codspeed:
    strategy:
      matrix:
        include:
          - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error vortex-mask vortex-compute" }
          - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" }
          - { shard: 3, name: "Main library", packages: "vortex" }
          - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" }
          - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
          - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
          - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" }
          - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks vortex-row" }
    name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh
      - uses: ./.github/actions/setup-prebuild
      - uses: ./.github/actions/system-info
      - run: |
          sudo apt-get update
          sudo apt-get install -y libc6-dbg
      - name: Install Codspeed
        uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
        with:
          tool: cargo-codspeed
      - name: Build benchmarks
        env:
          RUSTFLAGS: "-C target-feature=+avx2"
        run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench
      - name: Run benchmarks
        uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2
        with:
          run: bash scripts/bench-taskset.sh cargo codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}
          mode: "simulation"

  # Getting a GPU box is slow, in the future we can build on a box without one and only run
  # on GPU machines.
  bench-codspeed-cuda-build:
    if: github.repository == 'vortex-data/vortex'
    name: "Build Codspeed CUDA benchmarks"
    timeout-minutes: 30
    runs-on: >-
      runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-build
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Codspeed
        uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
        with:
          tool: cargo-codspeed
      - name: Build benchmarks
        run: |
          cargo codspeed build -m walltime \
            --bench bitpacked_cuda \
            --bench dynamic_dispatch_cuda \
            --bench alp_cuda \
            --bench date_time_parts_cuda \
            --bench dict_cuda \
            --bench fsst_cuda \
            --bench runend_cuda \
            --profile bench
      - name: Verify CUDA kernels were built
        run: test -f vortex-cuda/kernels/gen/release/dynamic_dispatch.ptx
      - name: Stage CUDA kernels with Codspeed artifacts
        run: cp -R vortex-cuda/kernels/gen/release target/codspeed/cuda-kernels
      - name: Upload benchmark executables
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
        with:
          name: codspeed-cuda-benchmarks
          path: target/codspeed/
          retention-days: 1
          if-no-files-found: error

  bench-codspeed-cuda:
    if: github.repository == 'vortex-data/vortex'
    needs: [bench-codspeed-cuda-build]
    strategy:
      matrix:
        include:
          - { shard: 1, name: "Bitpacked", benches: "bitpacked_cuda" }
          - { shard: 2, name: "Dynamic dispatch", benches: "dynamic_dispatch_cuda" }
          - { shard: 3, name: "Standalone kernels", benches: "alp_cuda date_time_parts_cuda dict_cuda fsst_cuda runend_cuda" }
    name: "Benchmark with Codspeed (CUDA Shard #${{ matrix.shard }} - ${{ matrix.name }})"
    timeout-minutes: 30
    runs-on: >-
      runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-${{ matrix.shard }}
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Display NVIDIA SMI details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
      - run: |
          sudo apt-get update
          sudo apt-get install -y libc6-dbg
      - name: Install Codspeed
        uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
        with:
          tool: cargo-codspeed
      - name: Download benchmark executables
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
        with:
          name: codspeed-cuda-benchmarks
          path: target/codspeed
      - name: Restore executable permissions
        run: find target/codspeed -type f -exec chmod +x {} +
      - name: Run benchmarks
        uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2
        env:
          CARGO_MANIFEST_DIR: ${{ github.workspace }}/vortex-cuda
          VORTEX_CUDA_KERNELS_DIR: ${{ github.workspace }}/target/codspeed/cuda-kernels
        with:
          run: cargo codspeed run $(printf -- '--bench %s ' ${{ matrix.benches }})
          token: ${{ secrets.CODSPEED_TOKEN }}
          mode: "walltime"