test-build-deploy.yml

name: ci
permissions: read-all
on:
  push:
    branches: [master]
    tags:
      - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions
    paths-ignore:
      - 'build-image/**'
      - '.github/workflows/build-image.yml'
  pull_request:
    paths-ignore:
      - 'build-image/**'
      - '.github/workflows/build-image.yml'

jobs:
  lint:
    runs-on: ubuntu-24.04
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
        # Commands in the Makefile are hardcoded with an assumed file structure of the CI container
        # Symlink ensures paths specified in previous commands don’t break
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Lint
        run: make BUILD_IN_CONTAINER=false lint
      - name: Check Vendor Directory
        run: make BUILD_IN_CONTAINER=false mod-check
      - name: Check Protos
        run: make BUILD_IN_CONTAINER=false check-protos
      - name: Check Modernize
        run: make BUILD_IN_CONTAINER=false check-modernize

  test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: amd64
            runner: ubuntu-24.04
          - name: arm64
            runner: ubuntu-24.04-arm
    name: test (${{ matrix.name }})
    runs-on: ${{ matrix.runner }}
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Run Tests
        run: make BUILD_IN_CONTAINER=false test

  test-no-race:
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: amd64
            runner: ubuntu-24.04
          - name: arm64
            runner: ubuntu-24.04-arm
    name: test-no-race (${{ matrix.name }})
    runs-on: ${{ matrix.runner }}
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Run Tests
        run: make BUILD_IN_CONTAINER=false test-no-race

  security:
    name: CodeQL
    if: github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-24.04
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3.29.5
        with:
          languages: go

      - name: Autobuild
        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3.29.5

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3.29.5

  
  discover-tags:
    # Single source of truth for the integration build tags. Greps //go:build headers under
    # ./integration/ and emits both a CSV (for go test -c tag list) and the matrix JSON
    # (cross-product of arches × tags) consumed by `integration` below.
    runs-on: ubuntu-24.04
    outputs:
      tags_csv: ${{ steps.discover.outputs.tags_csv }}
      matrix: ${{ steps.discover.outputs.matrix }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - id: discover
        shell: bash
        run: |
          TAGS=()
          while IFS= read -r line; do
            TAGS+=("$line")
          done < <(grep -hE "^//go:build " integration/*.go \
                   | sed -E 's|^//go:build ||' \
                   | sort -u | grep -v '^integration$')
          if [ "${#TAGS[@]}" -eq 0 ]; then
            echo "ERROR: no //go:build tags found under integration/" >&2
            exit 1
          fi
          TAGS_CSV=$(IFS=,; echo "${TAGS[*]}")
          echo "tags_csv=${TAGS_CSV}" >> "$GITHUB_OUTPUT"
          # Build matrix JSON: every tag runs on both amd64 and arm64.
          TAGS_JSON=$(printf '%s\n' "${TAGS[@]}" | jq -R . | jq -s .)
          MATRIX=$(jq -c -n --argjson tags "$TAGS_JSON" '{
            include: [
              ($tags[] | {runner: "ubuntu-24.04",     arch: "amd64", tags: .}),
              ($tags[] | {runner: "ubuntu-24.04-arm", arch: "arm64", tags: .})
            ]
          }')
          echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT"
          echo "Discovered tags: ${TAGS_CSV}"

  build:
    runs-on: ubuntu-24.04
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    outputs:
      image_tag: ${{ steps.image-tag.outputs.image_tag }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
        env:
          DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}}
          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Compute Image Tag
        id: image-tag
        run: echo "image_tag=$(make image-tag)" >> "$GITHUB_OUTPUT"
      - name: Build Image
        run: |
          touch build-image/.uptodate
          make BUILD_IN_CONTAINER=false
      - name: Save Images
        run: |
          mkdir /tmp/images
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false save-images
      - name: Create Docker Images Archive
        run: tar -cvf images.tar /tmp/images
      - name: Upload Docker Images Artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: Docker Images
          path: ./images.tar

  build-integration-tests:
    needs: [discover-tags]
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: ubuntu-24.04
            arch: amd64
          - runner: ubuntu-24.04-arm
            arch: arm64
    name: build-integration-tests (${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Compile Integration Test Binary
        # Tag list comes from the discover-tags job — single source of truth.
        run: |
          ALL_TAGS="slicelabels,integration,${{ needs.discover-tags.outputs.tags_csv }}"
          mkdir -p out/bin
          go test -c -tags="${ALL_TAGS}" -o out/bin/integration.test ./integration/
      - name: Generate Run-Pattern Manifest
        # For each build tag, derive a -test.run regex listing every TestX function in source files
        # gated by that tag. The integration job will read these to select which tests to run.
        shell: bash
        run: |
          IFS=',' read -ra TAGS <<< "${{ needs.discover-tags.outputs.tags_csv }}"
          for tag in "${TAGS[@]}"; do
            names=$(grep -lE "^//go:build.*\b${tag}\b" integration/*.go 2>/dev/null \
                    | xargs -r grep -hE "^func Test[A-Z]" \
                    | sed -E 's/func (Test[A-Za-z0-9_]+).*/\1/' | sort -u | tr '\n' '|' | sed 's/|$//')
            if [ -z "$names" ]; then
              echo "ERROR: no tests found for tag ${tag}" >&2
              exit 1
            fi
            printf '^(%s)$\n' "$names" > "out/bin/run-pattern-${tag}.txt"
            echo "tag=${tag} tests=$(echo "$names" | tr '|' '\n' | wc -l)"
          done
      - name: Stage Test Data
        # The integration tests read docs/configuration/*.yaml at runtime via getCortexProjectDir()
        # in integration/util.go. Bundling these so the integration job doesn't need a checkout.
        run: |
          mkdir -p out/testdata/docs
          cp -r docs/configuration out/testdata/docs/
      - name: Create Integration Tests Archive
        run: tar -C out -czvf integration-tests-${{ matrix.arch }}.tar.gz bin testdata
      - name: Upload Integration Tests Artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: integration-tests-${{ matrix.arch }}
          path: ./integration-tests-${{ matrix.arch }}.tar.gz

  integration:
    needs: [discover-tags, build, build-integration-tests, lint]
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 50
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.discover-tags.outputs.matrix) }}
    steps:
      - name: Download Docker Images Artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Docker Images
        # Load every saved docker image tar into the runner's docker daemon. Each tar in /tmp/images
        # was produced by `make save-images` in the build job (one file per image:tag-arch).
        run: |
          for img in /tmp/images/*; do
            [ -f "$img" ] || continue
            docker load -i "$img"
          done
      - name: Download Integration Tests Artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: integration-tests-${{ matrix.arch }}
      - name: Extract Integration Tests Archive
        run: tar -xzvf integration-tests-${{ matrix.arch }}.tar.gz
      - name: Preload Images
        # We download docker images used by integration tests so that all images are available
        # locally and the download time doesn't account in the test execution time, which is subject
        # to a timeout. Each pull is wrapped in a small retry helper so a transient Docker Hub /
        # quay.io registry hiccup (e.g. "context deadline exceeded") doesn't fail the whole job.
        run: |
          # Retry a command up to 3 times with exponential backoff (5s, then 10s). A transient
          # docker pull failure is retried; a genuine, persistent failure still fails the step
          # because the final `return 1` propagates under `bash -e`.
          retry() {
            local max_attempts=3 attempt=1 delay=5
            until "$@"; do
              if [ "$attempt" -ge "$max_attempts" ]; then
                echo "ERROR: '$*' failed after ${max_attempts} attempts." >&2
                return 1
              fi
              echo "WARNING: '$*' failed (attempt ${attempt}/${max_attempts}); retrying in ${delay}s..." >&2
              sleep "$delay"
              attempt=$((attempt + 1))
              delay=$((delay * 2))
            done
          }

          retry docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z
          retry docker pull consul:1.8.4
          retry docker pull quay.io/coreos/etcd:v3.5.29
          if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
            retry docker pull quay.io/cortexproject/cortex:v1.16.1
            retry docker pull quay.io/cortexproject/cortex:v1.17.2
            retry docker pull quay.io/cortexproject/cortex:v1.18.1
            retry docker pull quay.io/cortexproject/cortex:v1.19.1
            retry docker pull quay.io/cortexproject/cortex:v1.20.1
            retry docker pull quay.io/cortexproject/cortex:v1.21.0
          elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
            retry docker pull quay.io/cortexproject/cortex:v1.20.1
            retry docker pull quay.io/prometheus/prometheus:v3.8.1
          elif [ "$TEST_TAGS" = "integration_configs_db" ]; then
            retry docker pull postgres:9.6.16
          fi
          retry docker pull memcached:1.6.1
          retry docker pull redis:7.0.4-alpine
        env:
          TEST_TAGS: ${{ matrix.tags }}
      - name: Integration Tests
        timeout-minutes: 45
        run: |
          export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
          export IMAGE_TAG="${{ needs.build.outputs.image_tag }}"
          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${IMAGE_TAG}-${{ matrix.arch }}"
          export CORTEX_CHECKOUT_DIR="$PWD/testdata"
          PATTERN="$(cat bin/run-pattern-${{ matrix.tags }}.txt)"
          echo "Running integration tests on ${{ matrix.arch }} with image: ${CORTEX_IMAGE}"
          echo "Selecting tests via -test.run=${PATTERN}"
          ./bin/integration.test -test.timeout=2400s -test.v -test.count=1 -test.run="${PATTERN}"
        env:
          IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}

  deploy:
    needs: [build, test, lint, integration]
    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-24.04
    container:
      image: quay.io/cortexproject/build-image:master-5f643d518c
    steps:
      - name: Checkout Repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
        env:
          DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}}
          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
      - name: Sym link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Docker Images Artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Images
        run: |
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false load-images
      - name: Deploy
        run: |
          if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then
            docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD"
          fi
          if [ -n "$QUAY_REGISTRY_PASSWORD" ]; then
            docker login -u "$QUAY_REGISTRY_USER" -p "$QUAY_REGISTRY_PASSWORD" quay.io;
          fi
          export IMAGE_TAG="${{ needs.build.outputs.image_tag }}"
          ./push-images $NOQUAY
        env:
          DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}}
          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
          QUAY_REGISTRY_USER: ${{secrets.QUAY_REGISTRY_USER}}
          QUAY_REGISTRY_PASSWORD: ${{secrets.QUAY_REGISTRY_PASSWORD}}
          NOQUAY: ${{secrets.NOQUAY}}