tests.yml

# If you change this name also do it in ci_metrics.yml
name: Tests

# The workflow will always run, but the actual tests will only execute when:
# - The workflow is triggered manually.
# - The push is to main or a release branch.
# - There are changes to relevant files on a pull request.
# Note: If no conditions are met, the workflow will complete successfully without running tests
# to satisfy Branch Protection rules.

on:
  workflow_dispatch: # Activate this workflow manually
  push:
    branches:
      - main
      # release branches have the form v1.9.x
      - "v[0-9].*[0-9].x"
    # when we push, we do not need to satisfy Branch Protection rules, so we can ignore PRs that just change docs
    paths-ignore:
      - 'docs/**'
      - 'docs-website/**'
  pull_request:
    types:
      - opened
      - reopened
      - synchronize

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
  CORE_AZURE_CS_API_KEY: ${{ secrets.CORE_AZURE_CS_API_KEY }}
  AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
  AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
  PYTHON_VERSION: "3.10"
  HATCH_VERSION: "1.16.5"

jobs:
  check-if-changed:
  # This job checks if the relevant files have been changed.
  # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
  # This ensures the "Mark tests as completed" job always runs, which is required by Branch Protection rules.
    name: Check if changed
    runs-on: ubuntu-slim
    permissions:
      pull-requests: read
    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
    outputs:
      changes: ${{ steps.changes.outputs.changes }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check for changed code
        id: changes
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          filters: |
            changes:
              - "haystack/**/*.py"
              - "test/**/*.py"
              - "pyproject.toml"
              - ".github/utils/*.py"
              - "scripts/*.py"

  format:
    needs: check-if-changed
    # Run tests if: manual trigger, push to main/release, or relevant files changed
    if: |
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'push' ||
      (needs.check-if-changed.outputs.changes == 'true')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Ruff - check format and linting
        run: hatch run fmt-check

      - name: Check presence of license header
        run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check

  check-imports:
    needs: format
    runs-on: ubuntu-slim
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Check imports
        run: hatch run python .github/utils/check_imports.py

  unit-tests:
    name: Unit / ${{ matrix.os }}
    needs: format
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest
          - windows-latest
          - macos-latest
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Run
        run: hatch run test:unit

      - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        id: cache
        if: matrix.os == 'macos-latest'
        with:
          path: ${{ steps.hatch.outputs.env }}
          key: ${{ runner.os }}-${{ github.sha }}

      - name: Coveralls
        # We upload only coverage for ubuntu as handling both os
        # complicates the workflow too much for little to no gain
        if: matrix.os == 'ubuntu-latest'
        uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
        continue-on-error: true
        with:
          path-to-lcov: coverage.xml

  mypy:
    needs: unit-tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # With the default value of 1, there are corner cases where tj-actions/changed-files
          # fails with a `no merge base` error
          fetch-depth: 0
      - name: Get changed files
        id: files
        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
        with:
          files: |
            **/*.py
            pyproject.toml
          files_ignore: |
            test/**
            .github/**
            scripts/**
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        if: steps.files.outputs.any_changed == 'true'
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        if: steps.files.outputs.any_changed == 'true'
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Mypy
        if: steps.files.outputs.any_changed == 'true'
        run: |
          mkdir .mypy_cache
          hatch run test:types

  integration-tests-linux:
    name: Integration / ubuntu-latest
    needs: unit-tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"


      - name: Run
        run: hatch run test:integration-only-fast

  integration-tests-macos:
    name: Integration / macos-latest
    needs: unit-tests
    runs-on: macos-latest
    timeout-minutes: 30
    env:
      HAYSTACK_MPS_ENABLED: false

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        id: cache
        with:
          path: ${{ steps.hatch.outputs.env }}
          key: ${{ runner.os }}-${{ github.sha }}


      - name: Run
        run: hatch run test:integration-only-fast

  integration-tests-windows:
    name: Integration / windows-latest
    needs: unit-tests
    runs-on: windows-latest
    timeout-minutes: 30
    env:
      HAYSTACK_XPU_ENABLED: false

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Run
        run: hatch run test:integration-only-fast

  notify-slack-on-failure:
    if: failure() && github.ref_name == 'main'
    needs:
      - check-imports
      - mypy
      - integration-tests-linux
      - integration-tests-macos
      - integration-tests-windows
    runs-on: ubuntu-slim
    steps:
      - uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

  tests-completed:
    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
    name: Mark tests as completed
    runs-on: ubuntu-slim
    if: ${{ always() && !cancelled() }}
    needs:
      - check-imports
      - mypy
      - integration-tests-linux
      - integration-tests-macos
      - integration-tests-windows

    steps:
    - name: Mark tests as completed
      run: |
        if [ "${{ needs.check-imports.result }}" = "failure" ] ||
           [ "${{ needs.mypy.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-linux.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-macos.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-windows.result }}" = "failure" ]; then
          echo "Tests failed!"
          exit 1
        else
          echo "Tests completed!"
        fi