ci-python.yml

name: CI Python

on:
  workflow_call:
    inputs:
      style-only:
        description: Only check the python code style (don't run test)
        required: true
        default: false
        type: boolean
  workflow_dispatch:
    inputs:
      style-only:
        description: Only check the python code style (don't run test)
        required: true
        default: false
        type: boolean

# We set `concurrency` to prevent having this workflow being run on code that is not up-to-date on a PR (a user make multiple push in a quick manner).
# But on the main branch, we don't want that behavior.
# Having the workflow run on each merge commit is something we would like, that could help us where a regression was made and missed by previous checks.
#
# For that we use `head_ref` that is only defined on `pull-request` and fallback to `run_id` (this is a counter, so it's value is unique between workflow call).
concurrency:
  group: ci-python-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  poetry-version: 2.2.1
  pytest-base-args: >-
    --log-level=DEBUG
    --durations=10
    --timeout=10
    -vv
    -x
  postgresql-version: 16
  libparsec-cache-version: 1

permissions:
  contents: read

jobs:
  test-python-server:
    # Only the server is in Python, and it is only meant to be run on Linux
    name: "(🐧 Linux only): 🐍 Python server tests"
    # Just a fail-safe timeout, see the fine grain per-task timeout instead
    timeout-minutes: 60
    # All linux jobs must run the same ubuntu version to avoid Rust caching issues!
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # pin v6.0.2
        timeout-minutes: 5

      - name: Retrieve runner specs
        id: runner-specs
        uses: ./.github/actions/system-info
        timeout-minutes: 1

      - name: Set apt mirror
        if: (!inputs.style-only)
        # GitHub Actions apt proxy is super unstable
        # see https://github.com/actions/runner-images/issues/7048
        shell: bash -eux -o pipefail {0}
        run: |
          COUNTRY=$(curl ipinfo.io | jq -r .country)
          if [ "$COUNTRY" == null ]; then
            COUNTRY=$(curl https://api.ipbase.com/v2/info | jq -r .data.location.country.alpha2)
          fi
          if [ "$COUNTRY" == null ]; then
            COUNTRY=$(curl 'https://api.ip2location.io/?format=json' | jq -r .country_code)
          fi
          MIRROR_FILE=$(mktemp)
          curl "http://mirrors.ubuntu.com/$COUNTRY.txt" > $MIRROR_FILE
          (
            # make sure there is a `\t` between URL and `priority:*` attributes
            printf 'http://azure.archive.ubuntu.com/ubuntu	priority:1\n';
            grep -e '^https' $MIRROR_FILE
          ) | sudo tee /etc/apt/mirrors.txt
          sudo sed -i 's/http:\/\/azure.archive.ubuntu.com\/ubuntu\//mirror+file:\/etc\/apt\/mirrors.txt/' /etc/apt/sources.list
        # Just continue if we cannot set APT repo
        continue-on-error: true

      - name: Configure PostgreSQL APT repository
        if: (!inputs.style-only)
        env:
          POSTGRES_APT_KEY_SHA_512: df557805862cd279f40819834af14e1723b18044df9dc22bea710b6980c98cc8ed39e75ed5c9adaa1932992710f1180f0491dc9437bfd485b4aa2b75776407d4  /usr/share/keyrings/postgresql-keyring.gpg
        shell: bash -ex -o pipefail {0}
        run: |
          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \
            | sudo gpg --dearmor --output /usr/share/keyrings/postgresql-keyring.gpg

          printenv POSTGRES_APT_KEY_SHA_512 | sha512sum --strict -c -

          echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
            | sudo tee /etc/apt/sources.list.d/postgresql.list

          sudo apt-get update
        timeout-minutes: 5

      # Note that ubuntu-22.04 runner already comes with PostgreSQL v14.x installed (minor versions can be updated).
      # See https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md#databases
      - name: Install PostgreSQL-${{ env.postgresql-version }}
        if: (!inputs.style-only)
        run: |
          # Retry the command until it succeed.
          # We retry because sometime the APT repo configured
          # by the runner seems drop the connection causing the command to fail.
          until sudo apt-get -y install ${{ env.POSTGRESQL_PACKAGE }}; do
            echo "Failed to install APT package, retrying...";
          done
        env:
          POSTGRESQL_PACKAGE: >-
            postgresql-${{ env.postgresql-version }}
        timeout-minutes: 5

      - uses: ./.github/actions/setup-python-poetry
        id: setup-python
        with:
          poetry-version: ${{ env.poetry-version }}
          project-path: ./server
        timeout-minutes: 10

      # libparsec is slow to compile, so we save it in cache and skip the
      # compilation entirely if the Rust code hasn't changed !

      # Key cache contains a hash of all the files that are used to produce _parsec.so
      # Hence if we have a cache hit we know that there is no need for a rebuild !
      - name: Setup cache-key
        id: cache-key
        run: echo "key=libparsec-v${{ env.libparsec-cache-version }}-${{ steps.runner-specs.outputs.os }}-${{ steps.runner-specs.outputs.release }}-${{ hashFiles('make.py', 'server/build.py', 'server/src/**', 'server/Cargo.toml', 'libparsec/**', 'rust-toolchain.toml', 'Cargo.toml', 'Cargo.lock') }}-no-bundle-extra-shared-libraries" >> $GITHUB_OUTPUT
        shell: bash

      - name: Restore libparsec if Rust hasn't been modified
        id: cache-libparsec
        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # pin v5.0.5
        with:
          key: ${{ steps.cache-key.outputs.key }}
          path: |
            server/parsec/_parsec.*.pyd
            server/parsec/_parsec.*.so
            # Note `server/parsec.libs/` is not included since we are going to build
            # with POETRY_LIBPARSEC_BUNDLE_EXTRA_SHARED_LIBRARIES=false (see below)
        timeout-minutes: 2

      - name: Setup Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # pin v1.16.1
        if: steps.cache-libparsec.outputs.cache-hit != 'true'
        with:
          # We setup the cache by hand, see below
          cache: false
          rustflags: "" # Set value to prevent default which deny cargo check warnings.
        timeout-minutes: 5

      - name: Retrieve Rust cache
        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # pin v2.9.1
        if: steps.cache-libparsec.outputs.cache-hit != 'true'
        with:
          # Cache is limited to 10Go (and cache is ~700mo per platform !). On top of that.
          # cache is only shared between master and the PRs (and not across PRs).
          # So we only save the cache on master build given it's the ones that are the
          # most likely to be reused.
          save-if: ${{ github.ref == 'refs/heads/master' }}
          key: ${{ steps.runner-specs.outputs.os }}-${{ steps.runner-specs.outputs.release }}
        timeout-minutes: 5

      - name: Install python deps
        shell: bash -ex {0}
        run: |
          poetry --directory ./server env info
          if ${{ env.SKIP_EXT_BUILD }}; then export POETRY_LIBPARSEC_BUILD_STRATEGY=no_build; fi
          python make.py python-ci-install

          # Make sure POETRY_LIBPARSEC_BUNDLE_EXTRA_SHARED_LIBRARIES=false worked
          if [ -d "./server/parsec.libs" ] && [ -n "$(ls -A ./server/parsec.libs)" ];
          then
            echo "::error title=Directory './server/parsec.libs' is not empty::Extra libs disabled but './server/parsec.libs/' is not empty: " ./server/parsec.libs/*
            exit 1
          fi
        env:
          SKIP_EXT_BUILD: ${{ steps.cache-libparsec.outputs.cache-hit == 'true' || inputs.style-only }}
          # No need to bundle given we compile and run on the same machine
          POETRY_LIBPARSEC_BUNDLE_EXTRA_SHARED_LIBRARIES: false
        timeout-minutes: 20

      - name: Install pre-commit
        id: pre-commit
        uses: ./.github/actions/use-pre-commit
        with:
          install-only: true

      - name: Check python code style
        shell: bash -eux {0}
        run: |
          poetry -P server run ruff version
          for step in ruff-format ruff-check pyright ${{ !inputs.style-only && 'sqlfluff sqlfluff-in-python' || '' }}; do
            python \
              ${{ steps.pre-commit.outputs.install-path }} \
              run \
              $step \
              --show-diff-on-failure \
              --verbose \
              --color=always \
              ${{ steps.pre-commit.outputs.suggested-args }}
          done
        timeout-minutes: 10

      # We only save the libparsec lib when:
      # - We are not in a github queue branch (they're a one time use so caching won't help)
      # - We haven't already cached it.
      - name: Save cached libparsec to be reused on later call
        if: >-
          (!inputs.style-only)
          && steps.cache-libparsec.outputs.cache-hit != 'true'
          && !contains(github.ref, 'gh-readonly-queue')
        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # pin v5.0.5
        with:
          key: ${{ steps.cache-key.outputs.key }}
          path: |
            server/parsec/_parsec.*.pyd
            server/parsec/_parsec.*.so
        timeout-minutes: 2

      - name: Basic tests
        if: (!inputs.style-only)
        run: poetry run pytest ${{ env.pytest-base-args }} tests -n auto
        timeout-minutes: 10
        working-directory: server

      - name: PostgreSQL tests
        if: (!inputs.style-only)
        run: poetry run pytest ${{ env.pytest-base-args }} tests -n auto --postgresql
        env:
          PGINSTALLATION: /usr/lib/postgresql/${{ env.postgresql-version }}/bin
        working-directory: server
        timeout-minutes: 10