integration.yml

name: Integration

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

env:
  CARGO_TERM_COLOR: always
  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse

concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: true

jobs:
  integration:
    name: Integration (${{ matrix.name }})
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: lifecycle
            namespaces: >-
              test-lifecycle
              test-invalid-secret
              test-missing-secret
              test-rw-schema
              test-ro-schema
            needs_non_pg_snapshot: false
          - name: error_paths
            namespaces: >-
              test-non-pg-data
              test-wrong-bucket
            needs_non_pg_snapshot: true

          - name: ttls
            namespaces: >-
              test-min-ttl
            needs_non_pg_snapshot: true

          - name: switchover
            namespaces: >-
              test-switchover
            needs_non_pg_snapshot: false

          - name: consecutive_failures
            namespaces: >-
              test-consecutive-failures
              test-phase-stuck
              test-subresource-reset
            needs_non_pg_snapshot: true

          - name: persistent_schemas
            namespaces: >-
              test-persistent-schemas
              test-ps-conflict
              test-ps-skip-missing
            needs_non_pg_snapshot: false

    steps:
      - uses: actions/checkout@v6

      - name: Configure toolchain
        run: |
          rustup toolchain install --profile minimal --no-self-update stable
          rustup default stable

      - uses: Swatinem/rust-cache@v2

      - name: Build operator
        run: cargo build --bin operator

      - name: Generate CRDs
        run: cargo run --bin gen-crds > crds.yaml

      - name: Install crane
        run: |
          CRANE_VERSION=v0.20.3
          curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" \
            | sudo tar -xzf - -C /usr/local/bin crane

      - name: Install kind
        run: |
          curl -fsSLo ./kind https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
          chmod +x ./kind
          sudo mv ./kind /usr/local/bin/kind

      - name: Create kind cluster
        run: |
          cat <<EOF | kind create cluster --config=-
          kind: Cluster
          apiVersion: kind.x-k8s.io/v1alpha4
          nodes:
            - role: control-plane
          EOF
          kubectl cluster-info
          kubectl wait --for=condition=Ready nodes --all --timeout=120s

      - name: Apply CRDs
        run: kubectl apply -f crds.yaml

      - name: Create operator namespace and config
        run: |
          kubectl create namespace pgro-system

          # Expose the host-network operator to in-cluster jobs so the
          # snapshot-list callback can reach it.
          HOST_IP=$(docker inspect kind-control-plane \
            --format '{{ .NetworkSettings.Networks.kind.Gateway }}')
          echo "Host IP (from kind network gateway): $HOST_IP"

          kubectl apply -f - <<EOF
          apiVersion: v1
          kind: Service
          metadata:
            name: postgres-restore-operator
            namespace: pgro-system
          spec:
            ports:
              - port: 8080
                targetPort: 8080
          ---
          apiVersion: v1
          kind: Endpoints
          metadata:
            name: postgres-restore-operator
            namespace: pgro-system
          subsets:
            - addresses:
                - ip: "$HOST_IP"
              ports:
                - port: 8080
          ---
          apiVersion: v1
          kind: ConfigMap
          metadata:
            name: postgres-restore-operator-config
            namespace: pgro-system
          data:
            maxConcurrentRestores: "4"
            usePortForward: "true"
          EOF

      - name: Pre-pull images on kind node
        run: |
          load_image() {
            local image="$1"
            local archive="/tmp/$(echo "$image" | tr '/:' '_').tar"
            crane pull --platform linux/amd64 "$image" "$archive"
            kind load image-archive "$archive"
            rm -f "$archive"
          }

          load_image minio/minio:latest
          load_image minio/mc:latest
          load_image kopia/kopia:0.22.3
          load_image postgres:16
          load_image postgres:16-alpine
          load_image alpine:latest

      - name: Deploy MinIO
        run: |
          kubectl apply -f tests/fixtures/minio.yaml
          kubectl wait --namespace minio --for=condition=Available deployment/minio --timeout=120s
          echo "MinIO is ready"

      - name: Set up test kopia repository (postgres data)
        run: |
          kubectl apply -f tests/fixtures/setup-kopia-repo.yaml
          for i in $(seq 1 60); do
            STATUS=$(kubectl get job/setup-kopia-repo -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null)
            FAILED=$(kubectl get job/setup-kopia-repo -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null)
            if [ "$STATUS" = "True" ]; then
              echo "Setup job completed successfully"
              break
            fi
            if [ "$FAILED" = "True" ]; then
              echo "Setup job failed!"
              kubectl describe job/setup-kopia-repo
              kubectl get pods -l job-name=setup-kopia-repo -o wide
              kubectl logs job/setup-kopia-repo --all-containers --prefix
              exit 1
            fi
            if [ "$i" = "60" ]; then
              echo "Setup job timed out after 300s"
              kubectl describe job/setup-kopia-repo
              kubectl get pods -l job-name=setup-kopia-repo -o wide
              kubectl describe pods -l job-name=setup-kopia-repo
              kubectl logs job/setup-kopia-repo --all-containers --prefix 2>/dev/null || true
              exit 1
            fi
            sleep 5
          done
          echo "--- Setup job logs ---"
          kubectl logs job/setup-kopia-repo --all-containers --prefix
          echo "--- Kopia repository ready ---"

      - name: Set up non-postgres kopia snapshot
        if: matrix.needs_non_pg_snapshot
        run: |
          kubectl apply -f tests/fixtures/setup-non-postgres-snapshot.yaml
          for i in $(seq 1 60); do
            STATUS=$(kubectl get job/setup-non-postgres-snapshot -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null)
            FAILED=$(kubectl get job/setup-non-postgres-snapshot -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null)
            if [ "$STATUS" = "True" ]; then
              echo "Non-postgres snapshot job completed successfully"
              break
            fi
            if [ "$FAILED" = "True" ]; then
              echo "Non-postgres snapshot job failed!"
              kubectl describe job/setup-non-postgres-snapshot
              kubectl logs job/setup-non-postgres-snapshot --all-containers --prefix
              exit 1
            fi
            if [ "$i" = "60" ]; then
              echo "Non-postgres snapshot job timed out after 300s"
              kubectl describe job/setup-non-postgres-snapshot
              kubectl logs job/setup-non-postgres-snapshot --all-containers --prefix 2>/dev/null || true
              exit 1
            fi
            sleep 5
          done
          echo "--- Non-postgres snapshot job logs ---"
          kubectl logs job/setup-non-postgres-snapshot --all-containers --prefix

      - name: Start operator (out-of-cluster)
        run: |
          RUST_LOG=info,tower_http=debug \
          OPERATOR_NAMESPACE=pgro-system \
          OPERATOR_SERVICE_NAME=postgres-restore-operator \
          KUBECONFIG="${HOME}/.kube/config" \
            ./target/debug/operator > /tmp/operator.log 2>&1 &
          echo $! > /tmp/operator.pid

          # Wait for operator to be ready
          for i in $(seq 1 30); do
            if curl -sf http://[::1]:8080/readyz > /dev/null 2>&1; then
              echo "Operator is ready"
              break
            fi
            if [ "$i" = "30" ]; then
              echo "Operator failed to become ready"
              cat /tmp/operator.log
              exit 1
            fi
            sleep 1
          done

      - name: Run integration tests
        run: |
          cargo test --test "${{ matrix.name }}" -- --include-ignored --nocapture
        env:
          RUST_LOG: info

      - name: Operator logs
        if: always()
        run: |
          echo "--- Operator logs ---"
          cat /tmp/operator.log || true

      - name: Cluster state on failure
        if: failure()
        run: |
          echo "=== Namespaces ==="
          kubectl get ns

          echo "=== All resources in test namespaces ==="
          for ns in ${{ matrix.namespaces }}; do
            echo "--- Namespace: $ns ---"
            kubectl get all -n "$ns" 2>/dev/null || true
          done

          echo "=== PostgresPhysicalReplicas ==="
          kubectl get postgresphysicalreplicas -A -o yaml 2>/dev/null || true

          echo "=== PostgresPhysicalRestores ==="
          kubectl get postgresphysicalrestores -A -o yaml 2>/dev/null || true

          echo "=== Jobs ==="
          kubectl get jobs -A -o wide 2>/dev/null || true

          echo "=== Pods ==="
          kubectl get pods -A -o wide 2>/dev/null || true

          echo "=== PVCs ==="
          kubectl get pvc -A -o wide 2>/dev/null || true

          echo "=== Events in test namespaces ==="
          for ns in ${{ matrix.namespaces }}; do
            echo "--- Events in $ns ---"
            kubectl get events -n "$ns" --sort-by=.lastTimestamp 2>/dev/null || true
          done

          echo "=== Failed pod logs ==="
          for ns in ${{ matrix.namespaces }}; do
            for pod in $(kubectl get pods -n "$ns" --no-headers -o custom-columns=":metadata.name" 2>/dev/null); do
              echo "--- Pod: $ns/$pod ---"
              kubectl logs -n "$ns" "$pod" --all-containers 2>/dev/null || true
            done
          done

      - name: Stop operator
        if: always()
        run: |
          if [ -f /tmp/operator.pid ]; then
            kill "$(cat /tmp/operator.pid)" 2>/dev/null || true
          fi