testbench/.github/workflows/ci.yml at main · agentic-layer/testbench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
name: Check

on:
  push:
    branches:
      - main
      - 'renovate/**'
  pull_request:

jobs:
  python:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: 'actions/checkout@v6'

      - name: Install uv
        uses: 'astral-sh/setup-uv@v7'
        with:
          version: "0.9.26"
          enable-cache: true

      - name: Install Dependencies
        run: uv sync

      - name: Run Checks
        run: uv run poe check

      - name: Run Tests
        run: uv run poe test

  push:
    name: Build and Push Docker Image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      id-token: write
    outputs:
      image-tags: ${{ steps.build-push.outputs.image-tags }}

    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@v5'
        with:
          fetch-depth: 0

      - name: Build and push Docker image
        id: build-push
        uses: ./.github/actions/build-push-image
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}

  helm:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: 'actions/checkout@v6'

      - name: Install Helm
        uses: azure/setup-helm@v4
        with:
          version: 'v3.14.0'

      - name: Lint Helm Chart
        run: helm lint chart/

  test-e2e:
    name: E2E Tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}

    steps:
      - name: Checkout
        uses: 'actions/checkout@v6'

      - name: Build Docker image
        run: docker build -t ghcr.io/agentic-layer/testbench/testworkflows:latest .

      - name: Set up Kubernetes (kind)
        uses: 'helm/kind-action@v1'
        with:
          cluster_name: 'kind'

      - name: Load image into Kind
        run: kind load docker-image ghcr.io/agentic-layer/testbench/testworkflows:latest --name kind

      - name: Install Tilt
        run: |
          curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
          tilt version

      - name: Run Tilt CI
        run: |
          # The tilt setup is a bit flaky on a fresh startup, so we retry a few times
          for i in {1..5}; do
            tilt ci && break
            if [ $i -eq 5 ]; then
              echo "Tilt CI failed after 5 attempts, exiting."
              exit 1
            fi
            echo "Tilt CI failed, retrying... ($i/4)"

            # Clean up stuck Helm releases before retrying
            helm rollback testkube -n testkube 2>/dev/null || helm uninstall testkube -n testkube 2>/dev/null || true

            # Wait a bit for resources to stabilize
            sleep 10
          done

      - name: Setup Testkube CLI
        uses: kubeshop/setup-testkube@v1

      - name: Run Test Workflow
        run: |
          testkube run testworkflow example-workflow --watch