Skip to content

Commit 0442b15

Browse files
author
MiaAppel
authored
FEAT: PAAL-151 Add Testworkflows
This commit adds functionality as described in PAAL-151. 4 Scripts are implemented, each with its own responsibilities - setup.py - Loads a dataset from a specified URL and converts it to native Ragas Dataset format - run.py - Executes test queries from the Dataset through a specified agent URL and collects its responses in native Ragas Experiment format - evaluate.py - Calculates specified Ragas Metrics on the agent responses found in the Ragas Experiment and formats results to JSON - publish.py - Publishes evaluation metrics to an OpenTelemetry OTLP endpoint
1 parent 9d54c0f commit 0442b15

23 files changed

Lines changed: 6072 additions & 40 deletions

‎.github/workflows/check.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#
2+
# GitHub Action: Check
3+
#
4+
# This workflow is triggered on pushes to main and renovate branches, as well as
5+
# pull requests. It runs backend checks to ensure code quality.
6+
#
7+
name: Check
8+
9+
on:
10+
push:
11+
branches:
12+
- main
13+
- 'renovate/**'
14+
pull_request:
15+
16+
jobs:
17+
test-testworkflows:
18+
name: Runs on Ubuntu
19+
runs-on: ubuntu-latest
20+
21+
defaults:
22+
run:
23+
working-directory: testworkflows
24+
25+
steps:
26+
- name: Clone the code
27+
uses: 'actions/checkout@v5'
28+
29+
- name: Install uv
30+
uses: 'astral-sh/setup-uv@v6'
31+
with:
32+
version: "0.8.17"
33+
enable-cache: true
34+
35+
- name: Setup Python
36+
uses: 'actions/setup-python@v6'
37+
with:
38+
python-version-file: "testworkflows/.python-version"
39+
40+
- name: Install Dependencies
41+
run: uv sync
42+
43+
- name: Run Checks
44+
run: uv run poe check

‎.github/workflows/test-e2e.yml‎

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
#
2+
# GitHub Action: E2E Tests
3+
#
4+
# This workflow is triggered on pushes to main and renovate branches, as well as
5+
# pull requests. It runs E2E Tests to ensure the code is working.
6+
#
17
name: E2E Tests
28

39
on:
@@ -7,32 +13,67 @@ on:
713
- 'renovate/**'
814
pull_request:
915

16+
env:
17+
# Required environment variables for the E2E test
18+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
19+
OPENAI_API_KEY: PLACEHOLDER
20+
1021
jobs:
11-
test-e2e:
22+
test:
1223
name: Run on Ubuntu
1324
runs-on: ubuntu-latest
14-
permissions:
15-
contents: 'read'
25+
26+
defaults:
27+
run:
28+
working-directory: testworkflows
1629

1730
steps:
1831
- name: Clone the code
19-
uses: actions/checkout@v5
32+
uses: 'actions/checkout@v5'
33+
34+
- name: Install uv
35+
uses: 'astral-sh/setup-uv@v6'
36+
with:
37+
version: "0.8.17"
38+
enable-cache: true
2039

21-
- name: Setup Go
22-
uses: actions/setup-go@v6
40+
- name: Setup Python
41+
uses: 'actions/setup-python@v6'
2342
with:
24-
go-version-file: go.mod
43+
python-version-file: "testworkflows/.python-version"
44+
45+
- name: Install Dependencies
46+
run: uv sync
2547

26-
- name: Install the latest version of kind
48+
- name: Set up Kubernetes (kind)
49+
uses: 'helm/kind-action@v1'
50+
with:
51+
cluster_name: testbench-cluster
52+
53+
- name: Install Tilt
2754
run: |
28-
curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
29-
chmod +x ./kind
30-
sudo mv ./kind /usr/local/bin/kind
55+
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
56+
tilt version
3157
32-
- name: Verify kind installation
33-
run: kind version
58+
- name: Start test data server
59+
run: uv run python -m http.server 8000 --directory tests/test_data &
3460

35-
- name: Running Test e2e
61+
- name: Start Tiltfile services
62+
run: tilt ci
63+
working-directory: .
64+
65+
- name: Wait for services to be ready
3666
run: |
37-
go mod tidy
38-
make test-e2e
67+
echo "Waiting for services to be ready..."
68+
kubectl wait --for=condition=ready pod -l app=weather-agent --timeout=300s || true
69+
kubectl wait --for=condition=ready pod -l app=ai-gateway-litellm --timeout=300s || true
70+
kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=lgtm --timeout=300s || true
71+
72+
echo "Checking service status..."
73+
kubectl get pods -A
74+
75+
echo "Waiting for test-data-server (HTTP server on port 8000)..."
76+
curl --retry 30 --retry-delay 2 --retry-connrefused -f http://localhost:8000/dataset.json > /dev/null
77+
78+
- name: Running Test e2e
79+
run: uv run poe test_e2e

‎.github/workflows/test.yml‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
#
2+
# GitHub Action: Test E2E
3+
#
4+
# This workflow is triggered on pushes to main and renovate branches, as well as
5+
# pull requests. It runs Unit Tests to ensure the code is working.
6+
#
17
name: Tests
28

39
on:
@@ -14,16 +20,27 @@ jobs:
1420
permissions:
1521
contents: 'read'
1622

23+
defaults:
24+
run:
25+
working-directory: testworkflows
26+
1727
steps:
1828
- name: Clone the code
19-
uses: actions/checkout@v5
29+
uses: 'actions/checkout@v5'
30+
31+
- name: Install uv
32+
uses: 'astral-sh/setup-uv@v6'
33+
with:
34+
version: "0.8.17"
35+
enable-cache: true
2036

21-
- name: Setup Go
22-
uses: actions/setup-go@v6
37+
- name: Setup Python
38+
uses: 'actions/setup-python@v6'
2339
with:
24-
go-version-file: go.mod
40+
python-version-file: "testworkflows/.python-version"
41+
42+
- name: Install Dependencies
43+
run: uv sync
2544

26-
- name: Running Tests
27-
run: |
28-
go mod tidy
29-
make test
45+
- name: Run Tests
46+
run: uv run poe test

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,5 @@ go.work
2525
*.swp
2626
*.swo
2727
*~
28+
29+
.env

‎Tiltfile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# -*- mode: Python -*-
2+
3+
# Increase Kubernetes upsert timeout for CRD installations
4+
update_settings(max_parallel_updates=10)
5+
6+
# Load .env file for environment variables
7+
load('ext://dotenv', 'dotenv')
8+
dotenv()
9+
10+
v1alpha1.extension_repo(name='agentic-layer', url='https://github.com/agentic-layer/tilt-extensions', ref='v0.3.1')
11+
12+
v1alpha1.extension(name='cert-manager', repo_name='agentic-layer', repo_path='cert-manager')
13+
load('ext://cert-manager', 'cert_manager_install')
14+
cert_manager_install()
15+
16+
v1alpha1.extension(name='agent-runtime', repo_name='agentic-layer', repo_path='agent-runtime')
17+
load('ext://agent-runtime', 'agent_runtime_install')
18+
agent_runtime_install(version='0.9.0')
19+
20+
v1alpha1.extension(name='ai-gateway-litellm', repo_name='agentic-layer', repo_path='ai-gateway-litellm')
21+
load('ext://ai-gateway-litellm', 'ai_gateway_litellm_install')
22+
ai_gateway_litellm_install(version='0.2.0')
23+
24+
# Webserver to serve test data (disabled in CI - manually started instead)
25+
if not os.getenv('CI'):
26+
local_resource(
27+
'test-data-server',
28+
serve_cmd='python3 -m http.server 8000 --directory testworkflows/tests/test_data',
29+
labels=['test-data'],
30+
links=['http://localhost:8000/dataset.json']
31+
)
32+
33+
# Apply Kubernetes manifests
34+
k8s_yaml(kustomize('deploy/local'))
35+
36+
k8s_resource('ai-gateway-litellm', port_forwards=['11001:4000'])
37+
k8s_resource('weather-agent', port_forwards='11010:8000', labels=['agents'], resource_deps=['agent-runtime'])
38+
k8s_resource('lgtm', port_forwards=['11000:3000', '9090:9090', '4318:4318'])

‎deploy/local/kustomization.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
resources:
4+
- weather-agent.yaml
5+
- lgtm.yaml

‎deploy/local/lgtm.yaml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# this is intended for demo / testing purposes only, not for production usage
2+
apiVersion: v1
3+
kind: Namespace
4+
metadata:
5+
name: monitoring
6+
---
7+
apiVersion: v1
8+
kind: Service
9+
metadata:
10+
name: lgtm
11+
namespace: monitoring
12+
spec:
13+
selector:
14+
app: lgtm
15+
ports:
16+
- name: grafana
17+
protocol: TCP
18+
port: 3000
19+
targetPort: 3000
20+
- name: otel-grpc
21+
protocol: TCP
22+
port: 4317
23+
targetPort: 4317
24+
- name: otel-http
25+
protocol: TCP
26+
port: 4318
27+
targetPort: 4318
28+
- name: prometheus
29+
protocol: TCP
30+
port: 9090
31+
targetPort: 9090
32+
---
33+
apiVersion: apps/v1
34+
kind: Deployment
35+
metadata:
36+
name: lgtm
37+
namespace: monitoring
38+
spec:
39+
replicas: 1
40+
selector:
41+
matchLabels:
42+
app: lgtm
43+
template:
44+
metadata:
45+
labels:
46+
app: lgtm
47+
spec:
48+
containers:
49+
- name: lgtm
50+
image: grafana/otel-lgtm:latest
51+
ports:
52+
- containerPort: 3000
53+
- containerPort: 4317
54+
- containerPort: 4318
55+
- containerPort: 9090
56+
readinessProbe:
57+
exec:
58+
command:
59+
- cat
60+
- /tmp/ready
61+
volumeMounts:
62+
- name: tempo-data
63+
mountPath: /data/tempo
64+
- name: grafana-data
65+
mountPath: /data/grafana
66+
- name: loki-data
67+
mountPath: /data/loki
68+
- name: loki-storage
69+
mountPath: /loki
70+
- name: p8s-storage
71+
mountPath: /data/prometheus
72+
- name: pyroscope-storage
73+
mountPath: /data/pyroscope
74+
volumes:
75+
- name: tempo-data
76+
emptyDir: {}
77+
- name: loki-data
78+
emptyDir: {}
79+
- name: grafana-data
80+
emptyDir: {}
81+
- name: loki-storage
82+
emptyDir: {}
83+
- name: p8s-storage
84+
emptyDir: {}
85+
- name: pyroscope-storage
86+
emptyDir: {}

‎deploy/local/weather-agent.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: sample-agents
5+
---
6+
apiVersion: runtime.agentic-layer.ai/v1alpha1
7+
kind: Agent
8+
metadata:
9+
labels:
10+
app.kubernetes.io/name: agent-runtime-operator
11+
app.kubernetes.io/managed-by: kustomize
12+
name: weather-agent
13+
namespace: sample-agents
14+
spec:
15+
framework: google-adk
16+
image: ghcr.io/agentic-layer/weather-agent:0.6.0
17+
protocols:
18+
- type: A2A
19+
exposed: true
20+
env:
21+
- name: OTEL_SDK_DISABLED
22+
value: "true"

‎testworkflows/.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.13

0 commit comments

Comments
 (0)