Skip to content
CI Pipeline

feat: add A/B testing framework with nprobe optimization results #24

Sign in to view logs

feat: add A/B testing framework with nprobe optimization results

feat: add A/B testing framework with nprobe optimization results #24

Workflow file for this run

.github/workflows/ci.yml at 7a4a5ab
name: CI Pipeline
permissions:
contents: read
packages: write
security-events: write
checks: write
pull-requests: write
on:
push:
branches: [main, develop]
pull_request:
branches: [main, develop]
workflow_dispatch:
env:
DOTNET_VERSION: '8.0.x'
PYTHON_VERSION: '3.12'
jobs:
# ══════════════════════════════════════════════════════════════════════════
# Job 1: .NET Build and Test
# ══════════════════════════════════════════════════════════════════════════
dotnet-build:
name: .NET Build & Test
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: ${{ env.DOTNET_VERSION }}
- name: Restore dependencies
run: dotnet restore
- name: Build solution
run: dotnet build --configuration Release --no-restore
- name: Run unit tests
run: dotnet test tests/VectorCatalog.Api.Tests/VectorCatalog.Api.Tests.csproj --configuration Release --no-build --verbosity normal --logger "trx;LogFileName=test-results.trx"
- name: Publish test results
uses: dorny/test-reporter@v1
if: always()
with:
name: .NET Test Results
path: '**/test-results.trx'
reporter: dotnet-trx
fail-on-error: true
# ══════════════════════════════════════════════════════════════════════════
# Job 2: Python Lint and Test
# ══════════════════════════════════════════════════════════════════════════
python-test:
name: Python Lint & Test
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install sidecar dependencies
working-directory: ./sidecar
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Generate gRPC stubs
working-directory: ./sidecar
run: |
python -m grpc_tools.protoc -I./protos --python_out=. --grpc_python_out=. protos/vector_service.proto
- name: Run pytest
working-directory: ./sidecar
run: |
python -m pytest tests/ -v --tb=short --color=yes
# ══════════════════════════════════════════════════════════════════════════
# Job 3: Docker Build (API)
# ══════════════════════════════════════════════════════════════════════════
docker-build-api:
name: Docker Build - API
runs-on: ubuntu-latest
needs: [dotnet-build]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
docker system prune -af --volumes
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build API Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./src/VectorCatalog.Api/Dockerfile
push: false
tags: vector-catalog-api:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
# ══════════════════════════════════════════════════════════════════════════
# Job 4: Docker Build (Sidecar)
# ══════════════════════════════════════════════════════════════════════════
docker-build-sidecar:
name: Docker Build - Sidecar
runs-on: ubuntu-latest
needs: [python-test]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
docker system prune -af --volumes
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Sidecar Docker image
uses: docker/build-push-action@v5
with:
context: ./sidecar
file: ./sidecar/Dockerfile
push: false
tags: vector-catalog-sidecar:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
# ══════════════════════════════════════════════════════════════════════════
# Job 5: Push to GHCR (only on main branch)
# ══════════════════════════════════════════════════════════════════════════
docker-push:
name: Push to GHCR
runs-on: ubuntu-latest
needs: [docker-build-api, docker-build-sidecar]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
permissions:
contents: read
packages: write
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract version from Chart.yaml
id: version
run: |
VERSION=$(grep '^appVersion:' helm/vector-catalog/Chart.yaml | awk '{print $2}' | tr -d '"')
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION"
- name: Build and push API image
uses: docker/build-push-action@v5
with:
context: .
file: ./src/VectorCatalog.Api/Dockerfile
push: true
tags: |
ghcr.io/${{ github.repository_owner }}/vector-catalog-api:latest
ghcr.io/${{ github.repository_owner }}/vector-catalog-api:${{ steps.version.outputs.version }}
ghcr.io/${{ github.repository_owner }}/vector-catalog-api:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Build and push Sidecar image
uses: docker/build-push-action@v5
with:
context: ./sidecar
file: ./sidecar/Dockerfile
push: true
tags: |
ghcr.io/${{ github.repository_owner }}/vector-catalog-sidecar:latest
ghcr.io/${{ github.repository_owner }}/vector-catalog-sidecar:${{ steps.version.outputs.version }}
ghcr.io/${{ github.repository_owner }}/vector-catalog-sidecar:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
# ══════════════════════════════════════════════════════════════════════════
# Job 6: Integration Check (Docker Compose smoke test)
# ══════════════════════════════════════════════════════════════════════════
integration-check:
name: Docker Compose Smoke Test
runs-on: ubuntu-latest
needs: [docker-build-api, docker-build-sidecar]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Create indexes directory
run: mkdir -p data/indexes
- name: Start services
run: |
docker compose up -d redis minio jaeger prometheus
sleep 10
- name: Check service health
run: |
docker compose ps
docker compose logs redis
docker compose logs minio
- name: Stop services
if: always()
run: docker compose down -v
# ══════════════════════════════════════════════════════════════════════════
# Job 7: Security Scan (Trivy)
# ══════════════════════════════════════════════════════════════════════════
security-scan:
name: Security Scan
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
scan-type: 'fs'
scan-ref: '.'
format: 'sarif'
output: 'trivy-results.sarif'
severity: 'CRITICAL,HIGH'
- name: Upload Trivy results to GitHub Security
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: 'trivy-results.sarif'
# ══════════════════════════════════════════════════════════════════════════
# Job 8: Build Success Summary
# ══════════════════════════════════════════════════════════════════════════
build-summary:
name: Build Summary
runs-on: ubuntu-latest
needs: [dotnet-build, python-test, docker-build-api, docker-build-sidecar, integration-check]
if: always()
steps:
- name: Check build status
run: |
echo "✓ .NET Build: ${{ needs.dotnet-build.result }}"
echo "✓ Python Test: ${{ needs.python-test.result }}"
echo "✓ Docker Build API: ${{ needs.docker-build-api.result }}"
echo "✓ Docker Build Sidecar: ${{ needs.docker-build-sidecar.result }}"
echo "✓ Integration Check: ${{ needs.integration-check.result }}"
- name: Fail if any job failed
if: contains(needs.*.result, 'failure')
run: exit 1