Validation Matrix #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validation Matrix | |
| on: | |
| # Tier 1: Every PR to main | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'crates/**' | |
| - '.github/workflows/validation.yml' | |
| # Tier 2: Nightly at 2 AM UTC | |
| schedule: | |
| - cron: '0 2 * * *' | |
| # Manual trigger with tier selection | |
| workflow_dispatch: | |
| inputs: | |
| tier: | |
| description: 'Validation tier to run (1, 2, or 3)' | |
| required: true | |
| default: '1' | |
| type: choice | |
| options: | |
| - '1' | |
| - '2' | |
| - '3' | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| # ─── Tier 1: PR Smoke Gate ─────────────────────────────── | |
| # Runs on every PR. Quick validation of core image cohort. | |
| tier1-smoke: | |
| name: Tier 1 Smoke (${{ matrix.image }}) | |
| if: >- | |
| github.event_name == 'pull_request' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.tier == '1') || | |
| github.event_name == 'schedule' | |
| runs-on: macos-14 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - alpine:3.20 | |
| - python:3.12-slim | |
| - nginx:1.27-alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: crates | |
| shared-key: validation-tier1 | |
| - name: Build validation harness | |
| working-directory: crates | |
| run: cargo build --release -p vz-cli | |
| - name: Run Tier 1 validation (dry-run) | |
| working-directory: crates | |
| run: | | |
| cargo run --release -p vz-cli -- validate run \ | |
| --tier 1 \ | |
| --dry-run \ | |
| --json \ | |
| --output ../validation-report-tier1-${{ strategy.job-index }}.json | |
| - name: Upload validation report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: validation-tier1-${{ strategy.job-index }} | |
| path: validation-report-tier1-${{ strategy.job-index }}.json | |
| retention-days: 30 | |
| # ─── Tier 1 Summary ────────────────────────────────────── | |
| tier1-summary: | |
| name: Tier 1 Summary | |
| needs: tier1-smoke | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: validation-tier1-* | |
| merge-multiple: true | |
| - name: Check results | |
| run: | | |
| echo "=== Tier 1 Validation Results ===" | |
| for f in validation-report-tier1-*.json; do | |
| if [ -f "$f" ]; then | |
| echo "--- $f ---" | |
| cat "$f" | python3 -c " | |
| import json, sys | |
| r = json.load(sys.stdin) | |
| total = len(r.get('results', [])) | |
| passed = sum(1 for x in r.get('results', []) if x['outcome'] == 'Pass') | |
| failed = total - passed | |
| print(f' Total: {total}, Passed: {passed}, Failed: {failed}') | |
| if failed > 0: | |
| sys.exit(1) | |
| " || exit 1 | |
| fi | |
| done | |
| echo "=== All Tier 1 checks passed ===" | |
| # ─── Tier 2: Nightly Conformance ───────────────────────── | |
| # Runs nightly or on manual dispatch with tier=2. | |
| # Full cohort matrix with all scenario kinds. | |
| tier2-nightly: | |
| name: Tier 2 Nightly | |
| if: >- | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.tier == '2') | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: crates | |
| shared-key: validation-tier2 | |
| - name: Build validation harness | |
| working-directory: crates | |
| run: cargo build --release -p vz-cli | |
| - name: Run Tier 2 validation (dry-run) | |
| working-directory: crates | |
| run: | | |
| cargo run --release -p vz-cli -- validate run \ | |
| --tier 2 \ | |
| --dry-run \ | |
| --json \ | |
| --output ../validation-report-tier2.json | |
| - name: Upload validation report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: validation-tier2-nightly | |
| path: validation-report-tier2.json | |
| retention-days: 90 | |
| - name: Upload manifest snapshot | |
| if: always() | |
| run: | | |
| cd crates && cargo run --release -p vz-cli -- \ | |
| validate manifest --output ../cohort-manifest.json | |
| - name: Upload manifest artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cohort-manifest-nightly | |
| path: cohort-manifest.json | |
| retention-days: 90 | |
| # ─── Tier 3: Weekly Stress ─────────────────────────────── | |
| # Runs on manual dispatch with tier=3 only. | |
| # Placeholder for stress/recovery tests. | |
| tier3-stress: | |
| name: Tier 3 Weekly Stress | |
| if: >- | |
| github.event_name == 'workflow_dispatch' && github.event.inputs.tier == '3' | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: crates | |
| shared-key: validation-tier3 | |
| - name: Build validation harness | |
| working-directory: crates | |
| run: cargo build --release -p vz-cli | |
| - name: Run Tier 3 validation (dry-run) | |
| working-directory: crates | |
| run: | | |
| cargo run --release -p vz-cli -- validate run \ | |
| --tier 3 \ | |
| --dry-run \ | |
| --json \ | |
| --output ../validation-report-tier3.json || true | |
| - name: Upload validation report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: validation-tier3-stress | |
| path: validation-report-tier3.json | |
| retention-days: 180 | |
| # ─── Unit Tests ────────────────────────────────────────── | |
| # Always run harness unit tests. | |
| harness-tests: | |
| name: Validation Harness Tests | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: crates | |
| shared-key: validation-harness | |
| - uses: taiki-e/install-action@nextest | |
| - name: Clippy | |
| working-directory: crates | |
| run: cargo clippy -p vz-validation -- -D warnings | |
| - name: Test | |
| working-directory: crates | |
| run: cargo nextest run -p vz-validation |