Wire comparator CI to call run-comparator.sh and attach files to email

.github/workflows/runJdbcComparator.yml

@@ -3,36 +3,46 @@ name: JDBC Driver Comparison
 on:
   schedule:
     - cron: '0 0 * * 1' # Run at 00:00 UTC on Monday (1)
-  workflow_dispatch:    # Allow manual trigger
+  workflow_dispatch:
+    # Manual trigger from Actions UI. Defaults below scope the run to a fast
+    # initial-test config; blank a field out at dispatch time for broader
+    # coverage. The weekly cron is unaffected (inputs are empty for schedule
+    # events, so the script's "empty = all" semantics give full coverage).
+    inputs:
+      suites:
+        description: 'SUITES_RUN_ONLY (comma-separated; leave blank for full coverage matching cron)'
+        required: false
+        default: ''
+      metadata_methods:
+        description: 'METADATA_RUN_ONLY_METHODS (comma-separated; leave blank for all methods)'
+        required: false
+        default: ''
+      connection_config:
+        description: 'CONNECTION_CONFIG (comma-separated; leave blank for all configs)'
+        required: false
+        default: ''
 
 permissions:
+  id-token: write   # for JFrog OIDC token exchange
   contents: read
 
 jobs:
   comparator:
-    runs-on: ubuntu-latest
+    # Public GitHub runners are blocked org-wide; use internal Databricks runners.
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
 
     steps:
-      - name: Checkout main branch
+      - name: Checkout comparator-v2 (or PR branch for dispatch)
+        # Schedule events from main (if this workflow ever moves to main): always
+        # checkout comparator-v2, where the comparator framework + script live.
+        # Dispatch events: checkout the triggering ref so PR branches can validate
+        # workflow/script changes pre-merge.
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
-          ref: main
-          fetch-depth: 0
-
-      - name: Configure Git
-        run: |
-          git config user.name "GitHub Actions"
-          git config user.email "actions@github.com"
-
-      - name: Merge main into comparator-v2
-        run: |
-          git checkout comparator-v2
-          git merge main --allow-unrelated-histories --no-edit -X theirs || {
-            echo "Force merging by accepting all changes from main"
-            git checkout --theirs .
-            git add .
-            git commit --no-edit
-          }
+          ref: ${{ github.event_name == 'schedule' && 'comparator-v2' || github.ref }}
+          fetch-depth: 1
 
       - name: Set up JDK 11
         uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
@@ -41,54 +51,153 @@ jobs:
           distribution: 'temurin'
           cache: maven
 
-      - name: Set up PAT
-        env:
-          DATABRICKS_COMPARATOR_TOKEN: ${{ secrets.DATABRICKS_COMPARATOR_TOKEN }}
+      - name: Get JFrog OIDC token
+        # Internal runners can't reach Maven Central directly; route mvn
+        # through Databricks Artifactory using a short-lived JFrog token
+        # obtained by exchanging the GitHub OIDC ID token.
         run: |
-          echo "DATABRICKS_COMPARATOR_TOKEN=${DATABRICKS_COMPARATOR_TOKEN}" >> $GITHUB_ENV
+          set -euo pipefail
 
-      - name: Run Tests
-        run: mvn -pl jdbc-core test -Dtest=JDBCDriverComparisonTest
+          # Get GitHub OIDC ID token
+          ID_TOKEN=$(curl -sLS \
+            -H "User-Agent: actions/oidc-client" \
+            -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"')
+          echo "::add-mask::${ID_TOKEN}"
 
-      - name: Format Email Content
+          # Exchange for JFrog access token
+          ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \
+            "https://databricks.jfrog.io/access/api/v1/oidc/token" \
+            -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"')
+          echo "::add-mask::${ACCESS_TOKEN}"
+
+          if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+            echo "FAIL: Could not extract JFrog access token"
+            exit 1
+          fi
+
+          echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV"
+
+          echo "JFrog OIDC token obtained successfully"
+
+      - name: Configure maven
         run: |
-          chmod +x bin/format-comparator-email.sh
-          ./bin/format-comparator-email.sh
+          set -euo pipefail
+
+          mkdir -p ~/.m2
+          cat > ~/.m2/settings.xml << EOF
+          <settings>
+            <mirrors>
+              <mirror>
+                <id>jfrog-central</id>
+                <mirrorOf>*</mirrorOf>
+                <url>https://databricks.jfrog.io/artifactory/db-maven/</url>
+              </mirror>
+            </mirrors>
+            <servers>
+              <server>
+                <id>jfrog-central</id>
+                <username>gha-service-account</username>
+                <password>${JFROG_ACCESS_TOKEN}</password>
+              </server>
+            </servers>
+          </settings>
+          EOF
+
+          echo "Maven configured to use JFrog registry"
+
+      - name: Run comparator
+        env:
+          DATABRICKS_COMPARATOR_TOKEN: ${{ secrets.DATABRICKS_COMPARATOR_TOKEN }}
+          COMPARATOR_HOST: ${{ vars.COMPARATOR_HOST }}
+          COMPARATOR_WAREHOUSE: ${{ vars.COMPARATOR_WAREHOUSE }}
+          PRO_WAREHOUSE_ID: ${{ vars.PRO_WAREHOUSE_ID }}
+          RUN_NAME: ci-weekly-${{ github.run_id }}
+          # Scoping knobs — workflow_dispatch.inputs populate these when
+          # manually triggered. For schedule events, inputs are empty,
+          # which the script treats as "run everything".
+          SUITES_RUN_ONLY: ${{ inputs.suites }}
+          METADATA_RUN_ONLY_METHODS: ${{ inputs.metadata_methods }}
+          CONNECTION_CONFIG: ${{ inputs.connection_config }}
+        run: bash src/test/java/com/databricks/jdbc/comparator/run-comparator.sh
 
-      - name: Check for Report and Differences
-        id: check_differences
+      - name: Locate generated files
+        id: outputs
+        if: always()
         run: |
-          REPORT=$(ls jdbc-comparison-report-*.txt 2>/dev/null | head -1)
-          if [ -n "$REPORT" ]; then
-            echo "report_file=$REPORT" >> $GITHUB_OUTPUT
-            if grep -q "No differences found" "$REPORT"; then
-              echo "has_differences=false" >> $GITHUB_OUTPUT
-            else
-              echo "has_differences=true" >> $GITHUB_OUTPUT
-            fi
-          else
+          shopt -s nullglob
+          report_files=(ci-weekly-*-report-*.txt)
+          csv_files=(ci-weekly-*-results-*.csv)
+          log_files=(ci-weekly-*-logs-*.txt)
+
+          REPORT="${report_files[0]:-}"
+          CSV="${csv_files[0]:-}"
+          LOG="${log_files[0]:-}"
+
+          if [ -z "$REPORT" ]; then
             echo "Report file not found"
             exit 1
           fi
 
-      - name: Send Email
-        if: steps.check_differences.outputs.has_differences == 'true'
+          # Build comma-separated attachments list of files that actually exist
+          ATTACHMENTS="$REPORT"
+          [ -n "$CSV" ] && ATTACHMENTS="$ATTACHMENTS,$CSV"
+          [ -n "$LOG" ] && ATTACHMENTS="$ATTACHMENTS,$LOG"
+
+          if grep -q "No differences found" "$REPORT"; then
+            HAS_DIFFS=false
+          else
+            HAS_DIFFS=true
+          fi
+
+          {
+            echo "report_file=$REPORT"
+            echo "csv_file=$CSV"
+            echo "log_file=$LOG"
+            echo "attachments=$ATTACHMENTS"
+            echo "has_differences=$HAS_DIFFS"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Send email
+        # Skip under `act` (local CI runner) — env.ACT is injected by act only.
+        if: ${{ env.ACT != 'true' && steps.outputs.outputs.has_differences == 'true' }}
         uses: dawidd6/action-send-mail@4226df7daafa6fc901a43789c49bf7ab309066e7 # v3
         with:
           server_address: smtp.gmail.com
           server_port: 465
           username: ${{ secrets.SMTP_USERNAME }}
           password: ${{ secrets.SMTP_PASSWORD }}
-          subject: JDBC Driver Comparison Results - 🚨Differences Found
-          html_body: file://jdbc-comparison-report.html
+          subject: "JDBC Driver Comparison Results — Differences Found (run ${{ github.run_id }})"
           to: ${{ secrets.EMAIL_RECIPIENTS }}
           from: JDBC Comparator Runner
-          content_type: text/html
+          body: |
+            Hi Team,
+
+            The weekly Thrift-vs-SEA JDBC driver comparison detected differences.
+
+            Run ID:    ${{ github.run_id }}
+            Workflow:  ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            Triggered: ${{ github.event_name }}
+
+            Attached:
+              - ${{ steps.outputs.outputs.report_file }} — diff report (TXT)
+              - ${{ steps.outputs.outputs.csv_file }}    — per-test results (CSV)
+              - ${{ steps.outputs.outputs.log_file }}    — full mvn log
+
+            To exclude known/accepted diffs from future reports, add a filter
+            to FILTER_JSON in
+            src/test/java/com/databricks/jdbc/comparator/run-comparator.sh
+            (under "metadataSkipFilters") and include the reason.
+
+            — JDBC Comparator Runner
+          attachments: ${{ steps.outputs.outputs.attachments }}
 
-      - name: Upload Report as Artifact
+      - name: Upload reports as artifact
+        if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: jdbc-comparison-report
           path: |
-            jdbc-comparison-report-*.txt
-            jdbc-comparison-report.html
+            ci-weekly-*-report-*.txt
+            ci-weekly-*-results-*.csv
+            ci-weekly-*-logs-*.txt