[All] Reorganize CI/CD and Release Workflows

.github/workflows/ci-go.yml

@@ -58,10 +58,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [linux-ubuntu-latest, windows-server-latest]
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+        include:
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+          - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: go
@@ -104,12 +108,15 @@ jobs:
       matrix:
         include:
           - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
             lib: lib/linux_amd64/libzerobus_ffi.a
           - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
             lib: lib/windows_amd64/libzerobus_ffi.a
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            lib: lib/darwin_arm64/libzerobus_ffi.a
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: go

.github/workflows/ci-java.yml

@@ -40,18 +40,30 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [linux-ubuntu-latest, windows-server-latest]
-        javaVersion: ['11', '17', '21']
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+        include:
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            javaVersion: '11'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            javaVersion: '17'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            javaVersion: '21'
+          - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
+            javaVersion: '11'
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            javaVersion: '11'
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: java
     steps:
       - uses: actions/checkout@v4
       - name: Unshallow
-        run: git fetch --prune --unshallow
+        run: git fetch --prune --unshallow || true
       - name: Set up JDK ${{ matrix.javaVersion }}
         uses: actions/setup-java@v4
         with:
@@ -63,6 +75,9 @@ jobs:
         shell: bash
         run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "sdk" }\n' >> ../rust/Cargo.toml
 
+      - name: Install Maven (macOS)
+        if: runner.os == 'macOS'
+        run: brew install maven 2>/dev/null || true
       - name: Build with Maven
         run: mvn clean compile
       - name: Run tests

.github/workflows/ci-python.yml

@@ -13,19 +13,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [linux-ubuntu-latest, windows-server-latest]
-        pyVersion: ['3.9', '3.10', '3.11', '3.12']
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+        include:
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            pyVersion: '3.9'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            pyVersion: '3.10'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            pyVersion: '3.11'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            pyVersion: '3.12'
+          - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
+            pyVersion: '3.9'
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            pyVersion: '3.9'
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: python
     steps:
       - uses: actions/checkout@v4
 
       - name: Unshallow
-        run: git fetch --prune --unshallow
+        run: git fetch --prune --unshallow || true
 
       - uses: actions/setup-python@v5
         with:

.github/workflows/ci-rust.yml

@@ -41,10 +41,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [linux-ubuntu-latest, windows-server-latest]
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+        include:
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+          - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: rust

.github/workflows/ci-typescript.yml

@@ -17,18 +17,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        settings:
-          - host: linux-ubuntu-latest
+        include:
+          - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
             target: x86_64-unknown-linux-gnu
-          - host: linux-ubuntu-latest
+          - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
             target: aarch64-unknown-linux-gnu
-          - host: windows-server-latest
+          - runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
             target: x86_64-pc-windows-msvc
+          - runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            target: aarch64-apple-darwin
+          - runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            target: x86_64-apple-darwin
 
-    name: Build - ${{ matrix.settings.target }}
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.settings.host }}
+    name: Build - ${{ matrix.target }}
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: typescript
@@ -58,7 +60,7 @@ jobs:
         shell: powershell
 
       - name: Add Rust target
-        run: rustup target add ${{ matrix.settings.target }}
+        run: rustup target add ${{ matrix.target }}
 
       - name: Cache cargo
         uses: actions/cache@v4
@@ -68,10 +70,10 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             typescript/target/
-          key: ${{ matrix.settings.target }}-cargo-${{ hashFiles('typescript/Cargo.lock') }}
+          key: ${{ matrix.target }}-cargo-${{ hashFiles('typescript/Cargo.lock') }}
 
       - name: Install cross-compilation tools (Linux ARM64)
-        if: matrix.settings.target == 'aarch64-unknown-linux-gnu'
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
         run: |
           sudo apt-get update
           sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
@@ -89,27 +91,39 @@ jobs:
         run: npm ci
 
       - name: Build
-        run: npm run build -- --target ${{ matrix.settings.target }}
+        run: npm run build -- --target ${{ matrix.target }}
         env:
           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: bindings-${{ matrix.settings.target }}${{ inputs.artifact_suffix }}
+          name: bindings-${{ matrix.target }}${{ inputs.artifact_suffix }}
           path: 'typescript/*.node'
           if-no-files-found: error
 
   test:
     name: Test Node.js ${{ matrix.node }} - ${{ matrix.os }}
-    runs-on:
-      group: databricks-protected-runner-group
-      labels: ${{ matrix.os }}
+    runs-on: ${{ matrix.runner }}
     strategy:
       fail-fast: false
       matrix:
-        os: [linux-ubuntu-latest, windows-server-latest]
-        node: ['16', '18', '20']
+        include:
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            node: '16'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            node: '18'
+          - os: linux-ubuntu-latest
+            runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest }
+            node: '20'
+          - os: windows-server-latest
+            runner: { group: databricks-protected-runner-group, labels: windows-server-latest }
+            node: '16'
+          - os: macos-arm64
+            runner: [self-hosted, macOS, ARM64, zerobus-sdk]
+            node: '16'
     defaults:
       run:
         working-directory: typescript

.github/workflows/cleanup-macos-runner.yml

@@ -0,0 +1,90 @@
+name: Cleanup macOS Runner
+run-name: "Cleanup macOS self-hosted runner"
+
+on:
+  schedule:
+    - cron: '0 3 * * 0'  # Sunday at 3 AM UTC
+  workflow_dispatch:
+
+jobs:
+  cleanup:
+    name: Cleanup
+    runs-on: [self-hosted, macOS, ARM64, zerobus-sdk]
+
+    steps:
+      - name: Disk usage before cleanup
+        run: |
+          echo "=== Disk Usage Before ==="
+          df -h /
+          echo ""
+          echo "=== Large directories ==="
+          du -sh ~/.cargo 2>/dev/null || echo "~/.cargo: not found"
+          du -sh ~/.rustup 2>/dev/null || echo "~/.rustup: not found"
+          du -sh ~/.npm 2>/dev/null || echo "~/.npm: not found"
+          du -sh ~/.m2 2>/dev/null || echo "~/.m2: not found"
+          du -sh ~/Library/Caches/Homebrew 2>/dev/null || echo "Homebrew cache: not found"
+          du -sh ${{ runner.tool_cache }} 2>/dev/null || echo "Tool cache: not found"
+
+      - name: Clean Cargo registry and cache
+        run: |
+          echo "=== Cleaning Cargo ==="
+          du -sh ~/.cargo/registry 2>/dev/null || true
+          rm -rf ~/.cargo/registry/cache
+          rm -rf ~/.cargo/registry/src
+          rm -rf ~/.cargo/git/checkouts
+          echo "Cargo registry and git checkouts cleaned"
+
+      - name: Clean npm cache
+        run: |
+          echo "=== Cleaning npm ==="
+          du -sh ~/.npm 2>/dev/null || true
+          npm cache clean --force 2>/dev/null || true
+          echo "npm cache cleaned"
+
+      - name: Clean Homebrew cache
+        run: |
+          echo "=== Cleaning Homebrew ==="
+          du -sh ~/Library/Caches/Homebrew 2>/dev/null || true
+          brew cleanup --prune=7 2>/dev/null || true
+          echo "Homebrew cache cleaned"
+
+      - name: Clean Maven cache
+        run: |
+          echo "=== Cleaning Maven ==="
+          du -sh ~/.m2/repository 2>/dev/null || true
+          rm -rf ~/.m2/repository
+          echo "Maven repository cleaned"
+
+      - name: Clean old tool cache versions
+        run: |
+          echo "=== Tool cache ==="
+          du -sh ${{ runner.tool_cache }}/* 2>/dev/null || true
+          echo "Keeping tool cache intact (managed by setup-* actions)"
+
+      - name: Clean workspace temp files
+        run: |
+          echo "=== Cleaning workspace temp ==="
+          WORK_DIR="${{ runner.workspace }}"
+          if [ -d "$WORK_DIR" ]; then
+            find "$WORK_DIR" -name "target" -type d -maxdepth 4 -exec du -sh {} \; 2>/dev/null || true
+            find "$WORK_DIR" -name "target" -type d -maxdepth 4 -exec rm -rf {} \; 2>/dev/null || true
+            echo "Workspace target directories cleaned"
+          fi
+
+      - name: Kill orphan processes
+        run: |
+          echo "=== Checking for orphan processes ==="
+          ps aux | grep -E "(cargo|rustc|node|java|python|mvn)" | grep -v grep || echo "No orphan processes found"
+
+      - name: Disk usage after cleanup
+        run: |
+          echo "=== Disk Usage After ==="
+          df -h /
+          echo ""
+          echo "=== Large directories ==="
+          du -sh ~/.cargo 2>/dev/null || echo "~/.cargo: not found"
+          du -sh ~/.rustup 2>/dev/null || echo "~/.rustup: not found"
+          du -sh ~/.npm 2>/dev/null || echo "~/.npm: not found"
+          du -sh ~/.m2 2>/dev/null || echo "~/.m2: not found"
+          du -sh ~/Library/Caches/Homebrew 2>/dev/null || echo "Homebrew cache: not found"
+          du -sh ${{ runner.tool_cache }} 2>/dev/null || echo "Tool cache: not found"