diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml index b177bc6..d22cdbc 100644 --- a/.github/workflows/ci-go.yml +++ b/.github/workflows/ci-go.yml @@ -58,10 +58,14 @@ jobs: strategy: fail-fast: false matrix: - os: [linux-ubuntu-latest, windows-server-latest] - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + runs-on: ${{ matrix.runner }} defaults: run: working-directory: go @@ -104,12 +108,15 @@ jobs: matrix: include: - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } lib: lib/linux_amd64/libzerobus_ffi.a - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } lib: lib/windows_amd64/libzerobus_ffi.a - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + lib: lib/darwin_arm64/libzerobus_ffi.a + runs-on: ${{ matrix.runner }} defaults: run: working-directory: go diff --git a/.github/workflows/ci-java.yml b/.github/workflows/ci-java.yml index 80afab3..d0c93a9 100644 --- a/.github/workflows/ci-java.yml +++ b/.github/workflows/ci-java.yml @@ -40,18 +40,30 @@ jobs: strategy: fail-fast: false matrix: - os: [linux-ubuntu-latest, windows-server-latest] - javaVersion: ['11', '17', '21'] - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + javaVersion: '11' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + javaVersion: '17' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + javaVersion: '21' + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + javaVersion: '11' + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + javaVersion: '11' + runs-on: ${{ matrix.runner }} defaults: run: working-directory: java steps: - uses: actions/checkout@v4 - name: Unshallow - run: git fetch --prune --unshallow + run: git fetch --prune --unshallow || true - name: Set up JDK ${{ matrix.javaVersion }} uses: actions/setup-java@v4 with: @@ -63,6 +75,9 @@ jobs: shell: bash run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "sdk" }\n' >> ../rust/Cargo.toml + - name: Install Maven (macOS) + if: runner.os == 'macOS' + run: brew install maven 2>/dev/null || true - name: Build with Maven run: mvn clean compile - name: Run tests diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 5bb1a39..513f58f 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -13,11 +13,26 @@ jobs: strategy: fail-fast: false matrix: - os: [linux-ubuntu-latest, windows-server-latest] - pyVersion: ['3.9', '3.10', '3.11', '3.12'] - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + pyVersion: '3.9' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + pyVersion: '3.10' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + pyVersion: '3.11' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + pyVersion: '3.12' + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + pyVersion: '3.9' + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + pyVersion: '3.9' + runs-on: ${{ matrix.runner }} defaults: run: working-directory: python @@ -25,7 +40,7 @@ jobs: - uses: actions/checkout@v4 - name: Unshallow - run: git fetch --prune --unshallow + run: git fetch --prune --unshallow || true - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 315b3be..35e6846 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -41,10 +41,14 @@ jobs: strategy: fail-fast: false matrix: - os: [linux-ubuntu-latest, windows-server-latest] - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + runs-on: ${{ matrix.runner }} defaults: run: working-directory: rust diff --git a/.github/workflows/ci-typescript.yml b/.github/workflows/ci-typescript.yml index c41e004..95c2764 100644 --- a/.github/workflows/ci-typescript.yml +++ b/.github/workflows/ci-typescript.yml @@ -17,18 +17,20 @@ jobs: strategy: fail-fast: false matrix: - settings: - - host: linux-ubuntu-latest + include: + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: x86_64-unknown-linux-gnu - - host: linux-ubuntu-latest + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: aarch64-unknown-linux-gnu - - host: windows-server-latest + - runner: { group: databricks-protected-runner-group, labels: windows-server-latest } target: x86_64-pc-windows-msvc + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: aarch64-apple-darwin + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: x86_64-apple-darwin - name: Build - ${{ matrix.settings.target }} - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.settings.host }} + name: Build - ${{ matrix.target }} + runs-on: ${{ matrix.runner }} defaults: run: working-directory: typescript @@ -58,7 +60,7 @@ jobs: shell: powershell - name: Add Rust target - run: rustup target add ${{ matrix.settings.target }} + run: rustup target add ${{ matrix.target }} - name: Cache cargo uses: actions/cache@v4 @@ -68,10 +70,10 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ typescript/target/ - key: ${{ matrix.settings.target }}-cargo-${{ hashFiles('typescript/Cargo.lock') }} + key: ${{ matrix.target }}-cargo-${{ hashFiles('typescript/Cargo.lock') }} - name: Install cross-compilation tools (Linux ARM64) - if: matrix.settings.target == 'aarch64-unknown-linux-gnu' + if: matrix.target == 'aarch64-unknown-linux-gnu' run: | sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu @@ -89,27 +91,39 @@ jobs: run: npm ci - name: Build - run: npm run build -- --target ${{ matrix.settings.target }} + run: npm run build -- --target ${{ matrix.target }} env: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: bindings-${{ matrix.settings.target }}${{ inputs.artifact_suffix }} + name: bindings-${{ matrix.target }}${{ inputs.artifact_suffix }} path: 'typescript/*.node' if-no-files-found: error test: name: Test Node.js ${{ matrix.node }} - ${{ matrix.os }} - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - os: [linux-ubuntu-latest, windows-server-latest] - node: ['16', '18', '20'] + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + node: '16' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + node: '18' + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + node: '20' + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + node: '16' + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + node: '16' defaults: run: working-directory: typescript diff --git a/.github/workflows/cleanup-macos-runner.yml b/.github/workflows/cleanup-macos-runner.yml new file mode 100644 index 0000000..9ac005a --- /dev/null +++ b/.github/workflows/cleanup-macos-runner.yml @@ -0,0 +1,90 @@ +name: Cleanup macOS Runner +run-name: "Cleanup macOS self-hosted runner" + +on: + schedule: + - cron: '0 3 * * 0' # Sunday at 3 AM UTC + workflow_dispatch: + +jobs: + cleanup: + name: Cleanup + runs-on: [self-hosted, macOS, ARM64, zerobus-sdk] + + steps: + - name: Disk usage before cleanup + run: | + echo "=== Disk Usage Before ===" + df -h / + echo "" + echo "=== Large directories ===" + du -sh ~/.cargo 2>/dev/null || echo "~/.cargo: not found" + du -sh ~/.rustup 2>/dev/null || echo "~/.rustup: not found" + du -sh ~/.npm 2>/dev/null || echo "~/.npm: not found" + du -sh ~/.m2 2>/dev/null || echo "~/.m2: not found" + du -sh ~/Library/Caches/Homebrew 2>/dev/null || echo "Homebrew cache: not found" + du -sh ${{ runner.tool_cache }} 2>/dev/null || echo "Tool cache: not found" + + - name: Clean Cargo registry and cache + run: | + echo "=== Cleaning Cargo ===" + du -sh ~/.cargo/registry 2>/dev/null || true + rm -rf ~/.cargo/registry/cache + rm -rf ~/.cargo/registry/src + rm -rf ~/.cargo/git/checkouts + echo "Cargo registry and git checkouts cleaned" + + - name: Clean npm cache + run: | + echo "=== Cleaning npm ===" + du -sh ~/.npm 2>/dev/null || true + npm cache clean --force 2>/dev/null || true + echo "npm cache cleaned" + + - name: Clean Homebrew cache + run: | + echo "=== Cleaning Homebrew ===" + du -sh ~/Library/Caches/Homebrew 2>/dev/null || true + brew cleanup --prune=7 2>/dev/null || true + echo "Homebrew cache cleaned" + + - name: Clean Maven cache + run: | + echo "=== Cleaning Maven ===" + du -sh ~/.m2/repository 2>/dev/null || true + rm -rf ~/.m2/repository + echo "Maven repository cleaned" + + - name: Clean old tool cache versions + run: | + echo "=== Tool cache ===" + du -sh ${{ runner.tool_cache }}/* 2>/dev/null || true + echo "Keeping tool cache intact (managed by setup-* actions)" + + - name: Clean workspace temp files + run: | + echo "=== Cleaning workspace temp ===" + WORK_DIR="${{ runner.workspace }}" + if [ -d "$WORK_DIR" ]; then + find "$WORK_DIR" -name "target" -type d -maxdepth 4 -exec du -sh {} \; 2>/dev/null || true + find "$WORK_DIR" -name "target" -type d -maxdepth 4 -exec rm -rf {} \; 2>/dev/null || true + echo "Workspace target directories cleaned" + fi + + - name: Kill orphan processes + run: | + echo "=== Checking for orphan processes ===" + ps aux | grep -E "(cargo|rustc|node|java|python|mvn)" | grep -v grep || echo "No orphan processes found" + + - name: Disk usage after cleanup + run: | + echo "=== Disk Usage After ===" + df -h / + echo "" + echo "=== Large directories ===" + du -sh ~/.cargo 2>/dev/null || echo "~/.cargo: not found" + du -sh ~/.rustup 2>/dev/null || echo "~/.rustup: not found" + du -sh ~/.npm 2>/dev/null || echo "~/.npm: not found" + du -sh ~/.m2 2>/dev/null || echo "~/.m2: not found" + du -sh ~/Library/Caches/Homebrew 2>/dev/null || echo "Homebrew cache: not found" + du -sh ${{ runner.tool_cache }} 2>/dev/null || echo "Tool cache: not found" diff --git a/.github/workflows/release-ffi.yml b/.github/workflows/release-ffi.yml index 41e612c..fc7386c 100644 --- a/.github/workflows/release-ffi.yml +++ b/.github/workflows/release-ffi.yml @@ -1,41 +1,53 @@ name: Release FFI -run-name: "Release FFI v${{ inputs.version }}" +run-name: "Release FFI ${{ github.ref_name }}" on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g., 0.2.0)' - required: true - type: string + push: + tags: + - "ffi/v*" + +permissions: + contents: write jobs: build-ffi: strategy: + fail-fast: false matrix: include: - - os: linux-ubuntu-latest + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: x86_64-unknown-linux-gnu static_lib: libzerobus_ffi.a dynamic_lib: libzerobus_ffi.so - artifact_dir: linux-x86-64 + artifact_dir: linux-x86_64 cross: false - - os: linux-ubuntu-latest + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: aarch64-unknown-linux-gnu static_lib: libzerobus_ffi.a dynamic_lib: libzerobus_ffi.so artifact_dir: linux-aarch64 cross: true - - os: windows-server-latest + - runner: { group: databricks-protected-runner-group, labels: windows-server-latest } target: x86_64-pc-windows-msvc static_lib: zerobus_ffi.lib dynamic_lib: zerobus_ffi.dll - artifact_dir: windows-x86-64 + artifact_dir: windows-x86_64 + cross: false + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: aarch64-apple-darwin + static_lib: libzerobus_ffi.a + dynamic_lib: libzerobus_ffi.dylib + artifact_dir: macos-aarch64 + cross: false + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: x86_64-apple-darwin + static_lib: libzerobus_ffi.a + dynamic_lib: libzerobus_ffi.dylib + artifact_dir: macos-x86_64 cross: false - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} + name: Build FFI - ${{ matrix.artifact_dir }} + runs-on: ${{ matrix.runner }} defaults: run: @@ -47,7 +59,7 @@ jobs: - name: Install protoc uses: arduino/setup-protoc@v3 - - name: Install cross-compilation toolchain + - name: Install cross-compilation toolchain (Linux ARM64) if: matrix.cross run: | sudo apt-get update -qq @@ -64,36 +76,34 @@ jobs: - name: Build FFI library run: cargo build --release -p zerobus-ffi --target ${{ matrix.target }} - - name: Create artifact directory - run: mkdir -p artifacts/${{ matrix.artifact_dir }} - - - name: Copy artifacts (Unix) - if: runner.os != 'Windows' + - name: Prepare artifacts + shell: bash run: | + mkdir -p artifacts/${{ matrix.artifact_dir }} cp target/${{ matrix.target }}/release/${{ matrix.static_lib }} artifacts/${{ matrix.artifact_dir }}/ cp target/${{ matrix.target }}/release/${{ matrix.dynamic_lib }} artifacts/${{ matrix.artifact_dir }}/ cp ffi/zerobus.h artifacts/${{ matrix.artifact_dir }}/ - - name: Copy artifacts (Windows) - if: runner.os == 'Windows' - run: | - copy target\${{ matrix.target }}\release\${{ matrix.static_lib }} artifacts\${{ matrix.artifact_dir }}\ - copy target\${{ matrix.target }}\release\${{ matrix.dynamic_lib }} artifacts\${{ matrix.artifact_dir }}\ - copy ffi\zerobus.h artifacts\${{ matrix.artifact_dir }}\ - - name: Upload artifact uses: actions/upload-artifact@v4 with: name: ffi-${{ matrix.artifact_dir }} path: rust/artifacts/${{ matrix.artifact_dir }} - package: + release: + name: Create GitHub Release needs: build-ffi runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF#refs/tags/ffi/v}" >> "$GITHUB_OUTPUT" + - name: Download all artifacts uses: actions/download-artifact@v4 with: @@ -101,46 +111,24 @@ jobs: pattern: ffi-* merge-multiple: false - - name: Organize artifacts + - name: Create release archive run: | - mkdir -p release/native + mkdir -p release for dir in native/ffi-*/; do platform=$(basename "$dir" | sed 's/ffi-//') - mkdir -p "release/native/$platform" - cp "$dir"/* "release/native/$platform/" + mkdir -p "release/$platform" + cp "$dir"/* "release/$platform/" done - ls -laR release/ + tar -czvf zerobus-ffi-${{ steps.version.outputs.version }}.tar.gz -C release . - - name: Create release archive + - name: Extract release notes from changelog run: | - cd release - tar -czvf ../zerobus-ffi-${{ inputs.version }}.tar.gz native/ - cd .. - ls -la - - - name: Upload release archive - uses: actions/upload-artifact@v4 - with: - name: zerobus-ffi-${{ inputs.version }} - path: zerobus-ffi-${{ inputs.version }}.tar.gz + version="v${{ steps.version.outputs.version }}" + awk "/^## Release ${version}/{found=1; next} /^## /{if(found) exit} found{print}" rust/ffi/CHANGELOG.md > /tmp/release-notes.md - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: - tag_name: ffi-v${{ inputs.version }} - name: FFI Libraries v${{ inputs.version }} - body: | - Native C FFI libraries for zerobus-sdk-rs v${{ inputs.version }} - - Use these libraries to build language bindings (Go, C#, C++, etc.) - - Platforms included: - - linux-x86-64 (.a, .so) - - linux-aarch64 (.a, .so) - - windows-x86-64 (.lib, .dll) - - Each platform includes: - - Static library for Go/C++ (libzerobus_ffi.a / zerobus_ffi.lib) - - Dynamic library for C#/others (libzerobus_ffi.so / .dll) - - C header file (zerobus.h) - files: zerobus-ffi-${{ inputs.version }}.tar.gz + name: FFI Libraries v${{ steps.version.outputs.version }} + body_path: /tmp/release-notes.md + files: zerobus-ffi-${{ steps.version.outputs.version }}.tar.gz diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 3a4309d..93c2ff7 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -1,158 +1,35 @@ -name: Go Release +name: Release Go run-name: "Release Go SDK ${{ github.ref_name }}" on: push: tags: - - 'go/v*' - workflow_dispatch: + - "go/v*" + +permissions: + contents: write jobs: - build-rust: - name: Build Rust (${{ matrix.target }}) + release: + name: Create GitHub Release runs-on: group: databricks-protected-runner-group - labels: ${{ matrix.os-label }} - strategy: - fail-fast: false - matrix: - include: - - os-label: linux-ubuntu-latest - arch: amd64 - target: x86_64-unknown-linux-gnu - use-zig: false - - os-label: linux-ubuntu-latest - arch: arm64 - target: aarch64-unknown-linux-gnu - use-zig: false - - os-label: linux-ubuntu-latest - arch: amd64 - target: x86_64-apple-darwin - use-zig: true - - os-label: linux-ubuntu-latest - arch: arm64 - target: aarch64-apple-darwin - use-zig: true - - os-label: windows-server-latest - arch: amd64 - target: x86_64-pc-windows-gnu - use-zig: false + labels: linux-ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - - name: Install Cross-Compilation Tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF#refs/tags/go/v}" >> "$GITHUB_OUTPUT" - - name: Install Zig and cargo-zigbuild (for macOS cross-compilation) - if: matrix.use-zig == true - shell: bash + - name: Extract release notes from changelog run: | - ARCH=$(uname -m) - case "$ARCH" in - x86_64) ZIG_ARCH="x86_64" ;; - aarch64) ZIG_ARCH="aarch64" ;; - *) echo "Unsupported architecture: $ARCH"; exit 1 ;; - esac - - ZIG_VERSION="0.13.0" - ZIG_URL="https://ziglang.org/download/${ZIG_VERSION}/zig-linux-${ZIG_ARCH}-${ZIG_VERSION}.tar.xz" - - echo "Downloading Zig from $ZIG_URL" - wget -q "$ZIG_URL" - tar -xf "zig-linux-${ZIG_ARCH}-${ZIG_VERSION}.tar.xz" - - mkdir -p $HOME/.local/bin - rm -rf $HOME/.local/zig - mv "zig-linux-${ZIG_ARCH}-${ZIG_VERSION}" $HOME/.local/zig - ln -sf $HOME/.local/zig/zig $HOME/.local/bin/zig - echo "$HOME/.local/bin" >> $GITHUB_PATH - - if ! command -v cargo-zigbuild &> /dev/null; then - cargo install cargo-zigbuild - fi - - - name: Build Rust Library (Standard) - if: matrix.use-zig == false - working-directory: rust/ffi - run: | - cargo build --release --target ${{ matrix.target }} - env: - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - - - name: Build Rust Library (Zig Cross-Compile) - if: matrix.use-zig == true - working-directory: rust/ffi - run: | - cargo zigbuild --release --target ${{ matrix.target }} - - - name: Prepare Artifact - shell: bash - run: | - case "${{ matrix.target }}" in - *-apple-darwin) OS_NAME="darwin" ;; - *-pc-windows-*) OS_NAME="windows" ;; - *-linux-*) OS_NAME="linux" ;; - *) OS_NAME="unknown" ;; - esac - - mkdir -p dist/${OS_NAME}_${{ matrix.arch }} - - if [ "${{ runner.os }}" = "Windows" ]; then - cp rust/target/${{ matrix.target }}/release/libzerobus_ffi.a dist/windows_${{ matrix.arch }}/libzerobus_ffi.a || \ - cp rust/target/${{ matrix.target }}/release/zerobus_ffi.lib dist/windows_${{ matrix.arch }}/libzerobus_ffi.a - else - cp rust/target/${{ matrix.target }}/release/libzerobus_ffi.a dist/${OS_NAME}_${{ matrix.arch }}/libzerobus_ffi.a - fi + version="v${{ steps.version.outputs.version }}" + awk "/^## Release ${version}/{found=1; next} /^## /{if(found) exit} found{print}" go/CHANGELOG.md > /tmp/release-notes.md - - name: Upload Artifact - uses: actions/upload-artifact@v4 + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 with: - name: lib-${{ matrix.target }} - path: dist/ - - update-tag: - name: Update Tag with Binaries - if: startsWith(github.ref, 'refs/tags/go/') - needs: build-rust - runs-on: - group: databricks-protected-runner-group - labels: linux-ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: lib-artifacts - merge-multiple: true - - - name: Prepare lib directory - run: | - mkdir -p go/lib - cp -r lib-artifacts/* go/lib/ - rm -rf lib-artifacts - - - name: Commit and Force-Push Tag - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - git add -f go/lib/ - - git commit -m "Release ${{ github.ref_name }} with pre-built binaries" - - git tag -f ${{ github.ref_name }} - git push origin ${{ github.ref_name }} --force \ No newline at end of file + name: Go SDK v${{ steps.version.outputs.version }} + body_path: /tmp/release-notes.md diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index f5a8c46..3177956 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -10,11 +10,85 @@ permissions: contents: write jobs: + build-jni: + strategy: + fail-fast: false + matrix: + include: + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + target: x86_64-unknown-linux-gnu + artifact: libzerobus_jni.so + native_dir: linux-x86_64 + cross: false + - os: linux-ubuntu-latest + runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + target: aarch64-unknown-linux-gnu + artifact: libzerobus_jni.so + native_dir: linux-aarch64 + cross: true + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } + target: x86_64-pc-windows-msvc + artifact: zerobus_jni.dll + native_dir: windows-x86_64 + cross: false + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: aarch64-apple-darwin + artifact: libzerobus_jni.dylib + native_dir: osx-aarch64 + cross: false + - os: macos-arm64 + runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: x86_64-apple-darwin + artifact: libzerobus_jni.dylib + native_dir: osx-x86_64 + cross: false + + name: Build JNI - ${{ matrix.native_dir }} + runs-on: ${{ matrix.runner }} + + defaults: + run: + working-directory: rust + + steps: + - uses: actions/checkout@v4 + + - name: Install protoc + uses: arduino/setup-protoc@v3 + + - name: Install cross-compilation toolchain (Linux ARM64) + if: matrix.cross + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq gcc-aarch64-linux-gnu + mkdir -p .cargo + echo '[target.aarch64-unknown-linux-gnu]' > .cargo/config.toml + echo 'linker = "aarch64-linux-gnu-gcc"' >> .cargo/config.toml + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Build JNI library + run: cargo build --release -p zerobus-jni --target ${{ matrix.target }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: jni-${{ matrix.native_dir }} + path: rust/target/${{ matrix.target }}/release/${{ matrix.artifact }} + if-no-files-found: error + publish: name: Publish to Maven Central runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest + needs: build-jni defaults: run: working-directory: java @@ -22,6 +96,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Download all JNI artifacts + uses: actions/download-artifact@v4 + with: + path: jni-artifacts + pattern: jni-* + + - name: Place JNI libraries in native directory + run: | + for dir in ../jni-artifacts/jni-*/; do + platform=$(basename "$dir" | sed 's/jni-//') + mkdir -p src/main/resources/native/"$platform" + cp "$dir"/* src/main/resources/native/"$platform"/ + done + find src/main/resources/native -type f | sort + - name: Set up Java uses: actions/setup-java@v4 with: @@ -54,6 +143,22 @@ jobs: id: version run: echo "version=${GITHUB_REF#refs/tags/java/v}" >> "$GITHUB_OUTPUT" + - name: Download all JNI artifacts + uses: actions/download-artifact@v4 + with: + path: jni-artifacts + pattern: jni-* + + - name: Create JNI archive + run: | + mkdir -p native + for dir in jni-artifacts/jni-*/; do + platform=$(basename "$dir" | sed 's/jni-//') + mkdir -p "native/$platform" + cp "$dir"/* "native/$platform/" + done + tar -czvf zerobus-jni-${{ steps.version.outputs.version }}.tar.gz native/ + - name: Extract release notes from changelog run: | version="v${{ steps.version.outputs.version }}" @@ -64,3 +169,4 @@ jobs: with: name: Java SDK v${{ steps.version.outputs.version }} body_path: /tmp/release-notes.md + files: zerobus-jni-${{ steps.version.outputs.version }}.tar.gz diff --git a/.github/workflows/release-jni.yml b/.github/workflows/release-jni.yml deleted file mode 100644 index d66a00a..0000000 --- a/.github/workflows/release-jni.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: Release JNI -run-name: "Release JNI v${{ inputs.version }}" - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g., 0.2.0)' - required: true - type: string - -jobs: - build-jni: - strategy: - matrix: - include: - - os: linux-ubuntu-latest - target: x86_64-unknown-linux-gnu - artifact: libzerobus_jni.so - artifact_dir: linux-x86-64 - cross: false - - os: linux-ubuntu-latest - target: aarch64-unknown-linux-gnu - artifact: libzerobus_jni.so - artifact_dir: linux-aarch64 - cross: true - - os: windows-server-latest - target: x86_64-pc-windows-msvc - artifact: zerobus_jni.dll - artifact_dir: windows-x86-64 - cross: false - - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.os }} - - defaults: - run: - working-directory: rust - - steps: - - uses: actions/checkout@v4 - - - name: Install protoc - uses: arduino/setup-protoc@v3 - - - name: Install cross-compilation toolchain - if: matrix.cross - run: | - sudo apt-get update -qq - sudo apt-get install -y -qq gcc-aarch64-linux-gnu - mkdir -p .cargo - echo '[target.aarch64-unknown-linux-gnu]' > .cargo/config.toml - echo 'linker = "aarch64-linux-gnu-gcc"' >> .cargo/config.toml - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - - name: Build JNI library - run: cargo build --release -p zerobus-jni --target ${{ matrix.target }} - - - name: Create artifact directory - run: mkdir -p artifacts/${{ matrix.artifact_dir }} - - - name: Copy artifact (Unix) - if: runner.os != 'Windows' - run: cp target/${{ matrix.target }}/release/${{ matrix.artifact }} artifacts/${{ matrix.artifact_dir }}/ - - - name: Copy artifact (Windows) - if: runner.os == 'Windows' - run: copy target\${{ matrix.target }}\release\${{ matrix.artifact }} artifacts\${{ matrix.artifact_dir }}\ - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: jni-${{ matrix.artifact_dir }} - path: rust/artifacts/${{ matrix.artifact_dir }} - - package: - needs: build-jni - runs-on: - group: databricks-protected-runner-group - labels: linux-ubuntu-latest - - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: native - pattern: jni-* - merge-multiple: false - - - name: Organize artifacts - run: | - mkdir -p release/native - for dir in native/jni-*/; do - platform=$(basename "$dir" | sed 's/jni-//') - mkdir -p "release/native/$platform" - cp "$dir"/* "release/native/$platform/" - done - ls -laR release/ - - - name: Create release archive - run: | - cd release - tar -czvf ../zerobus-jni-${{ inputs.version }}.tar.gz native/ - cd .. - ls -la - - - name: Upload release archive - uses: actions/upload-artifact@v4 - with: - name: zerobus-jni-${{ inputs.version }} - path: zerobus-jni-${{ inputs.version }}.tar.gz - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: jni-v${{ inputs.version }} - name: JNI Libraries v${{ inputs.version }} - body: | - Native JNI libraries for zerobus-sdk-java v${{ inputs.version }} - - Platforms included: - - linux-x86-64 - - linux-aarch64 - - windows-x86-64 - files: zerobus-jni-${{ inputs.version }}.tar.gz diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 2b830dd..8229111 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -1,4 +1,4 @@ -name: Build & Release to PyPI +name: Release Python run-name: "Release Python SDK ${{ github.ref_name }}" on: @@ -12,14 +12,15 @@ concurrency: cancel-in-progress: true permissions: - contents: read + contents: write jobs: - linux: + build-linux: runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest strategy: + fail-fast: false matrix: target: [x86_64, aarch64] steps: @@ -65,7 +66,7 @@ jobs: path: dist if-no-files-found: error - windows: + build-windows: runs-on: group: databricks-protected-runner-group labels: windows-server-latest @@ -99,6 +100,39 @@ jobs: path: dist if-no-files-found: error + build-macos: + runs-on: [self-hosted, macOS, ARM64, zerobus-sdk] + strategy: + fail-fast: false + matrix: + target: [aarch64-apple-darwin, x86_64-apple-darwin] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Copy root LICENSE for maturin + run: cp LICENSE python/LICENSE + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install maturin + run: pip install maturin + + - name: Build wheels + run: maturin build --release --target ${{ matrix.target }} --manifest-path python/rust/Cargo.toml --out dist + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.target }} + path: dist + if-no-files-found: error + sdist: runs-on: group: databricks-protected-runner-group @@ -130,13 +164,13 @@ jobs: path: dist if-no-files-found: error - release: - name: Release to PyPI + publish: + name: Publish to PyPI runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest if: startsWith(github.ref, 'refs/tags/python/') - needs: [linux, windows, sdist] + needs: [build-linux, build-windows, build-macos, sdist] permissions: id-token: write steps: @@ -154,3 +188,28 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: skip-existing: true + + release: + name: Create GitHub Release + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + needs: publish + + steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF#refs/tags/python/v}" >> "$GITHUB_OUTPUT" + + - name: Extract release notes from changelog + run: | + version="v${{ steps.version.outputs.version }}" + awk "/^## Release ${version}/{found=1; next} /^## /{if(found) exit} found{print}" python/CHANGELOG.md > /tmp/release-notes.md + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: Python SDK v${{ steps.version.outputs.version }} + body_path: /tmp/release-notes.md diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml new file mode 100644 index 0000000..aa0ec02 --- /dev/null +++ b/.github/workflows/release-rust.yml @@ -0,0 +1,56 @@ +name: Release Rust +run-name: "Release Rust SDK ${{ github.ref_name }}" + +on: + push: + tags: + - "rust/v*" + +permissions: + contents: write + id-token: write + +jobs: + publish: + name: Publish to crates.io + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + defaults: + run: + working-directory: rust + environment: crates-io + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Publish to crates.io (Trusted Publishing) + run: cargo publish -p databricks-zerobus-ingest-sdk + + release: + name: Create GitHub Release + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + needs: publish + + steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF#refs/tags/rust/v}" >> "$GITHUB_OUTPUT" + + - name: Extract release notes from changelog + run: | + version="v${{ steps.version.outputs.version }}" + awk "/^## Release ${version}/{found=1; next} /^## /{if(found) exit} found{print}" rust/CHANGELOG.md > /tmp/release-notes.md + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: Rust SDK v${{ steps.version.outputs.version }} + body_path: /tmp/release-notes.md diff --git a/.github/workflows/release-typescript.yml b/.github/workflows/release-typescript.yml index fa08c4a..cdd6e17 100644 --- a/.github/workflows/release-typescript.yml +++ b/.github/workflows/release-typescript.yml @@ -15,18 +15,20 @@ jobs: strategy: fail-fast: false matrix: - settings: - - host: linux-ubuntu-latest + include: + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: x86_64-unknown-linux-gnu - - host: linux-ubuntu-latest + - runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } target: aarch64-unknown-linux-gnu - - host: windows-server-latest + - runner: { group: databricks-protected-runner-group, labels: windows-server-latest } target: x86_64-pc-windows-msvc + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: aarch64-apple-darwin + - runner: [self-hosted, macOS, ARM64, zerobus-sdk] + target: x86_64-apple-darwin - name: Build - ${{ matrix.settings.target }} - runs-on: - group: databricks-protected-runner-group - labels: ${{ matrix.settings.host }} + name: Build - ${{ matrix.target }} + runs-on: ${{ matrix.runner }} defaults: run: working-directory: typescript @@ -39,8 +41,6 @@ jobs: with: node-version: 24 - - name: Use latest npm - run: npm install -g npm@latest - name: Install Rust (Unix) if: runner.os != 'Windows' @@ -57,10 +57,10 @@ jobs: shell: powershell - name: Add Rust target - run: rustup target add ${{ matrix.settings.target }} + run: rustup target add ${{ matrix.target }} - name: Install cross-compilation tools (Linux ARM64) - if: matrix.settings.target == 'aarch64-unknown-linux-gnu' + if: matrix.target == 'aarch64-unknown-linux-gnu' run: | sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu @@ -69,14 +69,14 @@ jobs: run: npm ci - name: Build - run: npm run build -- --target ${{ matrix.settings.target }} + run: npm run build -- --target ${{ matrix.target }} env: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: bindings-${{ matrix.settings.target }} + name: bindings-${{ matrix.target }} path: 'typescript/*.node' if-no-files-found: error @@ -99,8 +99,6 @@ jobs: node-version: 24 registry-url: 'https://registry.npmjs.org' - - name: Use latest npm - run: npm install -g npm@latest - name: Install dependencies run: npm ci @@ -110,17 +108,22 @@ jobs: with: path: typescript/artifacts - - name: List downloaded artifacts - run: ls -R artifacts/ + - name: Move artifacts into place + run: npx napi artifacts --artifacts-dir artifacts --output-dir . + + - name: List bindings + run: ls -la *.node + + - name: Generate platform packages + run: npx napi prepublish -t npm --skip-gh-release - - name: Copy bindings + - name: Publish platform packages run: | - cp artifacts/bindings-x86_64-unknown-linux-gnu/*.node . - cp artifacts/bindings-aarch64-unknown-linux-gnu/*.node . - cp artifacts/bindings-x86_64-pc-windows-msvc/*.node . - cp prebuilt/*.node . + for dir in npm/*/; do + npm publish "$dir" --access public --provenance + done - - name: Publish + - name: Publish main package run: npm publish --access public --provenance --ignore-scripts release: diff --git a/go/.gitignore b/go/.gitignore index d7aa05c..4a3a81c 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -1,10 +1,8 @@ -# Build artifacts - DO NOT COMMIT +# Build artifacts zerobus-ffi/target/ -lib/ libzerobus_ffi.a *.dylib *.so -*.a # Release artifacts releases/ diff --git a/java/.gitignore b/java/.gitignore index 241a369..6cc5af5 100644 --- a/java/.gitignore +++ b/java/.gitignore @@ -10,3 +10,4 @@ buildNumber.properties .mvn/timing.properties .mvn/wrapper/maven-wrapper.jar examples/proto/**/AirQualityProto.java +src/main/resources/native/ diff --git a/java/src/main/resources/native/linux-aarch64/libzerobus_jni.so b/java/src/main/resources/native/linux-aarch64/libzerobus_jni.so deleted file mode 100755 index 4b84d5d..0000000 Binary files a/java/src/main/resources/native/linux-aarch64/libzerobus_jni.so and /dev/null differ diff --git a/java/src/main/resources/native/linux-x86_64/libzerobus_jni.so b/java/src/main/resources/native/linux-x86_64/libzerobus_jni.so deleted file mode 100755 index cf1ef2b..0000000 Binary files a/java/src/main/resources/native/linux-x86_64/libzerobus_jni.so and /dev/null differ diff --git a/java/src/main/resources/native/osx-aarch64/libzerobus_jni.dylib b/java/src/main/resources/native/osx-aarch64/libzerobus_jni.dylib deleted file mode 100755 index 8b6eb6e..0000000 Binary files a/java/src/main/resources/native/osx-aarch64/libzerobus_jni.dylib and /dev/null differ diff --git a/java/src/main/resources/native/osx-x86_64/libzerobus_jni.dylib b/java/src/main/resources/native/osx-x86_64/libzerobus_jni.dylib deleted file mode 100755 index a137d28..0000000 Binary files a/java/src/main/resources/native/osx-x86_64/libzerobus_jni.dylib and /dev/null differ diff --git a/java/src/main/resources/native/windows-x86_64/zerobus_jni.dll b/java/src/main/resources/native/windows-x86_64/zerobus_jni.dll deleted file mode 100644 index 5b4ce03..0000000 Binary files a/java/src/main/resources/native/windows-x86_64/zerobus_jni.dll and /dev/null differ diff --git a/rust/ffi/CHANGELOG.md b/rust/ffi/CHANGELOG.md new file mode 100644 index 0000000..e801b06 --- /dev/null +++ b/rust/ffi/CHANGELOG.md @@ -0,0 +1,17 @@ +# Version changelog + +## Release v1.0.1 + +Initial tracked release of the FFI C bindings for the Zerobus SDK. + +### Platforms +- Linux x86_64 +- Linux aarch64 +- macOS x86_64 +- macOS aarch64 +- Windows x86_64 + +### Libraries +- Static library (.a / .lib) +- Dynamic library (.so / .dylib / .dll) +- C header file (zerobus.h) diff --git a/rust/ffi/Cargo.toml b/rust/ffi/Cargo.toml index bc812dd..480ec25 100644 --- a/rust/ffi/Cargo.toml +++ b/rust/ffi/Cargo.toml @@ -7,7 +7,7 @@ license-file = "../../LICENSE" publish = false [lib] -crate-type = ["staticlib"] +crate-type = ["staticlib", "cdylib"] [dependencies] # Core SDK logic - use local path dependency diff --git a/rust/ffi/NEXT_CHANGELOG.md b/rust/ffi/NEXT_CHANGELOG.md new file mode 100644 index 0000000..da753bf --- /dev/null +++ b/rust/ffi/NEXT_CHANGELOG.md @@ -0,0 +1,23 @@ +# NEXT CHANGELOG + +## Release v1.1.0 + +### Major Changes + +### New Features and Improvements + +- Added dynamic library (.so / .dylib / .dll) output alongside static library +- Added macOS aarch64 and x86_64 platform support + +### Bug Fixes + +### Documentation + +### Internal Changes + +### Breaking Changes + +### Deprecations + +### API Changes + diff --git a/typescript/.gitignore b/typescript/.gitignore index f4bd2a6..672d517 100644 --- a/typescript/.gitignore +++ b/typescript/.gitignore @@ -1,7 +1,6 @@ target/ Cargo.lock *.node -!prebuilt/*.node node_modules/ index.js index.d.ts diff --git a/typescript/package.json b/typescript/package.json index 7fc397c..9ca67a4 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -28,8 +28,7 @@ "schemas/", "Cargo.toml", "Cargo.lock", - "build.rs", - "*.node" + "build.rs" ], "napi": { "name": "zerobus-ingest-sdk", @@ -54,7 +53,6 @@ "build:debug": "napi build --platform", "build:debug:arrow": "napi build --platform --features arrow-flight", "build:proto": "mkdir -p examples/generated && pbjs -t static-module -w commonjs -o examples/generated/air_quality.js schemas/air_quality.proto && pbts -o examples/generated/air_quality.d.ts examples/generated/air_quality.js && protoc --descriptor_set_out=schemas/air_quality_descriptor.pb --include_imports schemas/air_quality.proto", - "prepublishOnly": "napi prepublish -t npm", "test": "tsx --test test/unit.test.ts test/integration.test.ts", "test:unit": "tsx --test test/unit.test.ts", "test:integration": "tsx --test test/integration.test.ts", @@ -91,12 +89,5 @@ }, "overrides": { "glob": "^10.0.0" - }, - "optionalDependencies": { - "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.0.0", - "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.0.0", - "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.0.0", - "@databricks/zerobus-ingest-sdk-darwin-x64": "1.0.0", - "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.0.0" } } diff --git a/typescript/prebuilt/zerobus-ingest-sdk.darwin-arm64.node b/typescript/prebuilt/zerobus-ingest-sdk.darwin-arm64.node deleted file mode 100755 index cc541ce..0000000 Binary files a/typescript/prebuilt/zerobus-ingest-sdk.darwin-arm64.node and /dev/null differ diff --git a/typescript/prebuilt/zerobus-ingest-sdk.darwin-x64.node b/typescript/prebuilt/zerobus-ingest-sdk.darwin-x64.node deleted file mode 100755 index f7cdddb..0000000 Binary files a/typescript/prebuilt/zerobus-ingest-sdk.darwin-x64.node and /dev/null differ