diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml index c6a369e..b177bc6 100644 --- a/.github/workflows/ci-go.yml +++ b/.github/workflows/ci-go.yml @@ -2,6 +2,11 @@ name: Go CI on: workflow_call: + inputs: + use_local_sdk: + description: 'Override databricks-zerobus-ingest-sdk with the local path dep (for integration testing unreleased Rust changes)' + type: boolean + default: false jobs: fmt: @@ -75,6 +80,10 @@ jobs: with: targets: x86_64-pc-windows-gnu + - name: Patch Rust workspace to use local SDK + if: inputs.use_local_sdk + run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "sdk" }\n' >> ../rust/Cargo.toml + - name: Build Rust FFI run: make build-rust @@ -88,6 +97,8 @@ jobs: run: make examples user-experience: + # Skip when testing local SDK — this job tests the published release download flow + if: ${{ !inputs.use_local_sdk }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/ci-java.yml b/.github/workflows/ci-java.yml index dff7b00..80afab3 100644 --- a/.github/workflows/ci-java.yml +++ b/.github/workflows/ci-java.yml @@ -2,6 +2,15 @@ name: Java CI on: workflow_call: + inputs: + use_local_sdk: + description: 'Override databricks-zerobus-ingest-sdk with the local path dep (for integration testing unreleased Rust changes)' + type: boolean + default: false + artifact_suffix: + description: 'Suffix to append to artifact names to avoid conflicts when called multiple times in one workflow run' + type: string + default: '' jobs: fmt: @@ -49,6 +58,11 @@ jobs: java-version: ${{ matrix.javaVersion }} distribution: 'temurin' cache: 'maven' + - name: Patch Rust workspace to use local SDK + if: inputs.use_local_sdk + shell: bash + run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "sdk" }\n' >> ../rust/Cargo.toml + - name: Build with Maven run: mvn clean compile - name: Run tests @@ -57,7 +71,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-results-${{ matrix.os }}-java-${{ matrix.javaVersion }} + name: test-results-${{ matrix.os }}-java-${{ matrix.javaVersion }}${{ inputs.artifact_suffix }} path: | java/target/surefire-reports/ java/target/test-classes/ diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index eb2747b..5bb1a39 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -2,6 +2,11 @@ name: Python CI on: workflow_call: + inputs: + use_local_sdk: + description: 'Override databricks-zerobus-ingest-sdk with the local path dep (for integration testing unreleased Rust changes)' + type: boolean + default: false jobs: test: @@ -29,6 +34,12 @@ jobs: - name: Copy root LICENSE for maturin run: cp ../LICENSE LICENSE + - name: Patch Cargo.toml to use local Rust SDK + if: inputs.use_local_sdk + working-directory: python/rust + run: | + printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "../../rust/sdk" }\n' >> Cargo.toml + - name: Run tests run: make dev test diff --git a/.github/workflows/ci-typescript.yml b/.github/workflows/ci-typescript.yml index fd430ae..c41e004 100644 --- a/.github/workflows/ci-typescript.yml +++ b/.github/workflows/ci-typescript.yml @@ -2,6 +2,15 @@ name: TypeScript CI on: workflow_call: + inputs: + use_local_sdk: + description: 'Override databricks-zerobus-ingest-sdk with the local path dep (for integration testing unreleased Rust changes)' + type: boolean + default: false + artifact_suffix: + description: 'Suffix to append to artifact names to avoid conflicts when called multiple times in one workflow run' + type: string + default: '' jobs: build: @@ -67,6 +76,15 @@ jobs: sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + - name: Patch Cargo.toml to use local Rust SDK (Unix) + if: inputs.use_local_sdk && runner.os != 'Windows' + run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "../rust/sdk" }\n' >> Cargo.toml + + - name: Patch Cargo.toml to use local Rust SDK (Windows) + if: inputs.use_local_sdk && runner.os == 'Windows' + run: Add-Content -Path Cargo.toml -Value "`n[patch.crates-io]`ndatabricks-zerobus-ingest-sdk = { path = `"../rust/sdk`" }`n" + shell: powershell + - name: Install dependencies run: npm ci @@ -78,7 +96,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: bindings-${{ matrix.settings.target }} + name: bindings-${{ matrix.settings.target }}${{ inputs.artifact_suffix }} path: 'typescript/*.node' if-no-files-found: error @@ -120,6 +138,15 @@ jobs: echo "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append shell: powershell + - name: Patch Cargo.toml to use local Rust SDK (Unix) + if: inputs.use_local_sdk && runner.os != 'Windows' + run: printf '\n[patch.crates-io]\ndatabricks-zerobus-ingest-sdk = { path = "../rust/sdk" }\n' >> Cargo.toml + + - name: Patch Cargo.toml to use local Rust SDK (Windows) + if: inputs.use_local_sdk && runner.os == 'Windows' + run: Add-Content -Path Cargo.toml -Value "`n[patch.crates-io]`ndatabricks-zerobus-ingest-sdk = { path = `"../rust/sdk`" }`n" + shell: powershell + - name: Install dependencies run: npm ci diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3b07d31..d299190 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,13 +15,17 @@ jobs: if: always() steps: - run: | - if [[ "${{ needs.rust.result }}" == "failure" ]] || \ - [[ "${{ needs.java.result }}" == "failure" ]] || \ - [[ "${{ needs.typescript.result }}" == "failure" ]] || \ - [[ "${{ needs.python.result }}" == "failure" ]] || \ - [[ "${{ needs.go.result }}" == "failure" ]]; then - exit 1 - fi + for result in \ + "${{ needs.rust.result }}" \ + "${{ needs.java.result }}" \ + "${{ needs.typescript.result }}" \ + "${{ needs.python.result }}" \ + "${{ needs.go.result }}"; do + if [[ "$result" != "success" && "$result" != "skipped" ]]; then + echo "Job failed or was cancelled: $result" + exit 1 + fi + done changes: runs-on: @@ -55,6 +59,7 @@ jobs: - 'go/**' - '.github/workflows/ci-go.yml' + # Blocking jobs — required for merge rust: needs: changes if: needs.changes.outputs.rust == 'true' @@ -69,14 +74,46 @@ jobs: needs: changes if: needs.changes.outputs.typescript == 'true' uses: ./.github/workflows/ci-typescript.yml - + python: needs: changes if: needs.changes.outputs.python == 'true' uses: ./.github/workflows/ci-python.yml - + go: needs: changes if: needs.changes.outputs.go == 'true' uses: ./.github/workflows/ci-go.yml - \ No newline at end of file + + # Non-blocking cross-SDK jobs — run all SDK tests against the local Rust + # source when rust/** changes, to catch breakage before a new FFI release. + # These are intentionally excluded from `gate` so they don't block merging. + cross-sdk-go: + needs: changes + if: needs.changes.outputs.rust == 'true' + uses: ./.github/workflows/ci-go.yml + with: + use_local_sdk: true + + cross-sdk-python: + needs: changes + if: needs.changes.outputs.rust == 'true' + uses: ./.github/workflows/ci-python.yml + with: + use_local_sdk: true + + cross-sdk-typescript: + needs: changes + if: needs.changes.outputs.rust == 'true' + uses: ./.github/workflows/ci-typescript.yml + with: + use_local_sdk: true + artifact_suffix: '-local-sdk' + + cross-sdk-java: + needs: changes + if: needs.changes.outputs.rust == 'true' + uses: ./.github/workflows/ci-java.yml + with: + use_local_sdk: true + artifact_suffix: '-local-sdk' diff --git a/.gitignore b/.gitignore index e31a805..e40addc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # IDE .idea/ +*/flycheck* *.iml *.iws *.ipr diff --git a/rust/ffi/Cargo.toml b/rust/ffi/Cargo.toml index 762d58c..bc812dd 100644 --- a/rust/ffi/Cargo.toml +++ b/rust/ffi/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["staticlib"] [dependencies] # Core SDK logic - use local path dependency -databricks-zerobus-ingest-sdk = { path = "../sdk", features = ["arrow-flight", "testing"] } +databricks-zerobus-ingest-sdk = { version="1.0.1", features = ["arrow-flight", "testing"] } # Arrow IPC for serializing/deserializing RecordBatches across the FFI boundary arrow-ipc = { version = "56.2.0", default-features = false } diff --git a/rust/jni/Cargo.toml b/rust/jni/Cargo.toml index 75f8af4..c4c67e4 100644 --- a/rust/jni/Cargo.toml +++ b/rust/jni/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib"] name = "zerobus_jni" [dependencies] -databricks-zerobus-ingest-sdk = { path = "../sdk", features = ["arrow-flight"] } +databricks-zerobus-ingest-sdk = { version = "1.0.1", features = ["arrow-flight"] } jni = "0.21" tokio = { version = "1.42", features = ["rt-multi-thread", "sync"] } prost = "0.13"