diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml index 58e68254986..98b93a6f8d0 100644 --- a/.github/workflows/velox_backend_ansi.yml +++ b/.github/workflows/velox_backend_ansi.yml @@ -124,8 +124,6 @@ jobs: ccache -sz bash dev/ci-velox-buildstatic-centos-7.sh ccache -s - mkdir -p /work/.m2/repository/org/apache/arrow/ - cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ " - name: Save ccache if: always() @@ -138,11 +136,6 @@ jobs: name: velox-native-lib-ansi-${{github.sha}} path: ./cpp/build/ if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: arrow-jars-ansi-${{github.sha}} - path: .m2/repository/org/apache/arrow/ - if-no-files-found: error spark-test-backends-velox-ansi41: needs: build-native-lib @@ -159,11 +152,6 @@ jobs: with: name: velox-native-lib-ansi-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-ansi-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf install -y python3.11 python3.11-pip python3.11-devel && \ @@ -236,11 +224,6 @@ jobs: with: name: velox-native-lib-ansi-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-ansi-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf install -y python3.11 python3.11-pip python3.11-devel && \ @@ -323,11 +306,6 @@ jobs: with: name: velox-native-lib-ansi-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-ansi-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf install -y python3.11 python3.11-pip python3.11-devel && \ @@ -400,11 +378,6 @@ jobs: with: name: velox-native-lib-ansi-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-ansi-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf install -y python3.11 python3.11-pip python3.11-devel && \ diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml index f7346b64b2d..d9b49b8f455 100644 --- a/.github/workflows/velox_backend_arm.yml +++ b/.github/workflows/velox_backend_arm.yml @@ -68,8 +68,6 @@ jobs: df -a bash dev/ci-velox-buildstatic-centos-9.sh ccache -s - mkdir -p $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/ - cp -r /root/.m2/repository/org/apache/arrow/* $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/ - name: "Save ccache" uses: actions/cache/save@v4 id: ccache @@ -81,11 +79,6 @@ jobs: name: velox-native-lib-centos-8-${{github.sha}} path: ./cpp/build/releases/ if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: arrow-jars-centos-8-${{github.sha}} - path: .m2/repository/org/apache/arrow/ - if-no-files-found: error tpc-test-centos8: needs: build-native-lib-centos-8 @@ -104,11 +97,6 @@ jobs: with: name: velox-native-lib-centos-8-${{github.sha}} path: ./cpp/build/releases/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-8-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true diff --git a/.github/workflows/velox_backend_enhanced.yml b/.github/workflows/velox_backend_enhanced.yml index ff669c7fa98..b3275df629f 100644 --- a/.github/workflows/velox_backend_enhanced.yml +++ b/.github/workflows/velox_backend_enhanced.yml @@ -75,8 +75,6 @@ jobs: mkdir -p /work/.ccache bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh ccache -s - mkdir -p /work/.m2/repository/org/apache/arrow/ - cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ " - name: "Save ccache" @@ -90,11 +88,6 @@ jobs: name: velox-native-lib-enhanced-centos-7-${{github.sha}} path: ./cpp/build/releases/ if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: arrow-jars-enhanced-centos-7-${{github.sha}} - path: .m2/repository/org/apache/arrow/ - if-no-files-found: error spark-test-spark34: needs: build-native-lib-centos-7 @@ -107,11 +100,6 @@ jobs: with: name: velox-native-lib-enhanced-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-enhanced-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare spark.test.home for Spark 3.4.4 (other tests) run: | dnf module -y install python39 && \ @@ -165,11 +153,6 @@ jobs: with: name: velox-native-lib-enhanced-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-enhanced-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare spark.test.home for Spark 3.5.5 (other tests) run: | dnf module -y install python39 && \ @@ -226,11 +209,6 @@ jobs: with: name: velox-native-lib-enhanced-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-enhanced-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update run: | rm -rf /opt/shims/spark35 @@ -277,11 +255,6 @@ jobs: with: name: velox-native-lib-enhanced-centos-7-${{github.sha}} path: ./cpp/build/releases/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-enhanced-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 8129beac8e0..58ffd6e1612 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -79,8 +79,6 @@ jobs: ccache -sz bash dev/ci-velox-buildstatic-centos-7.sh ccache -s - mkdir -p /work/.m2/repository/org/apache/arrow/ - cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ " - name: "Save ccache" @@ -95,11 +93,6 @@ jobs: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: .m2/repository/org/apache/arrow/ - if-no-files-found: error tpc-test-ubuntu: needs: build-native-lib-centos-7 @@ -157,11 +150,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Setup tzdata run: | sed -i 's|http://archive|http://eu.archive|g' /etc/apt/sources.list @@ -184,7 +172,6 @@ jobs: apt-get update && apt-get install -y openjdk-8-jdk wget apt remove openjdk-11* -y fi - ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/ - name: Install Hadoop & Setup HDFS if: matrix.os == 'ubuntu:22.04' && matrix.spark == 'spark-3.5' && matrix.java == 'java-8' shell: bash @@ -280,11 +267,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | if [ "${{ matrix.os }}" = "centos:7" ] || [ "${{ matrix.os }}" = "centos:8" ]; then @@ -347,11 +329,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: .m2/repository/org/apache/arrow/ - name: Build and run TPCH/DS tests run: | docker pull centos:7 @@ -406,11 +383,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java run: | sudo apt-get update @@ -521,11 +493,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java run: | sudo apt-get update @@ -564,11 +531,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Install Uniffle ${{ matrix.uniffle }} run: | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk @@ -615,11 +577,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download All Arrow Jar Artifacts - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true @@ -689,11 +646,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ @@ -749,11 +701,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 3.3.1 #TODO remove after image update run: | rm -rf /opt/shims/spark33 @@ -799,11 +746,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare spark.test.home for Spark 3.4.4 (other tests) run: | dnf module -y install python39 && \ @@ -860,11 +802,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 3.4.4 #TODO remove after image update run: | rm -rf /opt/shims/spark34 @@ -913,11 +850,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ @@ -971,11 +903,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ @@ -1025,11 +952,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update run: | rm -rf /opt/shims/spark35 @@ -1076,11 +998,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ @@ -1129,11 +1046,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update run: | rm -rf /opt/shims/spark35 @@ -1296,11 +1208,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf module -y install python39 && \ @@ -1354,11 +1261,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image update run: | rm -rf /opt/shims/spark40 @@ -1405,11 +1307,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare run: | dnf install -y python3.11 python3.11-pip python3.11-devel && \ @@ -1465,11 +1362,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/ - - name: Download Arrow Jars - uses: actions/download-artifact@v4 - with: - name: arrow-jars-centos-7-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ - name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image update run: | rm -rf /opt/shims/spark41 diff --git a/dev/build-arrow.sh b/dev/build-arrow.sh index 1bf4ef9d089..007047a8ad2 100755 --- a/dev/build-arrow.sh +++ b/dev/build-arrow.sh @@ -75,6 +75,21 @@ function build_arrow_cpp() { } function build_arrow_java() { + # Maven Central's arrow-c-data / arrow-dataset jars at ${VELOX_ARROW_BUILD_VERSION} + # already ship libarrow_cdata_jni / libarrow_dataset_jni for x86_64 (Linux/macOS/Windows) + # and aarch_64 (Linux/macOS), so contributors on those archs do not need a locally-built + # jar — gluten-arrow resolves the same artifact transitively. + # + # ppc64le has no native in the Central jar; support_ibm_power.patch (applied above) adds + # the ppc64le -> ppcle_64 arch case to JniLoader.java and the local mvn install step bakes + # a locally-built libarrow_cdata_jni.so for ppc64le into the resulting arrow-c-data jar in + # ~/.m2, overriding Central. Skip the Java build on every other arch. + local ARCH=$(uname -m) + if [[ "${ARCH}" != "ppc64le" ]]; then + echo "Skipping local Arrow Java build on ${ARCH} — gluten resolves arrow-c-data:${VELOX_ARROW_BUILD_VERSION} from Maven Central. Local build is only required on ppc64le for the patched JniLoader." + return 0 + fi + ARROW_INSTALL_DIR="${ARROW_PREFIX}/install" # Use Gluten's Maven wrapper @@ -96,8 +111,6 @@ function build_arrow_java() { export CMAKE_BUILD_PARALLEL_LEVEL=$NPROC pushd $ARROW_PREFIX/java - # Because arrow-bom module need the -DprocessAllModules - ${MVN_CMD} versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules ${MVN_CMD} clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \ -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly diff --git a/gluten-arrow/pom.xml b/gluten-arrow/pom.xml index 62d9bd243a7..defcfcbf26f 100644 --- a/gluten-arrow/pom.xml +++ b/gluten-arrow/pom.xml @@ -88,13 +88,13 @@ org.apache.arrow ${arrow-memory.artifact} - ${arrow-gluten.version} + ${arrow.version} runtime org.apache.arrow arrow-memory-core - ${arrow-gluten.version} + ${arrow.version} compile @@ -110,7 +110,7 @@ org.apache.arrow arrow-vector - ${arrow-gluten.version} + ${arrow.version} io.netty @@ -129,7 +129,7 @@ org.apache.arrow arrow-c-data - ${arrow-gluten.version} + ${arrow.version} compile @@ -145,7 +145,7 @@ org.apache.arrow arrow-dataset - ${arrow-gluten.version} + ${arrow.version} compile diff --git a/pom.xml b/pom.xml index 1b3a9d2a3cc..fc4dbb273fc 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,6 @@ 0.6.3 0.10.0 15.0.0 - 15.0.0-gluten arrow-memory-unsafe 2.7.4 4.9.3 @@ -1287,7 +1286,6 @@ 2.24.3 3.17.0 18.1.0 - 18.1.0 4.9.2 @@ -1367,7 +1365,6 @@ 2.24.3 3.17.0 18.1.0 - 18.1.0 4.9.5