diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml
index 58e68254986..98b93a6f8d0 100644
--- a/.github/workflows/velox_backend_ansi.yml
+++ b/.github/workflows/velox_backend_ansi.yml
@@ -124,8 +124,6 @@ jobs:
ccache -sz
bash dev/ci-velox-buildstatic-centos-7.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
"
- name: Save ccache
if: always()
@@ -138,11 +136,6 @@ jobs:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
spark-test-backends-velox-ansi41:
needs: build-native-lib
@@ -159,11 +152,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -236,11 +224,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -323,11 +306,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -400,11 +378,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml
index f7346b64b2d..d9b49b8f455 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -68,8 +68,6 @@ jobs:
df -a
bash dev/ci-velox-buildstatic-centos-9.sh
ccache -s
- mkdir -p $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/* $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
- name: "Save ccache"
uses: actions/cache/save@v4
id: ccache
@@ -81,11 +79,6 @@ jobs:
name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-centos-8-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
tpc-test-centos8:
needs: build-native-lib-centos-8
@@ -104,11 +97,6 @@ jobs:
with:
name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-8-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
diff --git a/.github/workflows/velox_backend_enhanced.yml b/.github/workflows/velox_backend_enhanced.yml
index ff669c7fa98..b3275df629f 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -75,8 +75,6 @@ jobs:
mkdir -p /work/.ccache
bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
"
- name: "Save ccache"
@@ -90,11 +88,6 @@ jobs:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
spark-test-spark34:
needs: build-native-lib-centos-7
@@ -107,11 +100,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
run: |
dnf module -y install python39 && \
@@ -165,11 +153,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.5.5 (other tests)
run: |
dnf module -y install python39 && \
@@ -226,11 +209,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update
run: |
rm -rf /opt/shims/spark35
@@ -277,11 +255,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml
index 8129beac8e0..58ffd6e1612 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -79,8 +79,6 @@ jobs:
ccache -sz
bash dev/ci-velox-buildstatic-centos-7.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
"
- name: "Save ccache"
@@ -95,11 +93,6 @@ jobs:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
tpc-test-ubuntu:
needs: build-native-lib-centos-7
@@ -157,11 +150,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Setup tzdata
run: |
sed -i 's|http://archive|http://eu.archive|g' /etc/apt/sources.list
@@ -184,7 +172,6 @@ jobs:
apt-get update && apt-get install -y openjdk-8-jdk wget
apt remove openjdk-11* -y
fi
- ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/
- name: Install Hadoop & Setup HDFS
if: matrix.os == 'ubuntu:22.04' && matrix.spark == 'spark-3.5' && matrix.java == 'java-8'
shell: bash
@@ -280,11 +267,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
if [ "${{ matrix.os }}" = "centos:7" ] || [ "${{ matrix.os }}" = "centos:8" ]; then
@@ -347,11 +329,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- name: Build and run TPCH/DS tests
run: |
docker pull centos:7
@@ -406,11 +383,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /home/runner/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
sudo apt-get update
@@ -521,11 +493,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /home/runner/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
sudo apt-get update
@@ -564,11 +531,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Install Uniffle ${{ matrix.uniffle }}
run: |
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
@@ -615,11 +577,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
@@ -689,11 +646,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -749,11 +701,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.3.1 #TODO remove after image update
run: |
rm -rf /opt/shims/spark33
@@ -799,11 +746,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
run: |
dnf module -y install python39 && \
@@ -860,11 +802,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.4.4 #TODO remove after image update
run: |
rm -rf /opt/shims/spark34
@@ -913,11 +850,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -971,11 +903,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1025,11 +952,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update
run: |
rm -rf /opt/shims/spark35
@@ -1076,11 +998,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1129,11 +1046,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update
run: |
rm -rf /opt/shims/spark35
@@ -1296,11 +1208,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1354,11 +1261,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image update
run: |
rm -rf /opt/shims/spark40
@@ -1405,11 +1307,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -1465,11 +1362,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image update
run: |
rm -rf /opt/shims/spark41
diff --git a/dev/build-arrow.sh b/dev/build-arrow.sh
index 1bf4ef9d089..007047a8ad2 100755
--- a/dev/build-arrow.sh
+++ b/dev/build-arrow.sh
@@ -75,6 +75,21 @@ function build_arrow_cpp() {
}
function build_arrow_java() {
+ # Maven Central's arrow-c-data / arrow-dataset jars at ${VELOX_ARROW_BUILD_VERSION}
+ # already ship libarrow_cdata_jni / libarrow_dataset_jni for x86_64 (Linux/macOS/Windows)
+ # and aarch_64 (Linux/macOS), so contributors on those archs do not need a locally-built
+ # jar — gluten-arrow resolves the same artifact transitively.
+ #
+ # ppc64le has no native in the Central jar; support_ibm_power.patch (applied above) adds
+ # the ppc64le -> ppcle_64 arch case to JniLoader.java and the local mvn install step bakes
+ # a locally-built libarrow_cdata_jni.so for ppc64le into the resulting arrow-c-data jar in
+ # ~/.m2, overriding Central. Skip the Java build on every other arch.
+ local ARCH=$(uname -m)
+ if [[ "${ARCH}" != "ppc64le" ]]; then
+ echo "Skipping local Arrow Java build on ${ARCH} — gluten resolves arrow-c-data:${VELOX_ARROW_BUILD_VERSION} from Maven Central. Local build is only required on ppc64le for the patched JniLoader."
+ return 0
+ fi
+
ARROW_INSTALL_DIR="${ARROW_PREFIX}/install"
# Use Gluten's Maven wrapper
@@ -96,8 +111,6 @@ function build_arrow_java() {
export CMAKE_BUILD_PARALLEL_LEVEL=$NPROC
pushd $ARROW_PREFIX/java
- # Because arrow-bom module need the -DprocessAllModules
- ${MVN_CMD} versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules
${MVN_CMD} clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \
-DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly
diff --git a/gluten-arrow/pom.xml b/gluten-arrow/pom.xml
index 62d9bd243a7..defcfcbf26f 100644
--- a/gluten-arrow/pom.xml
+++ b/gluten-arrow/pom.xml
@@ -88,13 +88,13 @@
org.apache.arrow
${arrow-memory.artifact}
- ${arrow-gluten.version}
+ ${arrow.version}
runtime
org.apache.arrow
arrow-memory-core
- ${arrow-gluten.version}
+ ${arrow.version}
compile
@@ -110,7 +110,7 @@
org.apache.arrow
arrow-vector
- ${arrow-gluten.version}
+ ${arrow.version}
io.netty
@@ -129,7 +129,7 @@
org.apache.arrow
arrow-c-data
- ${arrow-gluten.version}
+ ${arrow.version}
compile
@@ -145,7 +145,7 @@
org.apache.arrow
arrow-dataset
- ${arrow-gluten.version}
+ ${arrow.version}
compile
diff --git a/pom.xml b/pom.xml
index 1b3a9d2a3cc..fc4dbb273fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,6 @@
0.6.3
0.10.0
15.0.0
- 15.0.0-gluten
arrow-memory-unsafe
2.7.4
4.9.3
@@ -1287,7 +1286,6 @@
2.24.3
3.17.0
18.1.0
- 18.1.0
4.9.2
@@ -1367,7 +1365,6 @@
2.24.3
3.17.0
18.1.0
- 18.1.0
4.9.5