[CORE] Unbundle Arrow memory + vector from gluten-velox-bundle (Draft)

backends-velox/pom.xml

@@ -79,6 +79,24 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <!--
+      arrow-memory-core / arrow-vector are scope=provided in gluten-arrow so
+      they are not bundled. Re-declare them at provided scope here so the
+      compile classpath still resolves them. At runtime they come from
+      Spark's distribution.
+    -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-core</artifactId>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-vector</artifactId>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>com.github.ben-manes.caffeine</groupId>
       <artifactId>caffeine</artifactId>

dev/build-arrow.sh

@@ -31,7 +31,6 @@ function prepare_arrow_build() {
   #wget_and_untar https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz arrow_ep
   cd arrow_ep
   patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow.patch
-  patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow_dataset_scan_option.patch
   patch -p1 < $CURRENT_DIR/../ep/build-velox/src/cmake-compatibility.patch
   patch -p1 < $CURRENT_DIR/../ep/build-velox/src/support_ibm_power.patch
   popd
@@ -97,8 +96,6 @@ function build_arrow_java() {
     export CMAKE_BUILD_PARALLEL_LEVEL=$NPROC
 
     pushd $ARROW_PREFIX/java
-    # Because arrow-bom module need the -DprocessAllModules
-    ${MVN_CMD} versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules
 
     ${MVN_CMD} clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \
           -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly

gluten-arrow/pom.xml

@@ -85,17 +85,24 @@
       <version>${spark.version}</version>
       <scope>provided</scope>
     </dependency>
+    <!--
+      Arrow memory + vector come from Spark's distribution (declared in Spark
+      4.x's pom; bundled in $SPARK_HOME/jars/ for Spark 3.x). gluten compiles
+      against the same arrow.version it expects at runtime, but the artifacts
+      are scope=provided so they are NOT bundled into gluten-velox-bundle —
+      avoiding the #12225 class-shadowing problem entirely.
+    -->
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>${arrow-memory.artifact}</artifactId>
-      <version>${arrow-gluten.version}</version>
-      <scope>runtime</scope>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
-      <version>${arrow-gluten.version}</version>
-      <scope>compile</scope>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
       <exclusions>
         <exclusion>
           <groupId>io.netty</groupId>
@@ -110,7 +117,8 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-vector</artifactId>
-      <version>${arrow-gluten.version}</version>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
       <exclusions>
         <exclusion>
           <groupId>io.netty</groupId>
@@ -129,7 +137,7 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-c-data</artifactId>
-      <version>${arrow-gluten.version}</version>
+      <version>${arrow.version}</version>
       <scope>compile</scope>
       <exclusions>
         <exclusion>
@@ -145,7 +153,7 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-dataset</artifactId>
-      <version>${arrow-gluten.version}</version>
+      <version>${arrow.version}</version>
       <scope>compile</scope>
       <exclusions>
         <exclusion>

package/pom.xml

@@ -118,28 +118,15 @@
                     <include>com.google.gson.**</include>
                   </includes>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.arrow</pattern>
-                  <shadedPattern>${gluten.shade.packageName}.org.apache.arrow</shadedPattern>
-                  <!--
-                    arrow's C and dataset wrappers refer to the original class
-                    path, so they must not be relocated. Their public APIs also
-                    take and return org.apache.arrow.memory.* and
-                    org.apache.arrow.vector.* types, so those packages must also
-                    stay unshaded — otherwise the bundled (unshaded)
-                    ArrowArrayStream/ArrowSchema get compiled against the
-                    relocated BufferAllocator/VectorSchemaRoot, producing
-                    `NoSuchMethodError` for any caller passing a vanilla
-                    Apache Arrow allocator. See #12225.
-                  -->
-                  <excludes>
-                    <exclude>org.apache.arrow.c.*</exclude>
-                    <exclude>org.apache.arrow.c.jni.*</exclude>
-                    <exclude>org.apache.arrow.memory.**</exclude>
-                    <exclude>org.apache.arrow.vector.**</exclude>
-                    <exclude>org.apache.arrow.dataset.**</exclude>
-                  </excludes>
-                </relocation>
+                <!--
+                  org.apache.arrow.memory.* and org.apache.arrow.vector.* are
+                  now scope=provided in gluten-arrow/pom.xml — they come from
+                  Spark's distribution at runtime, so there is nothing to
+                  relocate. arrow-c-data and arrow-dataset are still bundled
+                  but never relocated (their JNI binds to the original class
+                  names), so no shade-relocation entry is needed for Arrow.
+                  See #12225 / #12226 for the historical context.
+                -->
                 <relocation>
                   <pattern>com.google.flatbuffers</pattern>
                   <shadedPattern>${gluten.shade.packageName}.com.google.flatbuffers</shadedPattern>

pom.xml

@@ -81,7 +81,6 @@
     <celeborn.version>0.6.3</celeborn.version>
     <uniffle.version>0.10.0</uniffle.version>
     <arrow.version>15.0.0</arrow.version>
-    <arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>
     <arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
     <hadoop.version>2.7.4</hadoop.version>
     <antlr4.version>4.9.3</antlr4.version>
@@ -1285,7 +1284,6 @@
         <log4j.version>2.24.3</log4j.version>
         <commons-lang3.version>3.17.0</commons-lang3.version>
         <arrow.version>18.1.0</arrow.version>
-        <arrow-gluten.version>18.1.0</arrow-gluten.version>
         <scala.compiler.version>4.9.2</scala.compiler.version>
       </properties>
       <dependencies>
@@ -1365,7 +1363,6 @@
         <log4j.version>2.24.3</log4j.version>
         <commons-lang3.version>3.17.0</commons-lang3.version>
         <arrow.version>18.1.0</arrow.version>
-        <arrow-gluten.version>18.1.0</arrow-gluten.version>
         <scala.compiler.version>4.9.5</scala.compiler.version>
       </properties>
       <dependencies>