[FEDE-8281] Upgrade Arrow Java to 19.0.0

README.md

@@ -17,7 +17,73 @@
   under the License.
 -->
 
-# Arrow Java
+# Siren fork of Arrow Java
+
+The properties `drill.enable_unsafe_memory_access` and
+`arrow.enable_unsafe_memory_access` are prefixed with `siren` and their
+default value is set to `true`. The first property is deprecated.
+
+## Check that Arrow uses Unsafe class to access off-heap memory for memory allocation
+In order to check that Arrow uses Unsafe class for memory allocation, run the unit test `CheckArrowTest` in
+`https://github.com/sirensolutions/siren-platform/blob/master/core/src/test/java/io/siren/federate/core/common/CheckArrowTest.java`.
+
+## Build
+
+To build the `memory`, `format`, `vector` and `algorithm` modules:
+
+```sh
+$ cd java
+$ mvn clean package
+```
+
+Because of the default value change of `unsafe_memory_access` property, some
+tests in `vector` fail.
+
+```sh
+mvn -pl memory,memory/memory-core,memory/memory-unsafe,format,vector,algorithm install -Dsiren.arrow.enable_unsafe_memory_access=false -Dsiren.drill.enable_unsafe_memory_access=false
+```
+
+## Make a new release of Siren's Apache Arrow
+
+- Tests should pass.
+
+- Make a new version:
+
+```sh
+mvn versions:set -DnewVersion=siren-0.14.1-2
+```
+
+- tag the commit for the release
+
+```sh
+git tag --sign siren-0.14.1-2
+````
+
+- Deploy to Siren's Google Artifact Registry:
+```sh
+# Deploy all modules (format, memory, vector, algorithm)
+$ mvn -pl memory,memory/memory-core,memory/memory-unsafe,format,vector,algorithm deploy \
+  -Dsiren.arrow.enable_unsafe_memory_access=false \
+  -Dsiren.drill.enable_unsafe_memory_access=false \
+  -DskipTests \
+  -DaltDeploymentRepository=gar-maven-local-siren-snapshot::default::artifactregistry://europe-west1-maven.pkg.dev/siren-cicd/maven-local-siren-snapshot
+
+# Deploy the parent POM
+$ mvn deploy:deploy-file \
+  -Durl=artifactregistry://europe-west1-maven.pkg.dev/siren-cicd/maven-local-siren-snapshot \
+  -DpomFile=pom.xml -Dfile=pom.xml \
+  -DgroupId=org.apache.arrow \
+  -DartifactId=arrow-java-root \
+  -Dversion=siren-19.0.0-1-SNAPSHOT \
+  -Dpackaging=pom
+```
+## Update to a new version of Siren's Apache Arrow
+Developer tips on updating to a new version of Arrow can be found here: https://sirensolutions.atlassian.net/wiki/spaces/EN/pages/3108864001/Upgrading+Federate+Apache+Arrow+Version .
+
+- add `git@github.com:apache/arrow-java.git` as the `upstream` remote.
+- execute `git fetch --all --tags`
+- create a temporary branch from `siren-changes`
+- rebase against the new tag.
 
 ## Getting Started
 

algorithm/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-algorithm</artifactId>
   <name>Arrow Algorithms</name>
@@ -32,7 +32,8 @@ under the License.
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-vector</artifactId>
-      <classifier>${arrow.vector.classifier}</classifier>
+      <version>${project.version}</version>
+      <classifier>shade</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
@@ -41,18 +42,10 @@ under the License.
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-netty</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.immutables</groupId>
       <artifactId>value-annotations</artifactId>
     </dependency>
   </dependencies>
+
 </project>

algorithm/src/main/java/module-info.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-module org.apache.arrow.algorithm {
+module arrow.algorithm.siren {
   exports org.apache.arrow.algorithm.search;
   exports org.apache.arrow.algorithm.deduplicate;
   exports org.apache.arrow.algorithm.dictionary;
@@ -24,6 +24,5 @@
   exports org.apache.arrow.algorithm.sort;
 
   requires jdk.unsupported;
-  requires org.apache.arrow.memory.core;
-  requires org.apache.arrow.vector;
+  requires arrow.vector.siren;
 }

format/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <artifactId>arrow-format</artifactId>

gandiva/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <groupId>org.apache.arrow.gandiva</groupId>
@@ -40,6 +40,7 @@ under the License.
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
+      <version>${project.version</version>
     </dependency>
     <dependency>
       <groupId>org.immutables</groupId>
@@ -66,6 +67,7 @@ under the License.
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
     </dependency>
   </dependencies>
 

memory/memory-core/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-memory</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <artifactId>arrow-memory-core</artifactId>
@@ -39,6 +39,7 @@ under the License.
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.immutables</groupId>

memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java

@@ -31,16 +31,18 @@ public class BoundsChecking {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
 
   static {
-    String envProperty = System.getenv("ARROW_ENABLE_UNSAFE_MEMORY_ACCESS");
-    String oldProperty = System.getProperty("drill.enable_unsafe_memory_access");
+    String envProperty =
+        System.getenv().getOrDefault("SIREN_ARROW_ENABLE_UNSAFE_MEMORY_ACCESS", "true");
+    String oldProperty = System.getProperty("siren.drill.enable_unsafe_memory_access", "true");
     if (oldProperty != null) {
       logger.warn(
-          "\"drill.enable_unsafe_memory_access\" has been renamed to \"arrow.enable_unsafe_memory_access\"");
+          "\"siren.drill.enable_unsafe_memory_access\" has been renamed to "
+              + "\"siren.arrow.enable_unsafe_memory_access\"");
       logger.warn(
-          "\"arrow.enable_unsafe_memory_access\" can be set to: "
-              + " true (to not check) or false (to check, default)");
+          "\"siren.arrow.enable_unsafe_memory_access\" can be set to: "
+              + " true (to not check, default) or false (to check)");
     }
-    String newProperty = System.getProperty("arrow.enable_unsafe_memory_access");
+    String newProperty = System.getProperty("siren.arrow.enable_unsafe_memory_access", "true");
 
     // The priority of determining the unsafe flag:
     // 1. The system properties take precedence over the environmental variable.

memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java

@@ -34,10 +34,10 @@ public class MemoryUtil {
   private static final @Nullable Constructor<?> DIRECT_BUFFER_CONSTRUCTOR;
 
   /** The unsafe object from which to access the off-heap memory. */
-  private static final Unsafe UNSAFE;
+  public static final Unsafe UNSAFE;
 
   /** The start offset of array data relative to the start address of the array object. */
-  private static final long BYTE_ARRAY_BASE_OFFSET;
+  public static final long BYTE_ARRAY_BASE_OFFSET;
 
   /** The offset of the address field with the {@link java.nio.ByteBuffer} object. */
   private static final long BYTE_BUFFER_ADDRESS_OFFSET;
@@ -82,7 +82,6 @@ public Object run() {
 
       // get the offset of the address field in a java.nio.Buffer object
       Field addressField = java.nio.Buffer.class.getDeclaredField("address");
-      addressField.setAccessible(true);
       BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField);
 
       Constructor<?> directBufferConstructor;
@@ -103,10 +102,7 @@ public Object run() {
                       constructor.setAccessible(true);
                       logger.debug("Constructor for direct buffer found and made accessible");
                       return constructor;
-                    } catch (NoSuchMethodException e) {
-                      logger.debug("Cannot get constructor for direct buffer allocation", e);
-                      return e;
-                    } catch (SecurityException e) {
+                    } catch (Exception e) {
                       logger.debug("Cannot get constructor for direct buffer allocation", e);
                       return e;
                     }

memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java

@@ -17,7 +17,6 @@
 package org.apache.arrow.memory;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.lang.reflect.Field;
 import java.net.URLClassLoader;
@@ -55,15 +54,15 @@ private boolean getFlagValue(ClassLoader classLoader) throws Exception {
   }
 
   /**
-   * Ensure the flag for bounds checking is enabled by default. This will protect users from JVM
-   * crashes.
+   * Siren: Ensure the flag for bounds checking is disabled by default. Enabling it will protect
+   * users from JVM crashes.
    */
   @Test
   public void testDefaultValue() throws Exception {
     ClassLoader classLoader = copyClassLoader();
     if (classLoader != null) {
       boolean boundsCheckingEnabled = getFlagValue(classLoader);
-      assertTrue(boundsCheckingEnabled);
+      assertFalse(boundsCheckingEnabled);
     }
   }
 

memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java

@@ -26,7 +26,7 @@
 public class TestOpens {
   /** Instantiating the RootAllocator should poke MemoryUtil and fail. */
   @Test
-  @EnabledForJreRange(min = JAVA_16)
+  @EnabledForJreRange(max = JAVA_16)
   public void testMemoryUtilFailsLoudly() {
     // This test is configured by Maven to run WITHOUT add-opens. So this should fail on JDK16+
     // (where JEP396 means that add-opens is required to access JDK internals).

memory/memory-netty-buffer-patch/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-memory</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <artifactId>arrow-memory-netty-buffer-patch</artifactId>
@@ -33,6 +33,7 @@ under the License.
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>

memory/memory-netty/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-memory</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <artifactId>arrow-memory-netty</artifactId>
@@ -33,6 +33,7 @@ under the License.
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

memory/memory-unsafe/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-memory</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
 
   <artifactId>arrow-memory-unsafe</artifactId>
@@ -33,6 +33,7 @@ under the License.
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.immutables</groupId>

memory/pom.xml

@@ -22,7 +22,7 @@ under the License.
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>19.0.0-SNAPSHOT</version>
+    <version>siren-19.0.0-1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <packaging>pom</packaging>