From 89fd0097cb58453aba3616202f4e54b5edc7bfcf Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 28 Jan 2025 13:30:32 -0800 Subject: [PATCH 01/19] Update main ver to 6.1.0-SNAPSHOT --- pom.xml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index ab4016d8..089a245f 100644 --- a/pom.xml +++ b/pom.xml @@ -25,16 +25,9 @@ under the License. 4.0.0 - - org.apache.datasketches datasketches-memory - 5.1.0-SNAPSHOT + 6.1.0-SNAPSHOT jar ${project.artifactId} From adb0f2efa8ca81457d6315799396a4a4d3f098e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Wiejacha?= Date: Thu, 30 Jan 2025 10:49:17 +0100 Subject: [PATCH 02/19] fixed operator precedence bug in setPosition() calls --- .../internal/NativeWritableBufferImpl.java | 24 +++++++++---------- .../internal/NonNativeWritableBufferImpl.java | 24 +++++++++---------- .../NativeWritableBufferImplTest.java | 16 +++++++++++++ .../NonNativeWritableBufferImplTest.java | 15 ++++++++++++ 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java index 588f0d65..31b2f640 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java @@ -89,7 +89,7 @@ public char getChar(final long offsetBytes) { public void getCharArray(final char[] dstArray, final int dstOffsetChars, final int lengthChars) { final long pos = getPosition(); getCharArr(seg, pos, dstArray, dstOffsetChars, lengthChars); - setPosition(pos + (lengthChars) << CHAR_SHIFT); + setPosition(pos + (lengthChars << CHAR_SHIFT)); } @Override @@ -108,7 +108,7 @@ public double getDouble(final long offsetBytes) { public void getDoubleArray(final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); getDoubleArr(seg, pos, dstArray, dstOffsetDoubles, lengthDoubles); - setPosition(pos + (lengthDoubles) << DOUBLE_SHIFT); + setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @Override @@ -127,7 +127,7 @@ public float getFloat(final long offsetBytes) { public void getFloatArray(final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { final long pos = getPosition(); getFloatArr(seg, pos, dstArray, dstOffsetFloats, lengthFloats); - setPosition(pos + (lengthFloats) << FLOAT_SHIFT); + setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @Override @@ -146,7 +146,7 @@ public int getInt(final long offsetBytes) { public void getIntArray(final int[] dstArray, final int dstOffsetInts, final int lengthInts) { final long pos = getPosition(); getIntArr(seg, pos, dstArray, dstOffsetInts, lengthInts); - setPosition(pos + (lengthInts) << INT_SHIFT); + setPosition(pos + (lengthInts << INT_SHIFT)); } @Override @@ -165,7 +165,7 @@ public long getLong(final long offsetBytes) { public void getLongArray(final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { final long pos = getPosition(); getLongArr(seg, pos, dstArray, dstOffsetLongs, lengthLongs); - setPosition(pos + (lengthLongs) << LONG_SHIFT); + setPosition(pos + (lengthLongs << LONG_SHIFT)); } @Override @@ -184,7 +184,7 @@ public short getShort(final long offsetBytes) { public void getShortArray(final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { final long pos = getPosition(); getShortArr(seg, pos, dstArray, dstOffsetShorts, lengthShorts); - setPosition(pos + (lengthShorts) << SHORT_SHIFT); + setPosition(pos + (lengthShorts << SHORT_SHIFT)); } //PRIMITIVE putX() and putXArray() @@ -204,7 +204,7 @@ public void putChar(final long offsetBytes, final char value) { public void putCharArray(final char[] srcArray, final int srcOffsetChars, final int lengthChars) { final long pos = getPosition(); putCharArr(seg, pos, srcArray, srcOffsetChars, lengthChars); - setPosition(pos + (lengthChars) << CHAR_SHIFT); + setPosition(pos + (lengthChars << CHAR_SHIFT)); } @Override @@ -223,7 +223,7 @@ public void putDouble(final long offsetBytes, final double value) { public void putDoubleArray(final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); putDoubleArr(seg, pos, srcArray, srcOffsetDoubles, lengthDoubles); - setPosition(pos + (lengthDoubles) << DOUBLE_SHIFT); + setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @Override @@ -242,7 +242,7 @@ public void putFloat(final long offsetBytes, final float value) { public void putFloatArray(final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { final long pos = getPosition(); putFloatArr(seg, pos, srcArray, srcOffsetFloats, lengthFloats); - setPosition(pos + (lengthFloats) << FLOAT_SHIFT); + setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @Override @@ -261,7 +261,7 @@ public void putInt(final long offsetBytes, final int value) { public void putIntArray(final int[] srcArray, final int srcOffsetInts, final int lengthInts) { final long pos = getPosition(); putIntArr(seg, pos, srcArray, srcOffsetInts, lengthInts); - setPosition(pos + (lengthInts) << INT_SHIFT); + setPosition(pos + (lengthInts << INT_SHIFT)); } @Override @@ -280,7 +280,7 @@ public void putLong(final long offsetBytes, final long value) { public void putLongArray(final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { final long pos = getPosition(); putLongArr(seg, pos, srcArray, srcOffsetLongs, lengthLongs); - setPosition(pos + (lengthLongs) << LONG_SHIFT); + setPosition(pos + (lengthLongs << LONG_SHIFT)); } @Override @@ -299,6 +299,6 @@ public void putShort(final long offsetBytes, final short value) { public void putShortArray(final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { final long pos = getPosition(); putShortArr(seg, pos, srcArray, srcOffsetShorts, lengthShorts); - setPosition(pos + (lengthShorts) << SHORT_SHIFT); + setPosition(pos + (lengthShorts << SHORT_SHIFT)); } } diff --git a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java index 84cf3491..29c5b1cc 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java @@ -90,7 +90,7 @@ public char getChar(final long offsetBytes) { public void getCharArray(final char[] dstArray, final int dstOffsetChars, final int lengthChars) { final long pos = getPosition(); getCharArr(seg, pos, dstArray, dstOffsetChars, lengthChars); - setPosition(pos + (lengthChars) << CHAR_SHIFT); + setPosition(pos + (lengthChars << CHAR_SHIFT)); } @Override @@ -109,7 +109,7 @@ public double getDouble(final long offsetBytes) { public void getDoubleArray(final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); getDoubleArr(seg, pos, dstArray, dstOffsetDoubles, lengthDoubles); - setPosition(pos + (lengthDoubles) << DOUBLE_SHIFT); + setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @Override @@ -128,7 +128,7 @@ public float getFloat(final long offsetBytes) { public void getFloatArray(final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { final long pos = getPosition(); getFloatArr(seg, pos, dstArray, dstOffsetFloats, lengthFloats); - setPosition(pos + (lengthFloats) << FLOAT_SHIFT); + setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @Override @@ -147,7 +147,7 @@ public int getInt(final long offsetBytes) { public void getIntArray(final int[] dstArray, final int dstOffsetInts, final int lengthInts) { final long pos = getPosition(); getIntArr(seg, pos, dstArray, dstOffsetInts, lengthInts); - setPosition(pos + (lengthInts) << INT_SHIFT); + setPosition(pos + (lengthInts << INT_SHIFT)); } @Override @@ -166,7 +166,7 @@ public long getLong(final long offsetBytes) { public void getLongArray(final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { final long pos = getPosition(); getLongArr(seg, pos, dstArray, dstOffsetLongs, lengthLongs); - setPosition(pos + (lengthLongs) << LONG_SHIFT); + setPosition(pos + (lengthLongs << LONG_SHIFT)); } @Override @@ -185,7 +185,7 @@ public short getShort(final long offsetBytes) { public void getShortArray(final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { final long pos = getPosition(); getShortArr(seg, pos, dstArray, dstOffsetShorts, lengthShorts); - setPosition(pos + (lengthShorts) << SHORT_SHIFT); + setPosition(pos + (lengthShorts << SHORT_SHIFT)); } //PRIMITIVE putX() and putXArray() @@ -205,7 +205,7 @@ public void putChar(final long offsetBytes, final char value) { public void putCharArray(final char[] srcArray, final int srcOffsetChars, final int lengthChars) { final long pos = getPosition(); putCharArr(seg, pos, srcArray, srcOffsetChars, lengthChars); - setPosition(pos + (lengthChars) << CHAR_SHIFT); + setPosition(pos + (lengthChars << CHAR_SHIFT)); } @Override @@ -224,7 +224,7 @@ public void putDouble(final long offsetBytes, final double value) { public void putDoubleArray(final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); putDoubleArr(seg, pos, srcArray, srcOffsetDoubles, lengthDoubles); - setPosition(pos + (lengthDoubles) << DOUBLE_SHIFT); + setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @Override @@ -243,7 +243,7 @@ public void putFloat(final long offsetBytes, final float value) { public void putFloatArray(final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { final long pos = getPosition(); putFloatArr(seg, pos, srcArray, srcOffsetFloats, lengthFloats); - setPosition(pos + (lengthFloats) << FLOAT_SHIFT); + setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @Override @@ -262,7 +262,7 @@ public void putInt(final long offsetBytes, final int value) { public void putIntArray(final int[] srcArray, final int srcOffsetInts, final int lengthInts) { final long pos = getPosition(); putIntArr(seg, pos, srcArray, srcOffsetInts, lengthInts); - setPosition(pos + (lengthInts) << INT_SHIFT); + setPosition(pos + (lengthInts << INT_SHIFT)); } @Override @@ -281,7 +281,7 @@ public void putLong(final long offsetBytes, final long value) { public void putLongArray(final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { final long pos = getPosition(); putLongArr(seg, pos, srcArray, srcOffsetLongs, lengthLongs); - setPosition(pos + (lengthLongs) << LONG_SHIFT); + setPosition(pos + (lengthLongs << LONG_SHIFT)); } @Override @@ -300,7 +300,7 @@ public void putShort(final long offsetBytes, final short value) { public void putShortArray(final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { final long pos = getPosition(); putShortArr(seg, pos, srcArray, srcOffsetShorts, lengthShorts); - setPosition(pos + (lengthShorts) << SHORT_SHIFT); + setPosition(pos + (lengthShorts << SHORT_SHIFT)); } } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java index 1a0a29f3..a746ebc8 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java @@ -468,6 +468,22 @@ public void checkDuplicateNonNative() { assertEquals(buf.getShort(0), 256); } + @Test + public void checkPutIntArray() { + WritableMemory wmem = WritableMemory.allocate(12); + WritableBuffer wbuf = wmem.asWritableBuffer(); + + wbuf.putInt(1); + int[] array = new int[] { 2 }; + wbuf.putIntArray(array, 0, 1); + wbuf.putInt(3); + + Buffer buf = wmem.asWritableBuffer(); + assertEquals(buf.getInt(), 1); + assertEquals(buf.getInt(), 2); + assertEquals(buf.getInt(), 3); + } + @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java index 155409bc..1de0c4cd 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java @@ -273,7 +273,22 @@ public void checkConversionByteOrder() { Memory mreg = mem.region(0, 8); assertEquals(mreg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); assertEquals(mreg.getChar(0), 1); + } + + @Test + public void checkPutIntArray() { + WritableMemory wmem = WritableMemory.allocate(12, ByteOrder.BIG_ENDIAN); + WritableBuffer wbuf = wmem.asWritableBuffer(ByteOrder.BIG_ENDIAN); + + wbuf.putInt(1); + int[] array = new int[] { 2 }; + wbuf.putIntArray(array, 0, 1); + wbuf.putInt(3); + Buffer buf = wmem.asWritableBuffer(); + assertEquals(buf.getInt(), 1); + assertEquals(buf.getInt(), 2); + assertEquals(buf.getInt(), 3); } } From b7b7437d025155fd7454aa590e0ac5e22fea0c5e Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 31 Jan 2025 15:43:21 -0800 Subject: [PATCH 03/19] Fixed all the tests which lacked the ability to test this bug. At the same time, I refactored the leaf classes to use the new APIs in Java 21 FFM, this should have been done before. Nonetheless, this eliminated a lot of unnecessary code. --- .../internal/NativeWritableBufferImpl.java | 51 +-- .../internal/NativeWritableMemoryImpl.java | 115 +----- .../internal/NonNativeWritableBufferImpl.java | 49 +-- .../internal/NonNativeWritableMemoryImpl.java | 117 +------ .../memory/internal/WritableBufferImpl.java | 13 - .../memory/internal/WritableMemoryImpl.java | 5 +- .../NativeWritableBufferImplTest.java | 263 +++++++++----- .../NativeWritableMemoryImplTest.java | 239 +++++++++---- .../NonNativeWritableBufferImplTest.java | 330 ++++++++---------- .../NonNativeWritableMemoryImplTest.java | 229 ++++++------ 10 files changed, 667 insertions(+), 744 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java index 31b2f640..27b77514 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java @@ -25,18 +25,6 @@ import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getCharArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getDoubleArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getFloatArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getIntArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getLongArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.getShortArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putCharArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putDoubleArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putFloatArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putIntArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putLongArr; -import static org.apache.datasketches.memory.internal.NativeWritableMemoryImpl.putShortArr; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -44,19 +32,6 @@ import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.WritableBuffer; -/* - * Developer notes: The heavier methods, such as put/get arrays, duplicate, region, clear, fill, - * compareTo, etc., use hard checks (check*() and incrementAndCheck*() methods), which execute at - * runtime and throw exceptions if violated. The cost of the runtime checks are minor compared to - * the rest of the work these methods are doing. - * - *

The light weight methods, such as put/get primitives, use asserts (assert*() and - * incrementAndAssert*() methods), which only execute when asserts are enabled and JIT will remove - * them entirely from production runtime code. The offset versions of the light weight methods will - * simplify to a single unsafe call, which is further simplified by JIT to an intrinsic that is - * often a single CPU instruction. - */ - /** * Implementation of {@link WritableBuffer} for native endian byte order. * @author Roman Leventov @@ -64,6 +39,7 @@ */ final class NativeWritableBufferImpl extends WritableBufferImpl { + //Pass-through ctor NativeWritableBufferImpl( final Arena arena, final MemorySegment seg, @@ -88,7 +64,7 @@ public char getChar(final long offsetBytes) { @Override public void getCharArray(final char[] dstArray, final int dstOffsetChars, final int lengthChars) { final long pos = getPosition(); - getCharArr(seg, pos, dstArray, dstOffsetChars, lengthChars); + MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED, pos, dstArray, dstOffsetChars, lengthChars); setPosition(pos + (lengthChars << CHAR_SHIFT)); } @@ -107,7 +83,7 @@ public double getDouble(final long offsetBytes) { @Override public void getDoubleArray(final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); - getDoubleArr(seg, pos, dstArray, dstOffsetDoubles, lengthDoubles); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED, pos, dstArray, dstOffsetDoubles, lengthDoubles); setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @@ -126,7 +102,7 @@ public float getFloat(final long offsetBytes) { @Override public void getFloatArray(final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { final long pos = getPosition(); - getFloatArr(seg, pos, dstArray, dstOffsetFloats, lengthFloats); + MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED, pos, dstArray, dstOffsetFloats, lengthFloats); setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @@ -145,7 +121,7 @@ public int getInt(final long offsetBytes) { @Override public void getIntArray(final int[] dstArray, final int dstOffsetInts, final int lengthInts) { final long pos = getPosition(); - getIntArr(seg, pos, dstArray, dstOffsetInts, lengthInts); + MemorySegment.copy(seg, JAVA_INT_UNALIGNED, pos, dstArray, dstOffsetInts, lengthInts); setPosition(pos + (lengthInts << INT_SHIFT)); } @@ -164,7 +140,7 @@ public long getLong(final long offsetBytes) { @Override public void getLongArray(final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { final long pos = getPosition(); - getLongArr(seg, pos, dstArray, dstOffsetLongs, lengthLongs); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, pos, dstArray, dstOffsetLongs, lengthLongs); setPosition(pos + (lengthLongs << LONG_SHIFT)); } @@ -183,7 +159,7 @@ public short getShort(final long offsetBytes) { @Override public void getShortArray(final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { final long pos = getPosition(); - getShortArr(seg, pos, dstArray, dstOffsetShorts, lengthShorts); + MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED, pos, dstArray, dstOffsetShorts, lengthShorts); setPosition(pos + (lengthShorts << SHORT_SHIFT)); } @@ -203,7 +179,7 @@ public void putChar(final long offsetBytes, final char value) { @Override public void putCharArray(final char[] srcArray, final int srcOffsetChars, final int lengthChars) { final long pos = getPosition(); - putCharArr(seg, pos, srcArray, srcOffsetChars, lengthChars); + MemorySegment.copy(srcArray, srcOffsetChars, seg, JAVA_CHAR_UNALIGNED, pos, lengthChars); setPosition(pos + (lengthChars << CHAR_SHIFT)); } @@ -222,7 +198,7 @@ public void putDouble(final long offsetBytes, final double value) { @Override public void putDoubleArray(final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); - putDoubleArr(seg, pos, srcArray, srcOffsetDoubles, lengthDoubles); + MemorySegment.copy(srcArray, srcOffsetDoubles, seg, JAVA_DOUBLE_UNALIGNED, pos, lengthDoubles); setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @@ -241,7 +217,7 @@ public void putFloat(final long offsetBytes, final float value) { @Override public void putFloatArray(final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { final long pos = getPosition(); - putFloatArr(seg, pos, srcArray, srcOffsetFloats, lengthFloats); + MemorySegment.copy(srcArray, srcOffsetFloats, seg, JAVA_FLOAT_UNALIGNED, pos, lengthFloats); setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @@ -260,7 +236,7 @@ public void putInt(final long offsetBytes, final int value) { @Override public void putIntArray(final int[] srcArray, final int srcOffsetInts, final int lengthInts) { final long pos = getPosition(); - putIntArr(seg, pos, srcArray, srcOffsetInts, lengthInts); + MemorySegment.copy(srcArray, srcOffsetInts, seg, JAVA_INT_UNALIGNED, pos, lengthInts); setPosition(pos + (lengthInts << INT_SHIFT)); } @@ -279,7 +255,7 @@ public void putLong(final long offsetBytes, final long value) { @Override public void putLongArray(final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { final long pos = getPosition(); - putLongArr(seg, pos, srcArray, srcOffsetLongs, lengthLongs); + MemorySegment.copy(srcArray, srcOffsetLongs, seg, JAVA_LONG_UNALIGNED, pos, lengthLongs); setPosition(pos + (lengthLongs << LONG_SHIFT)); } @@ -298,7 +274,8 @@ public void putShort(final long offsetBytes, final short value) { @Override public void putShortArray(final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { final long pos = getPosition(); - putShortArr(seg, pos, srcArray, srcOffsetShorts, lengthShorts); + MemorySegment.copy(srcArray, srcOffsetShorts, seg, JAVA_SHORT_UNALIGNED, pos, lengthShorts); setPosition(pos + (lengthShorts << SHORT_SHIFT)); } + } diff --git a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImpl.java index 4c5222ae..57649a33 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImpl.java @@ -19,17 +19,11 @@ package org.apache.datasketches.memory.internal; -import static java.lang.foreign.ValueLayout.JAVA_CHAR; import static java.lang.foreign.ValueLayout.JAVA_CHAR_UNALIGNED; -import static java.lang.foreign.ValueLayout.JAVA_DOUBLE; import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; -import static java.lang.foreign.ValueLayout.JAVA_FLOAT; import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; -import static java.lang.foreign.ValueLayout.JAVA_INT; import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; -import static java.lang.foreign.ValueLayout.JAVA_LONG; import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; -import static java.lang.foreign.ValueLayout.JAVA_SHORT; import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import java.lang.foreign.Arena; @@ -55,7 +49,7 @@ final class NativeWritableMemoryImpl extends WritableMemoryImpl { super(seg, typeId, memReqSvr, arena); } - ///PRIMITIVE getX() and getXArray() + //PRIMITIVE getX() and getXArray() @Override public char getChar(final long offsetBytes) { return seg.get(ValueLayout.JAVA_CHAR_UNALIGNED, offsetBytes); @@ -63,14 +57,7 @@ public char getChar(final long offsetBytes) { @Override public void getCharArray(final long offsetBytes, final char[] dstArray, final int dstOffsetChars, final int lengthChars) { - getCharArr(seg, offsetBytes, dstArray, dstOffsetChars, lengthChars); - } - - static void getCharArr( - final MemorySegment seg, final long offsetBytes, final char[] dstArray, final int dstOffsetChars, final int lengthChars) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetChars) << CHAR_SHIFT; - MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED, offsetBytes, dstSeg, JAVA_CHAR, dstOffsetBytes, lengthChars); + MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED, offsetBytes, dstArray, dstOffsetChars, lengthChars); } @Override @@ -80,14 +67,7 @@ public double getDouble(final long offsetBytes) { @Override public void getDoubleArray(final long offsetBytes, final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { - getDoubleArr(seg, offsetBytes, dstArray, dstOffsetDoubles, lengthDoubles); - } - - static void getDoubleArr( - final MemorySegment seg, final long offsetBytes, final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetDoubles) << DOUBLE_SHIFT; - MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED, offsetBytes, dstSeg, JAVA_DOUBLE, dstOffsetBytes, lengthDoubles); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED, offsetBytes, dstArray, dstOffsetDoubles, lengthDoubles); } @Override @@ -97,14 +77,7 @@ public float getFloat(final long offsetBytes) { @Override public void getFloatArray(final long offsetBytes, final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { - getFloatArr(seg, offsetBytes, dstArray, dstOffsetFloats, lengthFloats); - } - - static void getFloatArr( - final MemorySegment seg, final long offsetBytes, final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetFloats) << FLOAT_SHIFT; - MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED, offsetBytes, dstSeg, JAVA_FLOAT, dstOffsetBytes, lengthFloats); + MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED, offsetBytes, dstArray, dstOffsetFloats, lengthFloats); } @Override @@ -114,14 +87,7 @@ public int getInt(final long offsetBytes) { @Override public void getIntArray(final long offsetBytes, final int[] dstArray, final int dstOffsetInts, final int lengthInts) { - getIntArr(seg, offsetBytes, dstArray, dstOffsetInts, lengthInts); - } - - static void getIntArr( - final MemorySegment seg, final long offsetBytes, final int[] dstArray, final int dstOffsetInts, final int lengthInts) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetInts) << INT_SHIFT; - MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offsetBytes, dstSeg, JAVA_INT, dstOffsetBytes, lengthInts); + MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offsetBytes, dstArray, dstOffsetInts, lengthInts); } @Override @@ -131,14 +97,7 @@ public long getLong(final long offsetBytes) { @Override public void getLongArray(final long offsetBytes, final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { - getLongArr(seg, offsetBytes, dstArray, dstOffsetLongs, lengthLongs); - } - - static void getLongArr( - final MemorySegment seg, final long offsetBytes, final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetLongs) << LONG_SHIFT; - MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, offsetBytes, dstSeg, JAVA_LONG, dstOffsetBytes, lengthLongs); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, offsetBytes, dstArray, dstOffsetLongs, lengthLongs); } @Override @@ -148,14 +107,7 @@ public short getShort(final long offsetBytes) { @Override public void getShortArray(final long offsetBytes, final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { - getShortArr(seg, offsetBytes, dstArray, dstOffsetShorts, lengthShorts); - } - - static void getShortArr( - final MemorySegment seg, final long offsetBytes, final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetShorts) << SHORT_SHIFT; - MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED, offsetBytes, dstSeg, JAVA_SHORT, dstOffsetBytes, lengthShorts); + MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED, offsetBytes, dstArray, dstOffsetShorts, lengthShorts); } //PRIMITIVE putX() and putXArray() implementations @@ -166,14 +118,7 @@ public void putChar(final long offsetBytes, final char value) { @Override public void putCharArray(final long offsetBytes, final char[] srcArray, final int srcOffsetChars, final int lengthChars) { - putCharArr(seg, offsetBytes, srcArray, srcOffsetChars, lengthChars); - } - - static void putCharArr( - final MemorySegment seg, final long offsetBytes, final char[] srcArray, final int srcOffsetChars, final int lengthChars) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetChars) << CHAR_SHIFT; - MemorySegment.copy(srcSeg, JAVA_CHAR, srcOffsetBytes, seg, JAVA_CHAR_UNALIGNED, offsetBytes, lengthChars); + MemorySegment.copy(srcArray, srcOffsetChars, seg, JAVA_CHAR_UNALIGNED, offsetBytes, lengthChars); } @Override @@ -183,14 +128,7 @@ public void putDouble(final long offsetBytes, final double value) { @Override public void putDoubleArray(final long offsetBytes, final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { - putDoubleArr(seg, offsetBytes, srcArray, srcOffsetDoubles, lengthDoubles); - } - - static void putDoubleArr( - final MemorySegment seg, final long offsetBytes, final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetDoubles) << DOUBLE_SHIFT; - MemorySegment.copy(srcSeg, JAVA_DOUBLE, srcOffsetBytes, seg, JAVA_DOUBLE_UNALIGNED, offsetBytes, lengthDoubles); + MemorySegment.copy(srcArray, srcOffsetDoubles, seg, JAVA_DOUBLE_UNALIGNED, offsetBytes, lengthDoubles); } @Override @@ -200,14 +138,7 @@ public void putFloat(final long offsetBytes, final float value) { @Override public void putFloatArray(final long offsetBytes, final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { - putFloatArr(seg, offsetBytes, srcArray, srcOffsetFloats, lengthFloats); - } - - static void putFloatArr( - final MemorySegment seg, final long offsetBytes, final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetFloats) << FLOAT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_FLOAT, srcOffsetBytes, seg, JAVA_FLOAT_UNALIGNED, offsetBytes, lengthFloats); + MemorySegment.copy(srcArray, srcOffsetFloats, seg, JAVA_FLOAT_UNALIGNED, offsetBytes, lengthFloats); } @Override @@ -217,14 +148,7 @@ public void putInt(final long offsetBytes, final int value) { @Override public void putIntArray(final long offsetBytes, final int[] srcArray, final int srcOffsetInts, final int lengthInts) { - putIntArr(seg, offsetBytes, srcArray, srcOffsetInts, lengthInts); - } - - static void putIntArr( - final MemorySegment seg, final long offsetBytes, final int[] srcArray, final int srcOffsetInts, final int lengthInts) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetInts) << INT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_INT, srcOffsetBytes, seg, JAVA_INT_UNALIGNED, offsetBytes, lengthInts); + MemorySegment.copy(srcArray, srcOffsetInts, seg, JAVA_INT_UNALIGNED, offsetBytes, lengthInts); } @Override @@ -234,14 +158,7 @@ public void putLong(final long offsetBytes, final long value) { @Override public void putLongArray(final long offsetBytes, final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { - putLongArr(seg, offsetBytes, srcArray, srcOffsetLongs, lengthLongs); - } - - static void putLongArr( - final MemorySegment seg, final long offsetBytes, final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetLongs) << LONG_SHIFT; - MemorySegment.copy(srcSeg, JAVA_LONG, srcOffsetBytes, seg, JAVA_LONG_UNALIGNED, offsetBytes, lengthLongs); + MemorySegment.copy(srcArray, srcOffsetLongs, seg, JAVA_LONG_UNALIGNED, offsetBytes, lengthLongs); } @Override @@ -251,13 +168,7 @@ public void putShort(final long offsetBytes, final short value) { @Override public void putShortArray(final long offsetBytes, final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { - putShortArr(seg, offsetBytes, srcArray, srcOffsetShorts, lengthShorts); + MemorySegment.copy(srcArray, srcOffsetShorts, seg, JAVA_SHORT_UNALIGNED, offsetBytes, lengthShorts); } - static void putShortArr( - final MemorySegment seg, final long offsetBytes, final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetShorts) << SHORT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_SHORT, srcOffsetBytes, seg, JAVA_SHORT_UNALIGNED, offsetBytes, lengthShorts); - } } diff --git a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java index 29c5b1cc..a527d3a4 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImpl.java @@ -25,18 +25,6 @@ import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_INT_UNALIGNED_NON_NATIVE; import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_LONG_UNALIGNED_NON_NATIVE; import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_SHORT_UNALIGNED_NON_NATIVE; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getCharArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getDoubleArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getFloatArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getIntArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getLongArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.getShortArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putCharArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putDoubleArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putFloatArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putIntArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putLongArr; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImpl.putShortArr; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -44,19 +32,6 @@ import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.WritableBuffer; -/* - * Developer notes: The heavier methods, such as put/get arrays, duplicate, region, clear, fill, - * compareTo, etc., use hard checks (check*() and incrementAndCheck*() methods), which execute at - * runtime and throw exceptions if violated. The cost of the runtime checks are minor compared to - * the rest of the work these methods are doing. - * - *

The light weight methods, such as put/get primitives, use asserts (assert*() and - * incrementAndAssert*() methods), which only execute when asserts are enabled and JIT will remove - * them entirely from production runtime code. The offset versions of the light weight methods will - * simplify to a single unsafe call, which is further simplified by JIT to an intrinsic that is - * often a single CPU instruction. - */ - /** * Implementation of {@link WritableBuffer} for non-native endian byte order. * @author Roman Leventov @@ -89,7 +64,7 @@ public char getChar(final long offsetBytes) { @Override public void getCharArray(final char[] dstArray, final int dstOffsetChars, final int lengthChars) { final long pos = getPosition(); - getCharArr(seg, pos, dstArray, dstOffsetChars, lengthChars); + MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetChars, lengthChars); setPosition(pos + (lengthChars << CHAR_SHIFT)); } @@ -108,7 +83,7 @@ public double getDouble(final long offsetBytes) { @Override public void getDoubleArray(final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); - getDoubleArr(seg, pos, dstArray, dstOffsetDoubles, lengthDoubles); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetDoubles, lengthDoubles); setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @@ -127,7 +102,7 @@ public float getFloat(final long offsetBytes) { @Override public void getFloatArray(final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { final long pos = getPosition(); - getFloatArr(seg, pos, dstArray, dstOffsetFloats, lengthFloats); + MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetFloats, lengthFloats); setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @@ -146,7 +121,7 @@ public int getInt(final long offsetBytes) { @Override public void getIntArray(final int[] dstArray, final int dstOffsetInts, final int lengthInts) { final long pos = getPosition(); - getIntArr(seg, pos, dstArray, dstOffsetInts, lengthInts); + MemorySegment.copy(seg, JAVA_INT_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetInts, lengthInts); setPosition(pos + (lengthInts << INT_SHIFT)); } @@ -165,7 +140,7 @@ public long getLong(final long offsetBytes) { @Override public void getLongArray(final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { final long pos = getPosition(); - getLongArr(seg, pos, dstArray, dstOffsetLongs, lengthLongs); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetLongs, lengthLongs); setPosition(pos + (lengthLongs << LONG_SHIFT)); } @@ -184,7 +159,7 @@ public short getShort(final long offsetBytes) { @Override public void getShortArray(final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { final long pos = getPosition(); - getShortArr(seg, pos, dstArray, dstOffsetShorts, lengthShorts); + MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, pos, dstArray, dstOffsetShorts, lengthShorts); setPosition(pos + (lengthShorts << SHORT_SHIFT)); } @@ -204,7 +179,7 @@ public void putChar(final long offsetBytes, final char value) { @Override public void putCharArray(final char[] srcArray, final int srcOffsetChars, final int lengthChars) { final long pos = getPosition(); - putCharArr(seg, pos, srcArray, srcOffsetChars, lengthChars); + MemorySegment.copy(srcArray, srcOffsetChars, seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, pos, lengthChars); setPosition(pos + (lengthChars << CHAR_SHIFT)); } @@ -223,7 +198,7 @@ public void putDouble(final long offsetBytes, final double value) { @Override public void putDoubleArray(final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { final long pos = getPosition(); - putDoubleArr(seg, pos, srcArray, srcOffsetDoubles, lengthDoubles); + MemorySegment.copy(srcArray, srcOffsetDoubles, seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, pos, lengthDoubles); setPosition(pos + (lengthDoubles << DOUBLE_SHIFT)); } @@ -242,7 +217,7 @@ public void putFloat(final long offsetBytes, final float value) { @Override public void putFloatArray(final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { final long pos = getPosition(); - putFloatArr(seg, pos, srcArray, srcOffsetFloats, lengthFloats); + MemorySegment.copy(srcArray, srcOffsetFloats, seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, pos, lengthFloats); setPosition(pos + (lengthFloats << FLOAT_SHIFT)); } @@ -261,7 +236,7 @@ public void putInt(final long offsetBytes, final int value) { @Override public void putIntArray(final int[] srcArray, final int srcOffsetInts, final int lengthInts) { final long pos = getPosition(); - putIntArr(seg, pos, srcArray, srcOffsetInts, lengthInts); + MemorySegment.copy(srcArray, srcOffsetInts, seg, JAVA_INT_UNALIGNED_NON_NATIVE, pos, lengthInts); setPosition(pos + (lengthInts << INT_SHIFT)); } @@ -280,7 +255,7 @@ public void putLong(final long offsetBytes, final long value) { @Override public void putLongArray(final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { final long pos = getPosition(); - putLongArr(seg, pos, srcArray, srcOffsetLongs, lengthLongs); + MemorySegment.copy(srcArray, srcOffsetLongs, seg, JAVA_LONG_UNALIGNED_NON_NATIVE, pos, lengthLongs); setPosition(pos + (lengthLongs << LONG_SHIFT)); } @@ -299,7 +274,7 @@ public void putShort(final long offsetBytes, final short value) { @Override public void putShortArray(final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { final long pos = getPosition(); - putShortArr(seg, pos, srcArray, srcOffsetShorts, lengthShorts); + MemorySegment.copy(srcArray, srcOffsetShorts, seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, pos, lengthShorts); setPosition(pos + (lengthShorts << SHORT_SHIFT)); } diff --git a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImpl.java index 5744ffd0..68c90997 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImpl.java @@ -19,12 +19,6 @@ package org.apache.datasketches.memory.internal; -import static java.lang.foreign.ValueLayout.JAVA_CHAR; -import static java.lang.foreign.ValueLayout.JAVA_DOUBLE; -import static java.lang.foreign.ValueLayout.JAVA_FLOAT; -import static java.lang.foreign.ValueLayout.JAVA_INT; -import static java.lang.foreign.ValueLayout.JAVA_LONG; -import static java.lang.foreign.ValueLayout.JAVA_SHORT; import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_CHAR_UNALIGNED_NON_NATIVE; import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_DOUBLE_UNALIGNED_NON_NATIVE; import static org.apache.datasketches.memory.internal.NonNativeValueLayouts.JAVA_FLOAT_UNALIGNED_NON_NATIVE; @@ -37,6 +31,7 @@ import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.WritableMemory; +// /** * Implementation of {@link WritableMemory} for non-native endian byte order. @@ -54,7 +49,7 @@ final class NonNativeWritableMemoryImpl extends WritableMemoryImpl { super(seg, typeId, memReqSvr, arena); } - ///PRIMITIVE getX() and getXArray() + //PRIMITIVE getX() and getXArray() @Override public char getChar(final long offsetBytes) { return seg.get(JAVA_CHAR_UNALIGNED_NON_NATIVE, offsetBytes); @@ -62,14 +57,7 @@ public char getChar(final long offsetBytes) { @Override public void getCharArray(final long offsetBytes, final char[] dstArray, final int dstOffsetChars, final int lengthChars) { - getCharArr(seg, offsetBytes, dstArray, dstOffsetChars, lengthChars); - } - - static void getCharArr( - final MemorySegment seg, final long offsetBytes, final char[] dstArray, final int dstOffsetChars, final int lengthChars) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetChars) << CHAR_SHIFT; - MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_CHAR, dstOffsetBytes, lengthChars); + MemorySegment.copy(seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetChars, lengthChars); } @Override @@ -79,14 +67,7 @@ public double getDouble(final long offsetBytes) { @Override public void getDoubleArray(final long offsetBytes, final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { - getDoubleArr(seg, offsetBytes, dstArray, dstOffsetDoubles, lengthDoubles); - } - - static void getDoubleArr( - final MemorySegment seg, final long offsetBytes, final double[] dstArray, final int dstOffsetDoubles, final int lengthDoubles) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetDoubles) << DOUBLE_SHIFT; - MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_DOUBLE, dstOffsetBytes, lengthDoubles); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetDoubles, lengthDoubles); } @Override @@ -96,14 +77,7 @@ public float getFloat(final long offsetBytes) { @Override public void getFloatArray(final long offsetBytes, final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { - getFloatArr(seg, offsetBytes, dstArray, dstOffsetFloats, lengthFloats); - } - - static void getFloatArr( - final MemorySegment seg, final long offsetBytes, final float[] dstArray, final int dstOffsetFloats, final int lengthFloats) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetFloats) << FLOAT_SHIFT; - MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_FLOAT, dstOffsetBytes, lengthFloats); + MemorySegment.copy(seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetFloats, lengthFloats); } @Override @@ -113,14 +87,7 @@ public int getInt(final long offsetBytes) { @Override public void getIntArray(final long offsetBytes, final int[] dstArray, final int dstOffsetInts, final int lengthInts) { - getIntArr(seg, offsetBytes, dstArray, dstOffsetInts, lengthInts); - } - - static void getIntArr( - final MemorySegment seg, final long offsetBytes, final int[] dstArray, final int dstOffsetInts, final int lengthInts) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetInts) << INT_SHIFT; - MemorySegment.copy(seg, JAVA_INT_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_INT, dstOffsetBytes, lengthInts); + MemorySegment.copy(seg, JAVA_INT_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetInts, lengthInts); } @Override @@ -130,14 +97,7 @@ public long getLong(final long offsetBytes) { @Override public void getLongArray(final long offsetBytes, final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { - getLongArr(seg, offsetBytes, dstArray, dstOffsetLongs, lengthLongs); - } - - static void getLongArr( - final MemorySegment seg, final long offsetBytes, final long[] dstArray, final int dstOffsetLongs, final int lengthLongs) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetLongs) << LONG_SHIFT; - MemorySegment.copy(seg, JAVA_LONG_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_LONG, dstOffsetBytes, lengthLongs); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetLongs, lengthLongs); } @Override @@ -147,14 +107,7 @@ public short getShort(final long offsetBytes) { @Override public void getShortArray(final long offsetBytes, final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { - getShortArr(seg, offsetBytes, dstArray, dstOffsetShorts, lengthShorts); - } - - static void getShortArr( - final MemorySegment seg, final long offsetBytes, final short[] dstArray, final int dstOffsetShorts, final int lengthShorts) { - final MemorySegment dstSeg = MemorySegment.ofArray(dstArray); - final long dstOffsetBytes = ((long) dstOffsetShorts) << SHORT_SHIFT; - MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, offsetBytes, dstSeg, JAVA_SHORT, dstOffsetBytes, lengthShorts); + MemorySegment.copy(seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, offsetBytes, dstArray, dstOffsetShorts, lengthShorts); } //PRIMITIVE putX() and putXArray() implementations @@ -165,14 +118,7 @@ public void putChar(final long offsetBytes, final char value) { @Override public void putCharArray(final long offsetBytes, final char[] srcArray, final int srcOffsetChars, final int lengthChars) { - putCharArr(seg, offsetBytes, srcArray, srcOffsetChars, lengthChars); - } - - static void putCharArr( - final MemorySegment seg, final long offsetBytes, final char[] srcArray, final int srcOffsetChars, final int lengthChars) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetChars) << CHAR_SHIFT; - MemorySegment.copy(srcSeg, JAVA_CHAR, srcOffsetBytes, seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, offsetBytes, lengthChars); + MemorySegment.copy(srcArray, srcOffsetChars, seg, JAVA_CHAR_UNALIGNED_NON_NATIVE, offsetBytes, lengthChars); } @Override @@ -182,14 +128,7 @@ public void putDouble(final long offsetBytes, final double value) { @Override public void putDoubleArray(final long offsetBytes, final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { - putDoubleArr(seg, offsetBytes, srcArray, srcOffsetDoubles, lengthDoubles); - } - - static void putDoubleArr( - final MemorySegment seg, final long offsetBytes, final double[] srcArray, final int srcOffsetDoubles, final int lengthDoubles) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetDoubles) << DOUBLE_SHIFT; - MemorySegment.copy(srcSeg, JAVA_DOUBLE, srcOffsetBytes, seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, offsetBytes, lengthDoubles); + MemorySegment.copy(srcArray, srcOffsetDoubles, seg, JAVA_DOUBLE_UNALIGNED_NON_NATIVE, offsetBytes, lengthDoubles); } @Override @@ -199,14 +138,7 @@ public void putFloat(final long offsetBytes, final float value) { @Override public void putFloatArray(final long offsetBytes, final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { - putFloatArr(seg, offsetBytes, srcArray, srcOffsetFloats, lengthFloats); - } - - static void putFloatArr( - final MemorySegment seg, final long offsetBytes, final float[] srcArray, final int srcOffsetFloats, final int lengthFloats) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetFloats) << FLOAT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_FLOAT, srcOffsetBytes, seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, offsetBytes, lengthFloats); + MemorySegment.copy(srcArray, srcOffsetFloats, seg, JAVA_FLOAT_UNALIGNED_NON_NATIVE, offsetBytes, lengthFloats); } @Override @@ -216,14 +148,7 @@ public void putInt(final long offsetBytes, final int value) { @Override public void putIntArray(final long offsetBytes, final int[] srcArray, final int srcOffsetInts, final int lengthInts) { - putIntArr(seg, offsetBytes, srcArray, srcOffsetInts, lengthInts); - } - - static void putIntArr( - final MemorySegment seg, final long offsetBytes, final int[] srcArray, final int srcOffsetInts, final int lengthInts) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetInts) << INT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_INT, srcOffsetBytes, seg, JAVA_INT_UNALIGNED_NON_NATIVE, offsetBytes, lengthInts); + MemorySegment.copy(srcArray, srcOffsetInts, seg, JAVA_INT_UNALIGNED_NON_NATIVE, offsetBytes, lengthInts); } @Override @@ -233,14 +158,7 @@ public void putLong(final long offsetBytes, final long value) { @Override public void putLongArray(final long offsetBytes, final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { - putLongArr(seg, offsetBytes, srcArray, srcOffsetLongs, lengthLongs); - } - - static void putLongArr( - final MemorySegment seg, final long offsetBytes, final long[] srcArray, final int srcOffsetLongs, final int lengthLongs) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetLongs) << LONG_SHIFT; - MemorySegment.copy(srcSeg, JAVA_LONG, srcOffsetBytes, seg, JAVA_LONG_UNALIGNED_NON_NATIVE, offsetBytes, lengthLongs); + MemorySegment.copy(srcArray, srcOffsetLongs, seg, JAVA_LONG_UNALIGNED_NON_NATIVE, offsetBytes, lengthLongs); } @Override @@ -250,14 +168,7 @@ public void putShort(final long offsetBytes, final short value) { @Override public void putShortArray(final long offsetBytes, final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { - putShortArr(seg, offsetBytes, srcArray, srcOffsetShorts, lengthShorts); - } - - static void putShortArr( - final MemorySegment seg, final long offsetBytes, final short[] srcArray, final int srcOffsetShorts, final int lengthShorts) { - final MemorySegment srcSeg = MemorySegment.ofArray(srcArray); - final long srcOffsetBytes = ((long) srcOffsetShorts) << SHORT_SHIFT; - MemorySegment.copy(srcSeg, JAVA_SHORT, srcOffsetBytes, seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, offsetBytes, lengthShorts); + MemorySegment.copy(srcArray, srcOffsetShorts, seg, JAVA_SHORT_UNALIGNED_NON_NATIVE, offsetBytes, lengthShorts); } } diff --git a/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java index 5818cd92..b2e057f2 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java @@ -32,19 +32,6 @@ import org.apache.datasketches.memory.WritableBuffer; import org.apache.datasketches.memory.WritableMemory; -/* - * Developer notes: The heavier methods, such as put/get arrays, duplicate, region, clear, fill, - * compareTo, etc., use hard checks (check*() and incrementAndCheck*() methods), which execute at - * runtime and throw exceptions if violated. The cost of the runtime checks are minor compared to - * the rest of the work these methods are doing. - * - *

The light weight methods, such as put/get primitives, use asserts (assert*() and - * incrementAndAssert*() methods), which only execute when asserts are enabled and JIT will remove - * them entirely from production runtime code. The offset versions of the light weight methods will - * simplify to a single unsafe call, which is further simplified by JIT to an intrinsic that is - * often a single CPU instruction. - */ - /** * Common base of native-ordered and non-native-ordered {@link WritableBuffer} implementations. * Contains methods which are agnostic to the byte order. diff --git a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java index a3299115..4604e092 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java @@ -46,7 +46,6 @@ * Common base of native-ordered and non-native-ordered {@link WritableMemory} implementations. * Contains methods which are agnostic to the byte order. */ -@SuppressWarnings("preview") public abstract class WritableMemoryImpl extends ResourceImpl implements WritableMemory { //Pass-through constructor @@ -147,7 +146,7 @@ public static WritableMemory wrapByteBuffer( * @throws IllegalArgumentException if file is not readable. * @throws IOException if mapping is not successful. */ - @SuppressWarnings("resource") + @SuppressWarnings({"resource","preview"}) public static WritableMemory wrapMap( final File file, final long fileOffsetBytes, @@ -339,7 +338,7 @@ public final void writeToByteStream(final long offsetBytes, final int lengthByte out.writeBytes(bArr); } - // //PRIMITIVE putX() and putXArray() implementations + //PRIMITIVE putX() and putXArray() implementations @Override public final void putBoolean(final long offsetBytes, final boolean value) { diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java index a746ebc8..c1fbc36a 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java @@ -58,129 +58,236 @@ public void checkNativeCapacityAndClose() throws Exception { //Simple Heap arrays @Test - public void checkByteArray() { + public void checkGetByteArray() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - byte[] dstArray = new byte[8]; + final int len = srcArray.length; + final int half = len / 2; + byte[] dstArray = new byte[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getByteArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getByteArray(dstArray, 0, half); + buf.getByteArray(dstArray, half, half); + assertEquals(dstArray, srcArray); WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getByteArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getByteArray(dstArray, 0, half); + wbuf.getByteArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutByteArray() { + byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + byte[] dstArray = new byte[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Byte.BYTES).asWritableBuffer(); + wbuf.putByteArray(srcArray, 0, half); + wbuf.putByteArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getByteArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkCharArray() { + public void checkGetCharArray() { char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; - char[] dstArray = new char[8]; + final int len = srcArray.length; + final int half = len / 2; + char[] dstArray = new char[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getCharArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getCharArray(dstArray, 0, half); + buf.getCharArray(dstArray, half, half); + assertEquals(dstArray, srcArray); WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getCharArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getCharArray(dstArray, 0, half); + wbuf.getCharArray(dstArray, half, half); + assertEquals(dstArray, srcArray); } @Test - public void checkShortArray() { + public void checkPutCharArray() { + char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + char[] dstArray = new char[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES).asWritableBuffer(); + wbuf.putCharArray(srcArray, 0, half); + wbuf.putCharArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getCharArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkGetShortArray() { short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - short[] dstArray = new short[8]; + final int len = srcArray.length; + final int half = len / 2; + short[] dstArray = new short[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getShortArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getShortArray(dstArray, 0, half); + buf.getShortArray(dstArray, half, half); + assertEquals(dstArray, srcArray); WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getShortArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getShortArray(dstArray, 0, half); + wbuf.getShortArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutShortArray() { + short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + short[] dstArray = new short[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES).asWritableBuffer(); + wbuf.putShortArray(srcArray, 0, half); + wbuf.putShortArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getShortArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkIntArray() { + public void checkGetIntArray() { int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - int[] dstArray = new int[8]; + final int len = srcArray.length; + final int half = len / 2; + int[] dstArray = new int[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getIntArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getIntArray(dstArray, 0, half); + buf.getIntArray(dstArray, half, half); + assertEquals(dstArray, srcArray); WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getIntArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getIntArray(dstArray, 0, half); + wbuf.getIntArray(dstArray, half, half); + assertEquals(dstArray, srcArray); } @Test - public void checkLongArray() { + public void checkPutIntArray() { + int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + int[] dstArray = new int[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES).asWritableBuffer(); + wbuf.putIntArray(srcArray, 0, half); + wbuf.putIntArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getIntArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkGetLongArray() { long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - long[] dstArray = new long[8]; + final int len = srcArray.length; + final int half = len / 2; + long[] dstArray = new long[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getLongArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getLongArray(dstArray, 0, half); + buf.getLongArray(dstArray, half, half); + assertEquals(dstArray, srcArray); WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getLongArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getLongArray(dstArray, 0, half); + wbuf.getLongArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutLongArray() { + long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + long[] dstArray = new long[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES).asWritableBuffer(); + wbuf.putLongArray(srcArray, 0, half); + wbuf.putLongArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getLongArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkFloatArray() { + public void checkGetFloatArray() { float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - float[] dstArray = new float[8]; + final int len = srcArray.length; + final int half = len / 2; + float[] dstArray = new float[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getFloatArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getFloatArray(dstArray, 0, half); + buf.getFloatArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getFloatArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getFloatArray(dstArray, 0, half); + wbuf.getFloatArray(dstArray, half, half); + assertEquals(dstArray, srcArray); } @Test - public void checkDoubleArray() { + public void checkPutFloatArray() { + float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + float[] dstArray = new float[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES).asWritableBuffer(); + wbuf.putFloatArray(srcArray, 0, half); + wbuf.putFloatArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getFloatArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkGetDoubleArray() { double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - double[] dstArray = new double[8]; + final int len = srcArray.length; + final int half = len / 2; + double[] dstArray = new double[len]; Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getDoubleArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + buf.getDoubleArray(dstArray, 0, half); + buf.getDoubleArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getDoubleArray(dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wbuf.getDoubleArray(dstArray, 0, half); + wbuf.getDoubleArray(dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutDoubleArray() { + double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + double[] dstArray = new double[len]; + + WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES).asWritableBuffer(); + wbuf.putDoubleArray(srcArray, 0, half); + wbuf.putDoubleArray(srcArray, half, half); + wbuf.resetPosition(); + wbuf.getDoubleArray(dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test @@ -468,22 +575,6 @@ public void checkDuplicateNonNative() { assertEquals(buf.getShort(0), 256); } - @Test - public void checkPutIntArray() { - WritableMemory wmem = WritableMemory.allocate(12); - WritableBuffer wbuf = wmem.asWritableBuffer(); - - wbuf.putInt(1); - int[] array = new int[] { 2 }; - wbuf.putIntArray(array, 0, 1); - wbuf.putInt(3); - - Buffer buf = wmem.asWritableBuffer(); - assertEquals(buf.getInt(), 1); - assertEquals(buf.getInt(), 2); - assertEquals(buf.getInt(), 3); - } - @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java index 7a410b74..c58c6691 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java @@ -58,132 +58,231 @@ public void checkNativeCapacityAndClose() throws Exception { try { wmem.getArena().close(); } catch (IllegalStateException e) { } } - //Simple Native arrays + //Simple Heap arrays @Test - public void checkByteArray() { + public void checkGetByteArray() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - byte[] dstArray = new byte[8]; + final int len = srcArray.length; + final int half = len / 2; + byte[] dstArray = new byte[len]; Memory mem = Memory.wrap(srcArray); - mem.getByteArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getByteArray(0, dstArray, 0, half); + mem.getByteArray(half, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getByteArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getByteArray(0, dstArray, 0, half); + wmem.getByteArray(half, dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutByteArray() { + byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + byte[] dstArray = new byte[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Byte.BYTES); + wmem.putByteArray(0, srcArray, 0, half); + wmem.putByteArray(half * Byte.BYTES, srcArray, half, half); + wmem.getByteArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkCharArray() { + public void checkGetCharArray() { char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; - char[] dstArray = new char[8]; + final int len = srcArray.length; + final int half = len / 2; + char[] dstArray = new char[len]; Memory mem = Memory.wrap(srcArray); - mem.getCharArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getCharArray(0, dstArray, 0, half); + mem.getCharArray(half * Character.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getCharArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getCharArray(0, dstArray, 0, half); + wmem.getCharArray(half * Character.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); } @Test - public void checkShortArray() { + public void checkPutCharArray() { + char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + char[] dstArray = new char[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES); + wmem.putCharArray(0, srcArray, 0, half); + wmem.putCharArray(half * Character.BYTES, srcArray, half, half); + wmem.getCharArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkGetShortArray() { short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - short[] dstArray = new short[8]; + final int len = srcArray.length; + final int half = len / 2; + short[] dstArray = new short[len]; Memory mem = Memory.wrap(srcArray); - mem.getShortArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getShortArray(0, dstArray, 0, half); + mem.getShortArray(half * Short.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getShortArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getShortArray(0, dstArray, 0, half); + wmem.getShortArray(half * Short.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutShortArray() { + short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + short[] dstArray = new short[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES); + wmem.putShortArray(0, srcArray, 0, half); + wmem.putShortArray(half * Short.BYTES, srcArray, half, half); + wmem.getShortArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkIntArray() { + public void checkGetIntArray() { int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - int[] dstArray = new int[8]; + final int len = srcArray.length; + final int half = len / 2; + int[] dstArray = new int[len]; Memory mem = Memory.wrap(srcArray); - mem.getIntArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getIntArray(0, dstArray, 0, half); + mem.getIntArray(half * Integer.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getIntArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getIntArray(0, dstArray, 0, half); + wmem.getIntArray(half * Integer.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); } @Test - public void checkLongArray() { + public void checkPutIntArray() { + int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + int[] dstArray = new int[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES); + wmem.putIntArray(0, srcArray, 0, half); + wmem.putIntArray(half * Integer.BYTES, srcArray, half, half); + wmem.getIntArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkGetLongArray() { long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - long[] dstArray = new long[8]; + final int len = srcArray.length; + final int half = len / 2; + long[] dstArray = new long[len]; Memory mem = Memory.wrap(srcArray); - mem.getLongArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getLongArray(0, dstArray, 0, half); + mem.getLongArray(half * Long.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getLongArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getLongArray(0, dstArray, 0, half); + wmem.getLongArray(half * Long.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutLongArray() { + long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + long[] dstArray = new long[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES); + wmem.putLongArray(0, srcArray, 0, half); + wmem.putLongArray(half * Long.BYTES, srcArray, half, half); + wmem.getLongArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); } + @Test - public void checkFloatArray() { + public void checkGetFloatArray() { float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - float[] dstArray = new float[8]; + final int len = srcArray.length; + final int half = len / 2; + float[] dstArray = new float[len]; Memory mem = Memory.wrap(srcArray); - mem.getFloatArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getFloatArray(0, dstArray, 0, half); + mem.getFloatArray(half * Float.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getFloatArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getFloatArray(0, dstArray, 0, half); + wmem.getFloatArray(half * Float.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutFloatArray() { + float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + float[] dstArray = new float[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES); + wmem.putFloatArray(0, srcArray, 0, half); + wmem.putFloatArray(half * Float.BYTES, srcArray, half, half); + wmem.getFloatArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test - public void checkDoubleArray() { + public void checkGetDoubleArray() { double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; double[] dstArray = new double[8]; Memory mem = Memory.wrap(srcArray); - mem.getDoubleArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + mem.getDoubleArray(0, dstArray, 0, half); + mem.getDoubleArray(half * Double.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getDoubleArray(0, dstArray, 0, 8); - for (int i=0; i<8; i++) { - assertEquals(dstArray[i], srcArray[i]); - } + wmem.getDoubleArray(0, dstArray, 0, half); + wmem.getDoubleArray(half * Double.BYTES, dstArray, half, half); + assertEquals(dstArray, srcArray); + } + + @Test + public void checkPutDoubleArray() { + double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + final int len = srcArray.length; + final int half = len / 2; + double[] dstArray = new double[len]; + + WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES); + wmem.putDoubleArray(0, srcArray, 0, half); + wmem.putDoubleArray(half * Double.BYTES, srcArray, half, half); + wmem.getDoubleArray(0, dstArray, 0, len); + assertEquals(dstArray, srcArray); } @Test diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java index 1de0c4cd..8e598666 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java @@ -19,7 +19,12 @@ package org.apache.datasketches.memory.internal; +import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImplTest.doubleReverseBytes; +import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImplTest.floatReverseBytes; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import java.nio.ByteOrder; @@ -35,250 +40,203 @@ public class NonNativeWritableBufferImplTest { //Check primitives + @Test - public void checkCharacters() { - int n = 8; - int m = Character.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - char ch = 'a'; - for (int i = 0; i < n; i++) { wbuf.putChar(i * m, ch++); } - ch = 'a'; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getChar(i * m), ch++); + public void checkPutGetNonNativeCharacters() { + char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putCharArray(srcArray, 0, half); + wbuf.putCharArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Character.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Character.reverseBytes(wbuf2.getChar())); } - ch = 'a'; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putChar(ch++); } - ch = 'a'; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getChar(), ch++); - } - //getArr & putArr - char[] cArr = new char[n]; //native - wbuf.setPosition(0); - wbuf.getCharArray(cArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putCharArray(cArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + char[] dstArray = new char[len]; + wbuf.getCharArray(dstArray, 0, half); + wbuf.getCharArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkDoubles() { - int n = 8; - int m = Double.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - double dbl = 1.0; - for (int i = 0; i < n; i++) { wbuf.putDouble(i * m, dbl++); } - dbl = 1.0; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getDouble(i * m), dbl++); + public void checkPutGetNonNativeDoubles() { + double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putDoubleArray(srcArray, 0, half); + wbuf.putDoubleArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Double.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == doubleReverseBytes(wbuf2.getDouble())); } - dbl = 1.0; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putDouble(dbl++); } - dbl = 1.0; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getDouble(), dbl++); - } - //getArr & putArr - double[] dblArr = new double[n]; //native - wbuf.setPosition(0); - wbuf.getDoubleArray(dblArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putDoubleArray(dblArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + double[] dstArray = new double[len]; + wbuf.getDoubleArray(dstArray, 0, half); + wbuf.getDoubleArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkFloats() { - int n = 8; - int m = Float.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - float flt = 1.0F; - for (int i = 0; i < n; i++) { wbuf.putFloat(i * m, flt++); } - flt = 1.0F; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getFloat(i * m), flt++); - } - flt = 1.0F; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putFloat(flt++); } - flt = 1.0F; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getFloat(), flt++); + public void checkPutGetNonNativeFloats() { + float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putFloatArray(srcArray, 0, half); + wbuf.putFloatArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Float.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == floatReverseBytes(wbuf2.getFloat())); } - //getArr & putArr - float[] fltArr = new float[n]; //native - wbuf.setPosition(0); - wbuf.getFloatArray(fltArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putFloatArray(fltArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + float[] dstArray = new float[len]; + wbuf.getFloatArray(dstArray, 0, half); + wbuf.getFloatArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkInts() { - int n = 8; - int m = Integer.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - int intg = 1; - for (int i = 0; i < n; i++) { wbuf.putInt(i * m, intg++); } - intg = 1; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getInt(i * m), intg++); - } - intg = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putInt(intg++); } - intg = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getInt(), intg++); + public void checkPutGetNonNativeInts() { + int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putIntArray(srcArray, 0, half); + wbuf.putIntArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Integer.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Integer.reverseBytes(wbuf2.getInt())); } - //getArr & putArr - int[] intArr = new int[n]; //native - wbuf.setPosition(0); - wbuf.getIntArray(intArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putIntArray(intArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + int[] dstArray = new int[len]; + wbuf.getIntArray(dstArray, 0, half); + wbuf.getIntArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkLongs() { - int n = 8; - int m = Long.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - long lng = 1; - for (int i = 0; i < n; i++) { wbuf.putLong(i * m, lng++); } - lng = 1; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getLong(i * m), lng++); + public void checkPutGetNonNativeLongs() { + long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putLongArray(srcArray, 0, half); + wbuf.putLongArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Long.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Long.reverseBytes(wbuf2.getLong())); } - lng = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putLong(lng++); } - lng = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getLong(), lng++); - } - //getArr & putArr - long[] longArr = new long[n]; //native - wbuf.setPosition(0); - wbuf.getLongArray(longArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putLongArray(longArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + long[] dstArray = new long[len]; + wbuf.getLongArray(dstArray, 0, half); + wbuf.getLongArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkShorts() { - int n = 8; - int m = Short.BYTES; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(); - short sht = 1; - for (int i = 0; i < n; i++) { wbuf.putShort(i * m, sht++); } - sht = 1; - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getShort(i * m), sht++); - } - sht = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { wbuf.putShort(sht++); } - sht = 1; - wbuf.setPosition(0); - for (int i = 0; i < n; i++) { - assertEquals(wbuf.getShort(), sht++); + public void checkPutGetNonNativeShorts() { + short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); + wbuf.putShortArray(srcArray, 0, half); + wbuf.putShortArray(srcArray, half, half); + wbuf.resetPosition(); + //confirm + WritableBuffer wbuf2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Short.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + wbuf.resetPosition(); + wbuf2.resetPosition(); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Short.reverseBytes(wbuf2.getShort())); } - //getArr & putArr - short[] shortArr = new short[n]; //native - wbuf.setPosition(0); - wbuf.getShortArray(shortArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf2 = wmem2.asWritableBuffer(); - wbuf2.putShortArray(shortArr, 0, n); - assertEquals(arr2, arr1); + wbuf2.resetPosition(); + //get + short[] dstArray = new short[len]; + wbuf.getShortArray(dstArray, 0, half); + wbuf.getShortArray(dstArray, half, half); + assertEquals(srcArray, dstArray); } //check Duplicate, Region @Test public void checkDuplicate() { byte[] bArr = new byte[8]; - WritableMemory wmem = WritableMemory.writableWrap(bArr, ByteOrder.BIG_ENDIAN); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); WritableBuffer wbuf = wmem.asWritableBuffer(); WritableBuffer wdup = wbuf.writableDuplicate(); - assertEquals(wdup.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(wdup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); WritableBuffer wreg = wbuf.writableRegion(); - assertEquals(wreg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(wreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); } @Test public void checkConversionByteOrder() { byte[] bArr = new byte[8]; bArr[1] = 1; - WritableMemory wmem = WritableMemory.writableWrap(bArr, ByteOrder.BIG_ENDIAN); - assertEquals(wmem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); + assertEquals(wmem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(wmem.getChar(0), 1); Buffer buf = wmem.asBuffer(); - assertEquals(buf.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); // + assertEquals(buf.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); // assertEquals(buf.getChar(0), 1); Buffer dup = buf.duplicate(); - assertEquals(dup.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(dup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(dup.getChar(0), 1); Buffer reg = buf.region(); - assertEquals(reg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(reg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(reg.getChar(0), 1); Memory mem = reg.asMemory(); - assertEquals(mem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(mem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(mem.getChar(0), 1); Memory mreg = mem.region(0, 8); - assertEquals(mreg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); + assertEquals(mreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(mreg.getChar(0), 1); } @Test public void checkPutIntArray() { - WritableMemory wmem = WritableMemory.allocate(12, ByteOrder.BIG_ENDIAN); - WritableBuffer wbuf = wmem.asWritableBuffer(ByteOrder.BIG_ENDIAN); + WritableMemory wmem = WritableMemory.allocate(12, NON_NATIVE_BYTE_ORDER); + WritableBuffer wbuf = wmem.asWritableBuffer(NON_NATIVE_BYTE_ORDER); wbuf.putInt(1); int[] array = new int[] { 2 }; diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java index fcfff823..a43cb947 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java @@ -19,8 +19,12 @@ package org.apache.datasketches.memory.internal; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import org.apache.datasketches.memory.WritableMemory; @@ -33,135 +37,132 @@ public class NonNativeWritableMemoryImplTest { private byte[] bArr = new byte[8]; private final WritableMemory wmem = WritableMemory.writableWrap(bArr, ByteOrder.BIG_ENDIAN); -//Check primitives + //Check primitives + @Test - public void checkCharacters() { - int m = Character.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - for (int i = 0; i < n; i++) { wmem1.putChar(i * m, (char) i++); } - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getChar(i * m), (char) i++); + public void checkPutGetNonNativeCharacters() { + char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putCharArray(0, srcArray, 0, half); + wmem.putCharArray(half * Character.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Character.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Character.reverseBytes(wmem2.getChar(i * Character.BYTES))); } - //getArr & putArr - char[] cArr = new char[n]; //native - wmem1.getCharArray(0, cArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putCharArray(0, cArr, 0, n); - assertEquals(arr2, arr1); + //get + char[] dstArray = new char[len]; + wmem.getCharArray(0, dstArray, 0, half); + wmem.getCharArray(half * Character.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkDoubles() { - int m = Double.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - double dbl = 1.0; - for (int i = 0; i < n; i++) { wmem1.putDouble(i * m, dbl++); } - dbl = 1.0; - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getDouble(i * m), dbl++); + public void checkPutGetNonNativeDoubles() { + double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putDoubleArray(0, srcArray, 0, half); + wmem.putDoubleArray(half * Double.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Double.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == doubleReverseBytes(wmem2.getDouble(i * Double.BYTES))); } - //getArr & putArr - double[] dblArr = new double[n]; //native - wmem1.getDoubleArray(0, dblArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putDoubleArray(0, dblArr, 0, n); - assertEquals(arr2, arr1); + //get + double[] dstArray = new double[len]; + wmem.getDoubleArray(0, dstArray, 0, half); + wmem.getDoubleArray(half * Double.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkFloats() { - int m = Float.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - float flt = 1.0F; - for (int i = 0; i < n; i++) { wmem1.putFloat(i * m, flt++); } - flt = 1.0F; - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getFloat(i * m), flt++); + public void checkPutGetNonNativeFloats() { + float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putFloatArray(0, srcArray, 0, half); + wmem.putFloatArray(half * Float.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Float.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == floatReverseBytes(wmem2.getFloat(i * Float.BYTES))); } - //getArr & putArr - float[] fltArr = new float[n]; //native - wmem1.getFloatArray(0, fltArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putFloatArray(0, fltArr, 0, n); - assertEquals(arr2, arr1); + //get + float[] dstArray = new float[len]; + wmem.getFloatArray(0, dstArray, 0, half); + wmem.getFloatArray(half * Float.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkInts() { - int m = Integer.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - int intg = 1; - for (int i = 0; i < n; i++) { wmem1.putInt(i * m, intg++); } - intg = 1; - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getInt(i * m), intg++); + public void checkPutGetNonNativeInts() { + int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putIntArray(0, srcArray, 0, half); + wmem.putIntArray(half * Integer.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Integer.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Integer.reverseBytes(wmem2.getInt(i * Integer.BYTES))); } - //getArr & putArr - int[] intArr = new int[n]; //native - wmem1.getIntArray(0, intArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putIntArray(0, intArr, 0, n); - assertEquals(arr2, arr1); + //get + int[] dstArray = new int[len]; + wmem.getIntArray(0, dstArray, 0, half); + wmem.getIntArray(half * Integer.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkLongs() { - int m = Long.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - long lng = 1; - for (int i = 0; i < n; i++) { wmem1.putLong(i * m, lng++); } - lng = 1; - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getLong(i * m), lng++); + public void checkPutGetNonNativeLongs() { + long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putLongArray(0, srcArray, 0, half); + wmem.putLongArray(half * Long.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Long.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Long.reverseBytes(wmem2.getLong(i * Long.BYTES))); } - //getArr & putArr - long[] longArr = new long[n]; //native - wmem1.getLongArray(0, longArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putLongArray(0, longArr, 0, n); - assertEquals(arr2, arr1); + //get + long[] dstArray = new long[len]; + wmem.getLongArray(0, dstArray, 0, half); + wmem.getLongArray(half * Long.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } @Test - public void checkShorts() { - int m = Short.BYTES; - int n = ((1 << 20) / m) + m; - byte[] arr1 = new byte[n * m]; //non-native - //put & get - WritableMemory wmem1 = WritableMemory.writableWrap(arr1, ByteOrder.BIG_ENDIAN); - short sht = 1; - for (int i = 0; i < n; i++) { wmem1.putShort(i * m, sht++); } - sht = 1; - for (int i = 0; i < n; i++) { - assertEquals(wmem1.getShort(i * m), sht++); + public void checkPutGetNonNativeShorts() { + short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + final int len = srcArray.length; + final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER); + wmem.putShortArray(0, srcArray, 0, half); + wmem.putShortArray(half * Short.BYTES, srcArray, half, half); + //confirm + WritableMemory wmem2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Short.BYTES); + for (int i = 0; i < len; i++) { + assertTrue(srcArray[i] == Short.reverseBytes(wmem2.getShort(i * Short.BYTES))); } - //getArr & putArr - short[] shortArr = new short[n]; //native - wmem1.getShortArray(0, shortArr, 0, n); //wmem is non-native - byte[] arr2 = new byte[n * m]; - WritableMemory wmem2 = WritableMemory.writableWrap(arr2, ByteOrder.BIG_ENDIAN); - wmem2.putShortArray(0, shortArr, 0, n); - assertEquals(arr2, arr1); + //get + short[] dstArray = new short[len]; + wmem.getShortArray(0, dstArray, 0, half); + wmem.getShortArray(half * Short.BYTES, dstArray, half, half); + assertEquals(srcArray, dstArray); } //check Atomic Write Methods @@ -173,4 +174,18 @@ public void checkRegion() { assertEquals(wreg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); } + //Java does not provide reverse bytes on doubles or floats + + static double doubleReverseBytes(double value) { + long longIn = Double.doubleToRawLongBits(value); + long longOut = Long.reverseBytes(longIn); + return Double.longBitsToDouble(longOut); + } + + static float floatReverseBytes(float value) { + int intIn = Float.floatToRawIntBits(value); + int intOut = Integer.reverseBytes(intIn); + return Float.intBitsToFloat(intOut); + } + } From 4c73fb46b1b7286a88b128434ca90fba907e9511 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sat, 1 Feb 2025 16:13:10 -0800 Subject: [PATCH 04/19] a two letter typo. --- .../org/apache/datasketches/memory/MemoryRequestServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java b/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java index a5eab400..e6de60ac 100644 --- a/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java +++ b/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java @@ -38,7 +38,7 @@ public interface MemoryRequestServer { * @param alignmentBytes requested segment alignment. Typically 1, 2, 4 or 8. * @param byteOrder the given ByteOrder. It must be non-null. * @param arena the given arena to manage the new off-heap WritableMemory. - * If arena is null, the requested WritableMemory will be off-heap. + * If arena is null, the requested WritableMemory will be on-heap. * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. * @return new WritableMemory with the requested capacity. */ From cabf9ca9f26793977770ca7e95b6ce0d430c38d6 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 2 Feb 2025 16:15:26 -0800 Subject: [PATCH 05/19] More comprehensive tests. Brought coverage from 90% to 95.7% --- .../internal/NativeWritableBufferImpl.java | 4 +- .../memory/internal/ResourceImpl.java | 2 +- .../memory/internal/WritableBufferImpl.java | 2 +- .../NonNativeWritableBufferImplTest.java | 188 +++++++++++------- .../NonNativeWritableMemoryImplTest.java | 151 +++++++------- .../memory/internal/ResourceTest.java | 21 ++ .../memory/internal/UtilForTest.java | 34 ++++ 7 files changed, 262 insertions(+), 140 deletions(-) create mode 100644 src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java diff --git a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java index 27b77514..f9b07ac0 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/NativeWritableBufferImpl.java @@ -41,10 +41,10 @@ final class NativeWritableBufferImpl extends WritableBufferImpl { //Pass-through ctor NativeWritableBufferImpl( - final Arena arena, final MemorySegment seg, final int typeId, - final MemoryRequestServer memReqSvr) { + final MemoryRequestServer memReqSvr, + final Arena arena) { super(seg, typeId, memReqSvr, arena); } diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index 1245f65d..459af659 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -220,7 +220,7 @@ static final WritableBuffer selectBuffer( final MemoryRequestServer memReqSvr2 = (byteBufferType || mapType) ? null : memReqSvr; final WritableBuffer wbuf; if (nativeBOType) { - wbuf = new NativeWritableBufferImpl(null, segment, type, memReqSvr2); + wbuf = new NativeWritableBufferImpl(segment, type, memReqSvr2, null); } else { //non-native BO wbuf = new NonNativeWritableBufferImpl(segment, type, memReqSvr2, null); } diff --git a/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java index b2e057f2..caf75f1a 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/WritableBufferImpl.java @@ -90,7 +90,7 @@ public static WritableBuffer wrapByteBuffer( type |= NONNATIVE_BO; wbuf = new NonNativeWritableBufferImpl(seg, type, memReqSvr, null); } else { - wbuf = new NativeWritableBufferImpl(null, seg, type, memReqSvr); + wbuf = new NativeWritableBufferImpl(seg, type, memReqSvr, null); } wbuf.setStartPositionEnd(0, byteBuffer.position(), byteBuffer.limit()); return wbuf; diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java index 8e598666..316f1853 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java @@ -19,15 +19,17 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImplTest.doubleReverseBytes; -import static org.apache.datasketches.memory.internal.NonNativeWritableMemoryImplTest.floatReverseBytes; -import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; -import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.UtilForTest.CB; +import static org.apache.datasketches.memory.internal.UtilForTest.DB; +import static org.apache.datasketches.memory.internal.UtilForTest.FB; +import static org.apache.datasketches.memory.internal.UtilForTest.IB; +import static org.apache.datasketches.memory.internal.UtilForTest.LB; +import static org.apache.datasketches.memory.internal.UtilForTest.NBO; +import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; +import static org.apache.datasketches.memory.internal.UtilForTest.SB; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.nio.ByteOrder; - import org.apache.datasketches.memory.Buffer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableBuffer; @@ -43,26 +45,34 @@ public class NonNativeWritableBufferImplTest { @Test public void checkPutGetNonNativeCharacters() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putCharArray(srcArray, 0, half); - wbuf.putCharArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * CB, NNBO).asWritableBuffer(); + //put + wbuf.putCharArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putChar(qtr * CB, srcArray[qtr]); //put*(add, value) + wbuf.putChar((qtr + 1) * CB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * CB); + for (int i = half; i < len; i++) { wbuf.putChar(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Character.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * CB, NBO).asWritableBuffer(); + for (int i = 0; i < len * CB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Character.reverseBytes(wbuf2.getChar())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); char[] dstArray = new char[len]; - wbuf.getCharArray(dstArray, 0, half); - wbuf.getCharArray(dstArray, half, half); + wbuf.getCharArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getChar(qtr * CB); //get*(add) + dstArray[qtr + 1] = wbuf.getChar((qtr + 1) * CB); //get*(add) + wbuf.setPosition(half * CB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(); } //get*() assertEquals(srcArray, dstArray); } @@ -71,23 +81,31 @@ public void checkPutGetNonNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putDoubleArray(srcArray, 0, half); - wbuf.putDoubleArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * DB, NNBO).asWritableBuffer(); + //put + wbuf.putDoubleArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putDouble(qtr * DB, srcArray[qtr]); //put*(add, value) + wbuf.putDouble((qtr + 1) * DB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * DB); + for (int i = half; i < len; i++) { wbuf.putDouble(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Double.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * DB, NBO).asWritableBuffer(); + for (int i = 0; i < len * DB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == doubleReverseBytes(wbuf2.getDouble())); + assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wbuf2.getDouble())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); double[] dstArray = new double[len]; - wbuf.getDoubleArray(dstArray, 0, half); - wbuf.getDoubleArray(dstArray, half, half); + wbuf.getDoubleArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getDouble(qtr * DB); //get*(add) + dstArray[qtr + 1] = wbuf.getDouble((qtr + 1) * DB); //get*(add) + wbuf.setPosition(half * DB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(); } //get*() assertEquals(srcArray, dstArray); } @@ -96,23 +114,31 @@ public void checkPutGetNonNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putFloatArray(srcArray, 0, half); - wbuf.putFloatArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * FB, NNBO).asWritableBuffer(); + //put + wbuf.putFloatArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putFloat(qtr * FB, srcArray[qtr]); //put*(add, value) + wbuf.putFloat((qtr + 1) * FB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * FB); + for (int i = half; i < len; i++) { wbuf.putFloat(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Float.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * FB, NBO).asWritableBuffer(); + for (int i = 0; i < len * FB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == floatReverseBytes(wbuf2.getFloat())); + assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wbuf2.getFloat())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); float[] dstArray = new float[len]; - wbuf.getFloatArray(dstArray, 0, half); - wbuf.getFloatArray(dstArray, half, half); + wbuf.getFloatArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getFloat(qtr * FB); //get*(add) + dstArray[qtr + 1] = wbuf.getFloat((qtr + 1) * FB); //get*(add) + wbuf.setPosition(half * FB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(); } //get*() assertEquals(srcArray, dstArray); } @@ -121,23 +147,31 @@ public void checkPutGetNonNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putIntArray(srcArray, 0, half); - wbuf.putIntArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * IB, NNBO).asWritableBuffer(); + //put + wbuf.putIntArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putInt(qtr * IB, srcArray[qtr]); //put*(add, value) + wbuf.putInt((qtr + 1) * IB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * IB); + for (int i = half; i < len; i++) { wbuf.putInt(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Integer.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * IB, NBO).asWritableBuffer(); + for (int i = 0; i < len * IB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Integer.reverseBytes(wbuf2.getInt())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); int[] dstArray = new int[len]; - wbuf.getIntArray(dstArray, 0, half); - wbuf.getIntArray(dstArray, half, half); + wbuf.getIntArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getInt(qtr * IB); //get*(add) + dstArray[qtr + 1] = wbuf.getInt((qtr + 1) * IB); //get*(add) + wbuf.setPosition(half * IB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(); } //get*() assertEquals(srcArray, dstArray); } @@ -146,23 +180,31 @@ public void checkPutGetNonNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putLongArray(srcArray, 0, half); - wbuf.putLongArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * LB, NNBO).asWritableBuffer(); + //put + wbuf.putLongArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putLong(qtr * LB, srcArray[qtr]); //put*(add, value) + wbuf.putLong((qtr + 1) * LB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * LB); + for (int i = half; i < len; i++) { wbuf.putLong(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Long.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * LB, NBO).asWritableBuffer(); + for (int i = 0; i < len * LB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Long.reverseBytes(wbuf2.getLong())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); long[] dstArray = new long[len]; - wbuf.getLongArray(dstArray, 0, half); - wbuf.getLongArray(dstArray, half, half); + wbuf.getLongArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getLong(qtr * LB); //get*(add) + dstArray[qtr + 1] = wbuf.getLong((qtr + 1) * LB); //get*(add) + wbuf.setPosition(half * LB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(); } //get*() assertEquals(srcArray, dstArray); } @@ -171,23 +213,31 @@ public void checkPutGetNonNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); - wbuf.putShortArray(srcArray, 0, half); - wbuf.putShortArray(srcArray, half, half); + final int qtr = len / 4; + WritableBuffer wbuf = WritableMemory.allocate(len * SB, NNBO).asWritableBuffer(); + //put + wbuf.putShortArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) + wbuf.putShort(qtr * SB, srcArray[qtr]); //put*(add, value) + wbuf.putShort((qtr + 1) * SB, srcArray[qtr + 1]); //put*(add, value) + wbuf.setPosition(half * SB); + for (int i = half; i < len; i++) { wbuf.putShort(srcArray[i]); } //put*(value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); - for (int i = 0; i < len * Short.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * SB, NBO).asWritableBuffer(); + for (int i = 0; i < len * SB; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Short.reverseBytes(wbuf2.getShort())); } - wbuf2.resetPosition(); //get + wbuf2.resetPosition(); short[] dstArray = new short[len]; - wbuf.getShortArray(dstArray, 0, half); - wbuf.getShortArray(dstArray, half, half); + wbuf.getShortArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) + dstArray[qtr] = wbuf.getShort(qtr * SB); //get*(add) + dstArray[qtr + 1] = wbuf.getShort((qtr + 1) * SB); //get*(add) + wbuf.setPosition(half * SB); + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(); } //get*() assertEquals(srcArray, dstArray); } @@ -195,48 +245,48 @@ public void checkPutGetNonNativeShorts() { @Test public void checkDuplicate() { byte[] bArr = new byte[8]; - WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); WritableBuffer wbuf = wmem.asWritableBuffer(); WritableBuffer wdup = wbuf.writableDuplicate(); - assertEquals(wdup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(wdup.getTypeByteOrder(), NNBO); WritableBuffer wreg = wbuf.writableRegion(); - assertEquals(wreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(wreg.getTypeByteOrder(), NNBO); } @Test public void checkConversionByteOrder() { byte[] bArr = new byte[8]; bArr[1] = 1; - WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); - assertEquals(wmem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); + assertEquals(wmem.getTypeByteOrder(), NNBO); assertEquals(wmem.getChar(0), 1); Buffer buf = wmem.asBuffer(); - assertEquals(buf.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); // + assertEquals(buf.getTypeByteOrder(), NNBO); // assertEquals(buf.getChar(0), 1); Buffer dup = buf.duplicate(); - assertEquals(dup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(dup.getTypeByteOrder(), NNBO); assertEquals(dup.getChar(0), 1); Buffer reg = buf.region(); - assertEquals(reg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(reg.getTypeByteOrder(), NNBO); assertEquals(reg.getChar(0), 1); Memory mem = reg.asMemory(); - assertEquals(mem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(mem.getTypeByteOrder(), NNBO); assertEquals(mem.getChar(0), 1); Memory mreg = mem.region(0, 8); - assertEquals(mreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); + assertEquals(mreg.getTypeByteOrder(), NNBO); assertEquals(mreg.getChar(0), 1); } @Test public void checkPutIntArray() { - WritableMemory wmem = WritableMemory.allocate(12, NON_NATIVE_BYTE_ORDER); - WritableBuffer wbuf = wmem.asWritableBuffer(NON_NATIVE_BYTE_ORDER); + WritableMemory wmem = WritableMemory.allocate(12, NNBO); + WritableBuffer wbuf = wmem.asWritableBuffer(NNBO); wbuf.putInt(1); int[] array = new int[] { 2 }; diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java index a43cb947..5dc6d928 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java @@ -19,8 +19,14 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; -import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.UtilForTest.CB; +import static org.apache.datasketches.memory.internal.UtilForTest.DB; +import static org.apache.datasketches.memory.internal.UtilForTest.FB; +import static org.apache.datasketches.memory.internal.UtilForTest.IB; +import static org.apache.datasketches.memory.internal.UtilForTest.LB; +import static org.apache.datasketches.memory.internal.UtilForTest.NBO; +import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; +import static org.apache.datasketches.memory.internal.UtilForTest.SB; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -35,28 +41,33 @@ */ public class NonNativeWritableMemoryImplTest { private byte[] bArr = new byte[8]; - private final WritableMemory wmem = WritableMemory.writableWrap(bArr, ByteOrder.BIG_ENDIAN); + private final WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); //Check primitives @Test public void checkPutGetNonNativeCharacters() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putCharArray(0, srcArray, 0, half); - wmem.putCharArray(half * Character.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * CB, NNBO); + //put + wmem.putCharArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putChar((half + i) * CB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Character.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * CB, NBO); + wmem.copyTo(0, wmem2, 0, len * CB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Character.reverseBytes(wmem2.getChar(i * Character.BYTES))); + assertTrue(srcArray[i] == Character.reverseBytes(wmem2.getChar(i * CB))); } //get char[] dstArray = new char[len]; - wmem.getCharArray(0, dstArray, 0, half); - wmem.getCharArray(half * Character.BYTES, dstArray, half, half); + wmem.getCharArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getChar(i * CB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -65,19 +76,23 @@ public void checkPutGetNonNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putDoubleArray(0, srcArray, 0, half); - wmem.putDoubleArray(half * Double.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * DB, NNBO); + wmem.putDoubleArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putDouble((half + i) * DB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Double.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * DB, NBO); + wmem.copyTo(0, wmem2, 0, len * DB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == doubleReverseBytes(wmem2.getDouble(i * Double.BYTES))); + assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wmem2.getDouble(i * DB))); } //get double[] dstArray = new double[len]; - wmem.getDoubleArray(0, dstArray, 0, half); - wmem.getDoubleArray(half * Double.BYTES, dstArray, half, half); + wmem.getDoubleArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getDouble(i * DB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -86,19 +101,23 @@ public void checkPutGetNonNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putFloatArray(0, srcArray, 0, half); - wmem.putFloatArray(half * Float.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * FB, NNBO); + wmem.putFloatArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putFloat((half + i) * FB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Float.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * FB, NBO); + wmem.copyTo(0, wmem2, 0, len * FB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == floatReverseBytes(wmem2.getFloat(i * Float.BYTES))); + assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wmem2.getFloat(i * FB))); } //get float[] dstArray = new float[len]; - wmem.getFloatArray(0, dstArray, 0, half); - wmem.getFloatArray(half * Float.BYTES, dstArray, half, half); + wmem.getFloatArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getFloat(i * FB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -107,19 +126,23 @@ public void checkPutGetNonNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putIntArray(0, srcArray, 0, half); - wmem.putIntArray(half * Integer.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * IB, NNBO); + wmem.putIntArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putInt((half + i) * IB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Integer.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * IB, NBO); + wmem.copyTo(0, wmem2, 0, len * IB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Integer.reverseBytes(wmem2.getInt(i * Integer.BYTES))); + assertTrue(srcArray[i] == Integer.reverseBytes(wmem2.getInt(i * IB))); } //get int[] dstArray = new int[len]; - wmem.getIntArray(0, dstArray, 0, half); - wmem.getIntArray(half * Integer.BYTES, dstArray, half, half); + wmem.getIntArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getInt(i * IB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -128,19 +151,23 @@ public void checkPutGetNonNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putLongArray(0, srcArray, 0, half); - wmem.putLongArray(half * Long.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * LB, NNBO); + wmem.putLongArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putLong((half + i) * LB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Long.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * LB, NBO); + wmem.copyTo(0, wmem2, 0, len * LB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Long.reverseBytes(wmem2.getLong(i * Long.BYTES))); + assertTrue(srcArray[i] == Long.reverseBytes(wmem2.getLong(i * LB))); } //get long[] dstArray = new long[len]; - wmem.getLongArray(0, dstArray, 0, half); - wmem.getLongArray(half * Long.BYTES, dstArray, half, half); + wmem.getLongArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getLong(i *LB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -149,19 +176,23 @@ public void checkPutGetNonNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER); - wmem.putShortArray(0, srcArray, 0, half); - wmem.putShortArray(half * Short.BYTES, srcArray, half, half); + WritableMemory wmem = WritableMemory.allocate(len * SB, NNBO); + wmem.putShortArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) + for (int i = 0; i < half; i++) { + wmem.putShort((half + i) * SB, srcArray[half + i]); //put*(add, value) + } //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER); - wmem.copyTo(0, wmem2, 0, len * Short.BYTES); + WritableMemory wmem2 = WritableMemory.allocate(len * SB, NBO); + wmem.copyTo(0, wmem2, 0, len * SB); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Short.reverseBytes(wmem2.getShort(i * Short.BYTES))); + assertTrue(srcArray[i] == Short.reverseBytes(wmem2.getShort(i * SB))); } //get short[] dstArray = new short[len]; - wmem.getShortArray(0, dstArray, 0, half); - wmem.getShortArray(half * Short.BYTES, dstArray, half, half); + wmem.getShortArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) + for (int i = half; i < len; i++) { + dstArray[i] = wmem.getShort(i *SB); //get*(add) + } assertEquals(srcArray, dstArray); } @@ -171,21 +202,7 @@ public void checkPutGetNonNativeShorts() { @Test public void checkRegion() { WritableMemory wreg = wmem.writableRegion(0, wmem.getCapacity()); - assertEquals(wreg.getTypeByteOrder(), ByteOrder.BIG_ENDIAN); - } - - //Java does not provide reverse bytes on doubles or floats - - static double doubleReverseBytes(double value) { - long longIn = Double.doubleToRawLongBits(value); - long longOut = Long.reverseBytes(longIn); - return Double.longBitsToDouble(longOut); - } - - static float floatReverseBytes(float value) { - int intIn = Float.floatToRawIntBits(value); - int intOut = Integer.reverseBytes(intIn); - return Float.intBitsToFloat(intOut); + assertEquals(wreg.getTypeByteOrder(), NNBO); } } diff --git a/src/test/java/org/apache/datasketches/memory/internal/ResourceTest.java b/src/test/java/org/apache/datasketches/memory/internal/ResourceTest.java index c4fceba0..5be59c85 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/ResourceTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/ResourceTest.java @@ -270,6 +270,27 @@ public void checkXxHash64() { assertTrue(out != 0); } + @Test + public void checkMismatch() { + byte[] arr1 = {1,2,3,4}; + byte[] arr2 = {1,2,3,4}; + byte[] arr3 = {1,2,3,4,5}; + Memory mem1 = Memory.wrap(arr1); + Memory mem2 = Memory.wrap(arr2); + Memory mem3 = Memory.wrap(arr3); + assertEquals(mem1.mismatch(mem2), -1); + assertEquals(mem1.mismatch(mem3), 4); + + byte[] arr4 = {9,9,1,2,3,4,9,9}; + byte[] arr5 = {8,8,8,1,2,3,4,8}; + byte[] arr6 = {8,8,8,1,2,3,4,5}; + Memory mem4 = Memory.wrap(arr4); + Memory mem5 = Memory.wrap(arr5); + Memory mem6 = Memory.wrap(arr6); + assertEquals(mem4.mismatch(mem4, 2, 6, mem5, 3, 7), -1); + assertEquals(mem4.mismatch(mem4, 2, 7, mem6, 3, 8), 4); + } + /********************/ @Test public void printlnTest() { diff --git a/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java new file mode 100644 index 00000000..72cea396 --- /dev/null +++ b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java @@ -0,0 +1,34 @@ +package org.apache.datasketches.memory.internal; + +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; + +import java.nio.ByteOrder; + +public class UtilForTest { + + static final int BB = Byte.BYTES; + static final int CB = Character.BYTES; + static final int SB = Short.BYTES; + static final int IB = Integer.BYTES; + static final int LB = Long.BYTES; + static final int DB = Double.BYTES; + static final int FB = Float.BYTES; + static ByteOrder NNBO = NON_NATIVE_BYTE_ORDER; + static ByteOrder NBO = NATIVE_BYTE_ORDER; + + //Java does not provide reverse bytes on doubles or floats + + static double doubleReverseBytes(double value) { + long longIn = Double.doubleToRawLongBits(value); + long longOut = Long.reverseBytes(longIn); + return Double.longBitsToDouble(longOut); + } + + static float floatReverseBytes(float value) { + int intIn = Float.floatToRawIntBits(value); + int intOut = Integer.reverseBytes(intIn); + return Float.intBitsToFloat(intOut); + } + +} From 7f40e8b100f6fde8ca61a193f22fd4543beb28bd Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 2 Feb 2025 16:26:40 -0800 Subject: [PATCH 06/19] Add missing license --- .../memory/internal/UtilForTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java index 72cea396..81be1af1 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.datasketches.memory.internal; import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; From e0faea240e624f6278b671d551de057362db9cd0 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 3 Feb 2025 12:28:42 -0800 Subject: [PATCH 07/19] Better test coverage AND actual checking for the @pawel-weijacha bug for all the primitives. --- .../NativeWritableBufferImplTest.java | 315 +++++++----------- .../NativeWritableMemoryImplTest.java | 303 +++++++---------- .../NonNativeWritableBufferImplTest.java | 132 +++----- .../NonNativeWritableMemoryImplTest.java | 101 +++--- 4 files changed, 354 insertions(+), 497 deletions(-) diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java index c1fbc36a..b72f0da5 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java @@ -20,7 +20,14 @@ package org.apache.datasketches.memory.internal; import java.lang.foreign.Arena; -import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.UtilForTest.CB; +import static org.apache.datasketches.memory.internal.UtilForTest.DB; +import static org.apache.datasketches.memory.internal.UtilForTest.FB; +import static org.apache.datasketches.memory.internal.UtilForTest.IB; +import static org.apache.datasketches.memory.internal.UtilForTest.LB; +import static org.apache.datasketches.memory.internal.UtilForTest.NBO; +import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; +import static org.apache.datasketches.memory.internal.UtilForTest.SB; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -38,6 +45,9 @@ import org.testng.Assert; import org.testng.annotations.Test; +/** + * @author Lee Rhodes + */ public class NativeWritableBufferImplTest { private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr; @@ -55,239 +65,170 @@ public void checkNativeCapacityAndClose() throws Exception { try { wmem.getArena().close(); } catch (IllegalStateException e) { } } - //Simple Heap arrays + //Check primitives @Test - public void checkGetByteArray() { - byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetBooleans() { + boolean[] srcArray = {true, false, true, false, false, true, false, true}; final int len = srcArray.length; final int half = len / 2; - byte[] dstArray = new byte[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getByteArray(dstArray, 0, half); - buf.getByteArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getByteArray(dstArray, 0, half); - wbuf.getByteArray(dstArray, half, half); - assertEquals(dstArray, srcArray); + WritableBuffer wbuf = WritableMemory.allocate(len, NBO).asWritableBuffer(); + //put + for (int i = 0; i < half; i++) { wbuf.putBoolean(srcArray[i]); } //put*(value) + for (int i = half; i < len; i++) { wbuf.putBoolean(i, srcArray[i]); } //put*(add, value) + wbuf.resetPosition(); + //get + boolean[] dstArray = new boolean[len]; + for (int i = 0; i < half; i++) { dstArray[i] = wbuf.getBoolean(); } //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getBoolean(i); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutByteArray() { + public void checkPutGetBytes() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; final int len = srcArray.length; final int half = len / 2; - byte[] dstArray = new byte[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Byte.BYTES).asWritableBuffer(); - wbuf.putByteArray(srcArray, 0, half); - wbuf.putByteArray(srcArray, half, half); + WritableBuffer wbuf = WritableMemory.allocate(len, NBO).asWritableBuffer(); + //put + wbuf.putByte(srcArray[0]); //put*(value) + wbuf.putByteArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putByte(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putByte(i, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); - wbuf.getByteArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); + //get + byte[] dstArray = new byte[len]; + dstArray[0] = wbuf.getByte(); //get*() + wbuf.getByteArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getByte(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getByte(i); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkGetCharArray() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + public void checkPutGetNativeCharacters() { + char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * CB, NBO).asWritableBuffer(); + //put + wbuf.putChar(srcArray[0]); //put*(value) + wbuf.putCharArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putChar(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putChar(i * CB, srcArray[i]); } //put*(add, value) + wbuf.resetPosition(); + //get char[] dstArray = new char[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getCharArray(dstArray, 0, half); - buf.getCharArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getCharArray(dstArray, 0, half); - wbuf.getCharArray(dstArray, half, half); - assertEquals(dstArray, srcArray); + dstArray[0] = wbuf.getChar(); //get*() + wbuf.getCharArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getChar(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * CB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutCharArray() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + public void checkPutGetNativeDoubles() { + double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - char[] dstArray = new char[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES).asWritableBuffer(); - wbuf.putCharArray(srcArray, 0, half); - wbuf.putCharArray(srcArray, half, half); + WritableBuffer wbuf = WritableMemory.allocate(len * DB, NBO).asWritableBuffer(); + //put + wbuf.putDouble(srcArray[0]); //put*(value) + wbuf.putDoubleArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putDouble(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putDouble(i * DB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); - wbuf.getCharArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetShortArray() { - short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - short[] dstArray = new short[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getShortArray(dstArray, 0, half); - buf.getShortArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getShortArray(dstArray, 0, half); - wbuf.getShortArray(dstArray, half, half); - assertEquals(dstArray, srcArray); + //get + double[] dstArray = new double[len]; + dstArray[0] = wbuf.getDouble(); //get*() + wbuf.getDoubleArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getDouble(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * DB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutShortArray() { - short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeFloats() { + float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - short[] dstArray = new short[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES).asWritableBuffer(); - wbuf.putShortArray(srcArray, 0, half); - wbuf.putShortArray(srcArray, half, half); + WritableBuffer wbuf = WritableMemory.allocate(len * FB, NBO).asWritableBuffer(); + //put + wbuf.putFloat(srcArray[0]); //put*(value) + wbuf.putFloatArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putFloat(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putFloat(i * FB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); - wbuf.getShortArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); + //get + float[] dstArray = new float[len]; + dstArray[0] = wbuf.getFloat(); //get*() + wbuf.getFloatArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getFloat(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * FB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkGetIntArray() { - int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeInts() { + int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; + WritableBuffer wbuf = WritableMemory.allocate(len * IB, NBO).asWritableBuffer(); + //put + wbuf.putInt(srcArray[0]); //put*(value) + wbuf.putIntArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putInt(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putInt(i * IB, srcArray[i]); } //put*(add, value) + wbuf.resetPosition(); + //get int[] dstArray = new int[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getIntArray(dstArray, 0, half); - buf.getIntArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getIntArray(dstArray, 0, half); - wbuf.getIntArray(dstArray, half, half); - assertEquals(dstArray, srcArray); + dstArray[0] = wbuf.getInt(); //get*() + wbuf.getIntArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getInt(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * IB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutIntArray() { - int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeLongs() { + long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - int[] dstArray = new int[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES).asWritableBuffer(); - wbuf.putIntArray(srcArray, 0, half); - wbuf.putIntArray(srcArray, half, half); + WritableBuffer wbuf = WritableMemory.allocate(len * LB, NNBO).asWritableBuffer(); + //put + wbuf.putLong(srcArray[0]); //put*(value) + wbuf.putLongArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putLong(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putLong(i * LB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); - wbuf.getIntArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetLongArray() { - long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - long[] dstArray = new long[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getLongArray(dstArray, 0, half); - buf.getLongArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getLongArray(dstArray, 0, half); - wbuf.getLongArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkPutLongArray() { - long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; + //get long[] dstArray = new long[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES).asWritableBuffer(); - wbuf.putLongArray(srcArray, 0, half); - wbuf.putLongArray(srcArray, half, half); - wbuf.resetPosition(); - wbuf.getLongArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetFloatArray() { - float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - float[] dstArray = new float[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getFloatArray(dstArray, 0, half); - buf.getFloatArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getFloatArray(dstArray, 0, half); - wbuf.getFloatArray(dstArray, half, half); - assertEquals(dstArray, srcArray); + dstArray[0] = wbuf.getLong(); //get*() + wbuf.getLongArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getLong(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * LB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutFloatArray() { - float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeShorts() { + short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - float[] dstArray = new float[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES).asWritableBuffer(); - wbuf.putFloatArray(srcArray, 0, half); - wbuf.putFloatArray(srcArray, half, half); + WritableBuffer wbuf = WritableMemory.allocate(len * SB, NBO).asWritableBuffer(); + //put + wbuf.putShort(srcArray[0]); //put*(value) + wbuf.putShortArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putShort(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putShort(i * SB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); - wbuf.getFloatArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetDoubleArray() { - double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - double[] dstArray = new double[len]; - - Buffer buf = Memory.wrap(srcArray).asBuffer(); - buf.getDoubleArray(dstArray, 0, half); - buf.getDoubleArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - - - WritableBuffer wbuf = WritableMemory.writableWrap(srcArray).asWritableBuffer(); - wbuf.getDoubleArray(dstArray, 0, half); - wbuf.getDoubleArray(dstArray, half, half); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkPutDoubleArray() { - double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - double[] dstArray = new double[len]; - - WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES).asWritableBuffer(); - wbuf.putDoubleArray(srcArray, 0, half); - wbuf.putDoubleArray(srcArray, half, half); - wbuf.resetPosition(); - wbuf.getDoubleArray(dstArray, 0, len); - assertEquals(dstArray, srcArray); + //get + short[] dstArray = new short[len]; + dstArray[0] = wbuf.getShort(); //get*() + wbuf.getShortArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getShort(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * SB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test @@ -571,7 +512,7 @@ public void checkZeroBuffer() { public void checkDuplicateNonNative() { WritableMemory wmem = WritableMemory.allocate(64); wmem.putShort(0, (short) 1); - Buffer buf = wmem.asWritableBuffer().duplicate(NON_NATIVE_BYTE_ORDER); + Buffer buf = wmem.asWritableBuffer().duplicate(NNBO); assertEquals(buf.getShort(0), 256); } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java index c58c6691..07726dd3 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java @@ -19,7 +19,14 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.UtilForTest.CB; +import static org.apache.datasketches.memory.internal.UtilForTest.DB; +import static org.apache.datasketches.memory.internal.UtilForTest.FB; +import static org.apache.datasketches.memory.internal.UtilForTest.IB; +import static org.apache.datasketches.memory.internal.UtilForTest.LB; +import static org.apache.datasketches.memory.internal.UtilForTest.NBO; +import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; +import static org.apache.datasketches.memory.internal.UtilForTest.SB; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -58,231 +65,159 @@ public void checkNativeCapacityAndClose() throws Exception { try { wmem.getArena().close(); } catch (IllegalStateException e) { } } - //Simple Heap arrays + //Check primitives @Test - public void checkGetByteArray() { - byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetBooleans() { + boolean[] srcArray = {true, false, true, false, false, true, false, true}; final int len = srcArray.length; - final int half = len / 2; - byte[] dstArray = new byte[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getByteArray(0, dstArray, 0, half); - mem.getByteArray(half, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getByteArray(0, dstArray, 0, half); - wmem.getByteArray(half, dstArray, half, half); - assertEquals(dstArray, srcArray); + WritableMemory wmem = WritableMemory.allocate(len, NBO); + //put + for (int i = 0; i < len; i++) { wmem.putBoolean(i, srcArray[i]); } //put*(add, value) + //get + boolean[] dstArray = new boolean[len]; + for (int i = 0; i < len; i++) { dstArray[i] = wmem.getBoolean(i); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutByteArray() { + public void checkPutGetBytes() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; final int len = srcArray.length; final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len, NBO); + //put + wmem.putByte(0, srcArray[0]); //put*(add, value) + wmem.putByteArray(1, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putByte(3, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putByte(i, srcArray[i]); } //put*(add, value) + //get byte[] dstArray = new byte[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Byte.BYTES); - wmem.putByteArray(0, srcArray, 0, half); - wmem.putByteArray(half * Byte.BYTES, srcArray, half, half); - wmem.getByteArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); + dstArray[0] = wmem.getByte(0); //get*(add) + wmem.getByteArray(1, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getByte(3); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getByte(i); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkGetCharArray() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; + public void checkPutGetNativeCharacters() { + char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * CB, NBO); + //put + wmem.putChar(0, srcArray[0]); //put*(add, value) + wmem.putCharArray(1 * CB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putChar(3 * CB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putChar(i * CB, srcArray[i]); } //put*(add, value) + //get char[] dstArray = new char[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getCharArray(0, dstArray, 0, half); - mem.getCharArray(half * Character.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getCharArray(0, dstArray, 0, half); - wmem.getCharArray(half * Character.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkPutCharArray() { - char[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; - final int len = srcArray.length; - final int half = len / 2; - char[] dstArray = new char[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES); - wmem.putCharArray(0, srcArray, 0, half); - wmem.putCharArray(half * Character.BYTES, srcArray, half, half); - wmem.getCharArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetShortArray() { - short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - short[] dstArray = new short[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getShortArray(0, dstArray, 0, half); - mem.getShortArray(half * Short.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getShortArray(0, dstArray, 0, half); - wmem.getShortArray(half * Short.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); + dstArray[0] = wmem.getChar(0); //get*(add) + wmem.getCharArray(1 * CB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getChar(3 * CB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * CB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutShortArray() { - short[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeDoubles() { + double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - short[] dstArray = new short[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES); - wmem.putShortArray(0, srcArray, 0, half); - wmem.putShortArray(half * Short.BYTES, srcArray, half, half); - wmem.getShortArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); + WritableMemory wmem = WritableMemory.allocate(len * DB, NBO); + //put + wmem.putDouble(0, srcArray[0]); //put*(add, value) + wmem.putDoubleArray(1 * DB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putDouble(3 * DB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putDouble(i * DB, srcArray[i]); } //put*(add, value) + //get + double[] dstArray = new double[len]; + dstArray[0] = wmem.getDouble(0); //get*(add) + wmem.getDoubleArray(1 * DB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getDouble(3 * DB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * DB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkGetIntArray() { - int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeFloats() { + float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - int[] dstArray = new int[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getIntArray(0, dstArray, 0, half); - mem.getIntArray(half * Integer.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getIntArray(0, dstArray, 0, half); - wmem.getIntArray(half * Integer.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); + WritableMemory wmem = WritableMemory.allocate(len * FB, NBO); + //put + wmem.putFloat(0, srcArray[0]); //put*(add, value) + wmem.putFloatArray(1 * FB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putFloat(3 * FB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putFloat(i * FB, srcArray[i]); } //put*(add, value) + //get + float[] dstArray = new float[len]; + dstArray[0] = wmem.getFloat(0); //get*(add) + wmem.getFloatArray(1 * FB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getFloat(3 * FB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * FB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutIntArray() { - int[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeInts() { + int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * IB, NBO); + //put + wmem.putInt(0, srcArray[0]); //put*(add, value) + wmem.putIntArray(1 * IB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putInt(3 * IB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putInt(i * IB, srcArray[i]); } //put*(add, value) + //get int[] dstArray = new int[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES); - wmem.putIntArray(0, srcArray, 0, half); - wmem.putIntArray(half * Integer.BYTES, srcArray, half, half); - wmem.getIntArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); + dstArray[0] = wmem.getInt(0); //get*(add) + wmem.getIntArray(1 * IB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getInt(3 * IB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * IB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkGetLongArray() { - long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeLongs() { + long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; + WritableMemory wmem = WritableMemory.allocate(len * LB, NBO); + //put + wmem.putLong(0, srcArray[0]); //put*(add, value) + wmem.putLongArray(1 * LB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putLong(3 * LB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putLong(i * LB, srcArray[i]); } //put*(add, value) + //get long[] dstArray = new long[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getLongArray(0, dstArray, 0, half); - mem.getLongArray(half * Long.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getLongArray(0, dstArray, 0, half); - wmem.getLongArray(half * Long.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkPutLongArray() { - long[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - long[] dstArray = new long[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES); - wmem.putLongArray(0, srcArray, 0, half); - wmem.putLongArray(half * Long.BYTES, srcArray, half, half); - wmem.getLongArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - - @Test - public void checkGetFloatArray() { - float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - float[] dstArray = new float[len]; - - Memory mem = Memory.wrap(srcArray); - mem.getFloatArray(0, dstArray, 0, half); - mem.getFloatArray(half * Float.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getFloatArray(0, dstArray, 0, half); - wmem.getFloatArray(half * Float.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); + dstArray[0] = wmem.getLong(0); //get*(add) + wmem.getLongArray(1 * LB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getLong(3 * LB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * LB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test - public void checkPutFloatArray() { - float[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; + public void checkPutGetNativeShorts() { + short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - float[] dstArray = new float[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES); - wmem.putFloatArray(0, srcArray, 0, half); - wmem.putFloatArray(half * Float.BYTES, srcArray, half, half); - wmem.getFloatArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkGetDoubleArray() { - double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - double[] dstArray = new double[8]; - - Memory mem = Memory.wrap(srcArray); - mem.getDoubleArray(0, dstArray, 0, half); - mem.getDoubleArray(half * Double.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - - WritableMemory wmem = WritableMemory.writableWrap(srcArray); - wmem.getDoubleArray(0, dstArray, 0, half); - wmem.getDoubleArray(half * Double.BYTES, dstArray, half, half); - assertEquals(dstArray, srcArray); - } - - @Test - public void checkPutDoubleArray() { - double[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; - final int len = srcArray.length; - final int half = len / 2; - double[] dstArray = new double[len]; - - WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES); - wmem.putDoubleArray(0, srcArray, 0, half); - wmem.putDoubleArray(half * Double.BYTES, srcArray, half, half); - wmem.getDoubleArray(0, dstArray, 0, len); - assertEquals(dstArray, srcArray); + WritableMemory wmem = WritableMemory.allocate(len * SB, NBO); + //put + wmem.putShort(0, srcArray[0]); //put*(add, value) + wmem.putShortArray(1 * SB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putShort(3 * SB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putShort(i * SB, srcArray[i]); } //put*(add, value) + //get + short[] dstArray = new short[len]; + dstArray[0] = wmem.getShort(0); //get*(add) + wmem.getShortArray(1 * SB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getShort(3 * SB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * SB); } //get*(add) + assertEquals(srcArray, dstArray); } @Test @@ -695,7 +630,7 @@ public void checkAsWritableBufferRO() { public void checkAsBufferNonNative() { WritableMemory wmem = WritableMemory.allocate(64); wmem.putShort(0, (short) 1); - Buffer buf = wmem.asBuffer(NON_NATIVE_BYTE_ORDER); + Buffer buf = wmem.asBuffer(NNBO); assertEquals(buf.getShort(0), 256); } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java index 316f1853..a10a9fd2 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java @@ -48,31 +48,27 @@ public void checkPutGetNonNativeCharacters() { char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * CB, NNBO).asWritableBuffer(); //put - wbuf.putCharArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putChar(qtr * CB, srcArray[qtr]); //put*(add, value) - wbuf.putChar((qtr + 1) * CB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * CB); - for (int i = half; i < len; i++) { wbuf.putChar(srcArray[i]); } //put*(value) + wbuf.putChar(srcArray[0]); //put*(value) + wbuf.putCharArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putChar(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putChar(i * CB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * CB, NBO).asWritableBuffer(); for (int i = 0; i < len * CB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Character.reverseBytes(wbuf2.getChar())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); char[] dstArray = new char[len]; - wbuf.getCharArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getChar(qtr * CB); //get*(add) - dstArray[qtr + 1] = wbuf.getChar((qtr + 1) * CB); //get*(add) - wbuf.setPosition(half * CB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(); } //get*() + dstArray[0] = wbuf.getChar(); //get*() + wbuf.getCharArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getChar(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * CB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -81,31 +77,27 @@ public void checkPutGetNonNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * DB, NNBO).asWritableBuffer(); //put - wbuf.putDoubleArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putDouble(qtr * DB, srcArray[qtr]); //put*(add, value) - wbuf.putDouble((qtr + 1) * DB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * DB); - for (int i = half; i < len; i++) { wbuf.putDouble(srcArray[i]); } //put*(value) + wbuf.putDouble(srcArray[0]); //put*(value) + wbuf.putDoubleArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putDouble(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putDouble(i * DB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * DB, NBO).asWritableBuffer(); for (int i = 0; i < len * DB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wbuf2.getDouble())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); double[] dstArray = new double[len]; - wbuf.getDoubleArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getDouble(qtr * DB); //get*(add) - dstArray[qtr + 1] = wbuf.getDouble((qtr + 1) * DB); //get*(add) - wbuf.setPosition(half * DB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(); } //get*() + dstArray[0] = wbuf.getDouble(); //get*() + wbuf.getDoubleArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getDouble(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * DB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -114,31 +106,27 @@ public void checkPutGetNonNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * FB, NNBO).asWritableBuffer(); //put - wbuf.putFloatArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putFloat(qtr * FB, srcArray[qtr]); //put*(add, value) - wbuf.putFloat((qtr + 1) * FB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * FB); - for (int i = half; i < len; i++) { wbuf.putFloat(srcArray[i]); } //put*(value) + wbuf.putFloat(srcArray[0]); //put*(value) + wbuf.putFloatArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putFloat(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putFloat(i * FB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * FB, NBO).asWritableBuffer(); for (int i = 0; i < len * FB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wbuf2.getFloat())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); float[] dstArray = new float[len]; - wbuf.getFloatArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getFloat(qtr * FB); //get*(add) - dstArray[qtr + 1] = wbuf.getFloat((qtr + 1) * FB); //get*(add) - wbuf.setPosition(half * FB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(); } //get*() + dstArray[0] = wbuf.getFloat(); //get*() + wbuf.getFloatArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getFloat(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * FB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -147,31 +135,27 @@ public void checkPutGetNonNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * IB, NNBO).asWritableBuffer(); //put - wbuf.putIntArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putInt(qtr * IB, srcArray[qtr]); //put*(add, value) - wbuf.putInt((qtr + 1) * IB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * IB); - for (int i = half; i < len; i++) { wbuf.putInt(srcArray[i]); } //put*(value) + wbuf.putInt(srcArray[0]); //put*(value) + wbuf.putIntArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putInt(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putInt(i * IB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * IB, NBO).asWritableBuffer(); for (int i = 0; i < len * IB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Integer.reverseBytes(wbuf2.getInt())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); int[] dstArray = new int[len]; - wbuf.getIntArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getInt(qtr * IB); //get*(add) - dstArray[qtr + 1] = wbuf.getInt((qtr + 1) * IB); //get*(add) - wbuf.setPosition(half * IB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(); } //get*() + dstArray[0] = wbuf.getInt(); //get*() + wbuf.getIntArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getInt(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * IB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -180,31 +164,27 @@ public void checkPutGetNonNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * LB, NNBO).asWritableBuffer(); //put - wbuf.putLongArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putLong(qtr * LB, srcArray[qtr]); //put*(add, value) - wbuf.putLong((qtr + 1) * LB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * LB); - for (int i = half; i < len; i++) { wbuf.putLong(srcArray[i]); } //put*(value) + wbuf.putLong(srcArray[0]); //put*(value) + wbuf.putLongArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putLong(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putLong(i * LB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * LB, NBO).asWritableBuffer(); for (int i = 0; i < len * LB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Long.reverseBytes(wbuf2.getLong())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); long[] dstArray = new long[len]; - wbuf.getLongArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getLong(qtr * LB); //get*(add) - dstArray[qtr + 1] = wbuf.getLong((qtr + 1) * LB); //get*(add) - wbuf.setPosition(half * LB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(); } //get*() + dstArray[0] = wbuf.getLong(); //get*() + wbuf.getLongArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getLong(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * LB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -213,31 +193,27 @@ public void checkPutGetNonNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - final int qtr = len / 4; WritableBuffer wbuf = WritableMemory.allocate(len * SB, NNBO).asWritableBuffer(); //put - wbuf.putShortArray(srcArray, 0, qtr); //put*Array(src[], srcOff, len) - wbuf.putShort(qtr * SB, srcArray[qtr]); //put*(add, value) - wbuf.putShort((qtr + 1) * SB, srcArray[qtr + 1]); //put*(add, value) - wbuf.setPosition(half * SB); - for (int i = half; i < len; i++) { wbuf.putShort(srcArray[i]); } //put*(value) + wbuf.putShort(srcArray[0]); //put*(value) + wbuf.putShortArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) + wbuf.putShort(srcArray[3]); //put*(value) + for (int i = half; i < len; i++) { wbuf.putShort(i * SB, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm WritableBuffer wbuf2 = WritableMemory.allocate(len * SB, NBO).asWritableBuffer(); for (int i = 0; i < len * SB; i++) { wbuf2.putByte(wbuf.getByte()); } - wbuf.resetPosition(); wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Short.reverseBytes(wbuf2.getShort())); } //get - wbuf2.resetPosition(); + wbuf.resetPosition(); short[] dstArray = new short[len]; - wbuf.getShortArray(dstArray, 0, qtr); //get*Array(dst[], dstOff, len) - dstArray[qtr] = wbuf.getShort(qtr * SB); //get*(add) - dstArray[qtr + 1] = wbuf.getShort((qtr + 1) * SB); //get*(add) - wbuf.setPosition(half * SB); - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(); } //get*() + dstArray[0] = wbuf.getShort(); //get*() + wbuf.getShortArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) + dstArray[3] = wbuf.getShort(); //get*() + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * SB); } //get*(add) assertEquals(srcArray, dstArray); } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java index 5dc6d928..f296d430 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java @@ -52,10 +52,10 @@ public void checkPutGetNonNativeCharacters() { final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * CB, NNBO); //put - wmem.putCharArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putChar((half + i) * CB, srcArray[half + i]); //put*(add, value) - } + wmem.putChar(0, srcArray[0]); //put*(add, value) + wmem.putCharArray(1 * CB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putChar(3 * CB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putChar(i * CB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * CB, NBO); wmem.copyTo(0, wmem2, 0, len * CB); @@ -64,10 +64,10 @@ public void checkPutGetNonNativeCharacters() { } //get char[] dstArray = new char[len]; - wmem.getCharArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getChar(i * CB); //get*(add) - } + dstArray[0] = wmem.getChar(0); //get*(add) + wmem.getCharArray(1 * CB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getChar(3 * CB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * CB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -77,10 +77,11 @@ public void checkPutGetNonNativeDoubles() { final int len = srcArray.length; final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * DB, NNBO); - wmem.putDoubleArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putDouble((half + i) * DB, srcArray[half + i]); //put*(add, value) - } + //put + wmem.putDouble(0, srcArray[0]); //put*(add, value) + wmem.putDoubleArray(1 * DB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putDouble(3 * DB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putDouble(i * DB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * DB, NBO); wmem.copyTo(0, wmem2, 0, len * DB); @@ -89,10 +90,10 @@ public void checkPutGetNonNativeDoubles() { } //get double[] dstArray = new double[len]; - wmem.getDoubleArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getDouble(i * DB); //get*(add) - } + dstArray[0] = wmem.getDouble(0); //get*(add) + wmem.getDoubleArray(1 * DB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getDouble(3 * DB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * DB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -102,10 +103,11 @@ public void checkPutGetNonNativeFloats() { final int len = srcArray.length; final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * FB, NNBO); - wmem.putFloatArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putFloat((half + i) * FB, srcArray[half + i]); //put*(add, value) - } + //put + wmem.putFloat(0, srcArray[0]); //put*(add, value) + wmem.putFloatArray(1 * FB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putFloat(3 * FB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putFloat(i * FB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * FB, NBO); wmem.copyTo(0, wmem2, 0, len * FB); @@ -114,10 +116,10 @@ public void checkPutGetNonNativeFloats() { } //get float[] dstArray = new float[len]; - wmem.getFloatArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getFloat(i * FB); //get*(add) - } + dstArray[0] = wmem.getFloat(0); //get*(add) + wmem.getFloatArray(1 * FB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getFloat(3 * FB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * FB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -127,10 +129,11 @@ public void checkPutGetNonNativeInts() { final int len = srcArray.length; final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * IB, NNBO); - wmem.putIntArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putInt((half + i) * IB, srcArray[half + i]); //put*(add, value) - } + //put + wmem.putInt(0, srcArray[0]); //put*(add, value) + wmem.putIntArray(1 * IB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putInt(3 * IB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putInt(i * IB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * IB, NBO); wmem.copyTo(0, wmem2, 0, len * IB); @@ -139,10 +142,10 @@ public void checkPutGetNonNativeInts() { } //get int[] dstArray = new int[len]; - wmem.getIntArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getInt(i * IB); //get*(add) - } + dstArray[0] = wmem.getInt(0); //get*(add) + wmem.getIntArray(1 * IB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getInt(3 * IB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * IB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -152,10 +155,11 @@ public void checkPutGetNonNativeLongs() { final int len = srcArray.length; final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * LB, NNBO); - wmem.putLongArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putLong((half + i) * LB, srcArray[half + i]); //put*(add, value) - } + //put + wmem.putLong(0, srcArray[0]); //put*(add, value) + wmem.putLongArray(1 * LB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putLong(3 * LB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putLong(i * LB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * LB, NBO); wmem.copyTo(0, wmem2, 0, len * LB); @@ -164,10 +168,10 @@ public void checkPutGetNonNativeLongs() { } //get long[] dstArray = new long[len]; - wmem.getLongArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getLong(i *LB); //get*(add) - } + dstArray[0] = wmem.getLong(0); //get*(add) + wmem.getLongArray(1 * LB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getLong(3 * LB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * LB); } //get*(add) assertEquals(srcArray, dstArray); } @@ -177,10 +181,11 @@ public void checkPutGetNonNativeShorts() { final int len = srcArray.length; final int half = len / 2; WritableMemory wmem = WritableMemory.allocate(len * SB, NNBO); - wmem.putShortArray(0, srcArray, 0, half); //put*Array(add, src[], srcOff, len) - for (int i = 0; i < half; i++) { - wmem.putShort((half + i) * SB, srcArray[half + i]); //put*(add, value) - } + //put + wmem.putShort(0, srcArray[0]); //put*(add, value) + wmem.putShortArray(1 * SB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putShort(3 * SB, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putShort(i * SB, srcArray[i]); } //put*(add, value) //confirm WritableMemory wmem2 = WritableMemory.allocate(len * SB, NBO); wmem.copyTo(0, wmem2, 0, len * SB); @@ -189,10 +194,10 @@ public void checkPutGetNonNativeShorts() { } //get short[] dstArray = new short[len]; - wmem.getShortArray(0, dstArray, 0, half); //get*Array(add, dst[], dstOff, len) - for (int i = half; i < len; i++) { - dstArray[i] = wmem.getShort(i *SB); //get*(add) - } + dstArray[0] = wmem.getShort(0); //get*(add) + wmem.getShortArray(1 * SB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getShort(3 * SB); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * SB); } //get*(add) assertEquals(srcArray, dstArray); } From 072294345eca9329254e45ad0fdab0c5c88d21c1 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 3 Feb 2025 17:19:20 -0800 Subject: [PATCH 08/19] Redo var names --- .../NativeWritableBufferImplTest.java | 55 +++---- .../NativeWritableMemoryImplTest.java | 100 ++++++------- .../NonNativeWritableBufferImplTest.java | 94 ++++++------ .../NonNativeWritableMemoryImplTest.java | 137 ++++++++---------- .../memory/internal/UtilForTest.java | 10 -- 5 files changed, 179 insertions(+), 217 deletions(-) diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java index b72f0da5..c84b5b6a 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableBufferImplTest.java @@ -19,20 +19,13 @@ package org.apache.datasketches.memory.internal; -import java.lang.foreign.Arena; -import static org.apache.datasketches.memory.internal.UtilForTest.CB; -import static org.apache.datasketches.memory.internal.UtilForTest.DB; -import static org.apache.datasketches.memory.internal.UtilForTest.FB; -import static org.apache.datasketches.memory.internal.UtilForTest.IB; -import static org.apache.datasketches.memory.internal.UtilForTest.LB; -import static org.apache.datasketches.memory.internal.UtilForTest.NBO; -import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; -import static org.apache.datasketches.memory.internal.UtilForTest.SB; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import java.lang.foreign.Arena; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -72,7 +65,7 @@ public void checkPutGetBooleans() { boolean[] srcArray = {true, false, true, false, false, true, false, true}; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len, NATIVE_BYTE_ORDER).asWritableBuffer(); //put for (int i = 0; i < half; i++) { wbuf.putBoolean(srcArray[i]); } //put*(value) for (int i = half; i < len; i++) { wbuf.putBoolean(i, srcArray[i]); } //put*(add, value) @@ -89,7 +82,7 @@ public void checkPutGetBytes() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putByte(srcArray[0]); //put*(value) wbuf.putByteArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) @@ -110,19 +103,19 @@ public void checkPutGetNativeCharacters() { char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * CB, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putChar(srcArray[0]); //put*(value) wbuf.putCharArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putChar(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putChar(i * CB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putChar(i * Character.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get char[] dstArray = new char[len]; dstArray[0] = wbuf.getChar(); //get*() wbuf.getCharArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getChar(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * CB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * Character.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -131,19 +124,19 @@ public void checkPutGetNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * DB, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putDouble(srcArray[0]); //put*(value) wbuf.putDoubleArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putDouble(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putDouble(i * DB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putDouble(i * Double.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get double[] dstArray = new double[len]; dstArray[0] = wbuf.getDouble(); //get*() wbuf.getDoubleArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getDouble(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * DB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * Double.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -152,19 +145,19 @@ public void checkPutGetNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * FB, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putFloat(srcArray[0]); //put*(value) wbuf.putFloatArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putFloat(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putFloat(i * FB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putFloat(i * Float.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get float[] dstArray = new float[len]; dstArray[0] = wbuf.getFloat(); //get*() wbuf.getFloatArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getFloat(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * FB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * Float.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -173,19 +166,19 @@ public void checkPutGetNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * IB, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putInt(srcArray[0]); //put*(value) wbuf.putIntArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putInt(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putInt(i * IB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putInt(i * Integer.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get int[] dstArray = new int[len]; dstArray[0] = wbuf.getInt(); //get*() wbuf.getIntArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getInt(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * IB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * Integer.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -194,19 +187,19 @@ public void checkPutGetNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * LB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putLong(srcArray[0]); //put*(value) wbuf.putLongArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putLong(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putLong(i * LB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putLong(i * Long.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get long[] dstArray = new long[len]; dstArray[0] = wbuf.getLong(); //get*() wbuf.getLongArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getLong(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * LB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * Long.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -215,19 +208,19 @@ public void checkPutGetNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * SB, NBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putShort(srcArray[0]); //put*(value) wbuf.putShortArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putShort(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putShort(i * SB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putShort(i * Short.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //get short[] dstArray = new short[len]; dstArray[0] = wbuf.getShort(); //get*() wbuf.getShortArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getShort(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * SB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * Short.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -512,7 +505,7 @@ public void checkZeroBuffer() { public void checkDuplicateNonNative() { WritableMemory wmem = WritableMemory.allocate(64); wmem.putShort(0, (short) 1); - Buffer buf = wmem.asWritableBuffer().duplicate(NNBO); + Buffer buf = wmem.asWritableBuffer().duplicate(NON_NATIVE_BYTE_ORDER); assertEquals(buf.getShort(0), 256); } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java index 07726dd3..92fef543 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NativeWritableMemoryImplTest.java @@ -19,14 +19,8 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.UtilForTest.CB; -import static org.apache.datasketches.memory.internal.UtilForTest.DB; -import static org.apache.datasketches.memory.internal.UtilForTest.FB; -import static org.apache.datasketches.memory.internal.UtilForTest.IB; -import static org.apache.datasketches.memory.internal.UtilForTest.LB; -import static org.apache.datasketches.memory.internal.UtilForTest.NBO; -import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; -import static org.apache.datasketches.memory.internal.UtilForTest.SB; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -71,7 +65,7 @@ public void checkNativeCapacityAndClose() throws Exception { public void checkPutGetBooleans() { boolean[] srcArray = {true, false, true, false, false, true, false, true}; final int len = srcArray.length; - WritableMemory wmem = WritableMemory.allocate(len, NBO); + WritableMemory wmem = WritableMemory.allocate(len, NATIVE_BYTE_ORDER); //put for (int i = 0; i < len; i++) { wmem.putBoolean(i, srcArray[i]); } //put*(add, value) //get @@ -85,7 +79,7 @@ public void checkPutGetBytes() { byte[] srcArray = { 1, -2, 3, -4, 5, -6, 7, -8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len, NBO); + WritableMemory wmem = WritableMemory.allocate(len, NATIVE_BYTE_ORDER); //put wmem.putByte(0, srcArray[0]); //put*(add, value) wmem.putByteArray(1, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) @@ -105,18 +99,18 @@ public void checkPutGetNativeCharacters() { char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * CB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER); //put wmem.putChar(0, srcArray[0]); //put*(add, value) - wmem.putCharArray(1 * CB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putChar(3 * CB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putChar(i * CB, srcArray[i]); } //put*(add, value) + wmem.putCharArray(1 * Character.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putChar(3 * Character.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putChar(i * Character.BYTES, srcArray[i]); } //put*(add, value) //get char[] dstArray = new char[len]; dstArray[0] = wmem.getChar(0); //get*(add) - wmem.getCharArray(1 * CB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getChar(3 * CB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * CB); } //get*(add) + wmem.getCharArray(1 * Character.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getChar(3 * Character.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * Character.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -125,18 +119,18 @@ public void checkPutGetNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * DB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER); //put wmem.putDouble(0, srcArray[0]); //put*(add, value) - wmem.putDoubleArray(1 * DB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putDouble(3 * DB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putDouble(i * DB, srcArray[i]); } //put*(add, value) + wmem.putDoubleArray(1 * Double.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putDouble(3 * Double.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putDouble(i * Double.BYTES, srcArray[i]); } //put*(add, value) //get double[] dstArray = new double[len]; dstArray[0] = wmem.getDouble(0); //get*(add) - wmem.getDoubleArray(1 * DB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getDouble(3 * DB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * DB); } //get*(add) + wmem.getDoubleArray(1 * Double.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getDouble(3 * Double.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * Double.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -145,18 +139,18 @@ public void checkPutGetNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * FB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER); //put wmem.putFloat(0, srcArray[0]); //put*(add, value) - wmem.putFloatArray(1 * FB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putFloat(3 * FB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putFloat(i * FB, srcArray[i]); } //put*(add, value) + wmem.putFloatArray(1 * Float.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putFloat(3 * Float.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putFloat(i * Float.BYTES, srcArray[i]); } //put*(add, value) //get float[] dstArray = new float[len]; dstArray[0] = wmem.getFloat(0); //get*(add) - wmem.getFloatArray(1 * FB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getFloat(3 * FB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * FB); } //get*(add) + wmem.getFloatArray(1 * Float.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getFloat(3 * Float.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * Float.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -165,18 +159,18 @@ public void checkPutGetNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * IB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER); //put wmem.putInt(0, srcArray[0]); //put*(add, value) - wmem.putIntArray(1 * IB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putInt(3 * IB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putInt(i * IB, srcArray[i]); } //put*(add, value) + wmem.putIntArray(1 * Integer.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putInt(3 * Integer.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putInt(i * Integer.BYTES, srcArray[i]); } //put*(add, value) //get int[] dstArray = new int[len]; dstArray[0] = wmem.getInt(0); //get*(add) - wmem.getIntArray(1 * IB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getInt(3 * IB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * IB); } //get*(add) + wmem.getIntArray(1 * Integer.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getInt(3 * Integer.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * Integer.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -185,18 +179,18 @@ public void checkPutGetNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * LB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER); //put wmem.putLong(0, srcArray[0]); //put*(add, value) - wmem.putLongArray(1 * LB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putLong(3 * LB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putLong(i * LB, srcArray[i]); } //put*(add, value) + wmem.putLongArray(1 * Long.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putLong(3 * Long.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putLong(i * Long.BYTES, srcArray[i]); } //put*(add, value) //get long[] dstArray = new long[len]; dstArray[0] = wmem.getLong(0); //get*(add) - wmem.getLongArray(1 * LB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getLong(3 * LB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * LB); } //get*(add) + wmem.getLongArray(1 * Long.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getLong(3 * Long.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * Long.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -205,18 +199,18 @@ public void checkPutGetNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * SB, NBO); + WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER); //put wmem.putShort(0, srcArray[0]); //put*(add, value) - wmem.putShortArray(1 * SB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putShort(3 * SB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putShort(i * SB, srcArray[i]); } //put*(add, value) + wmem.putShortArray(1 * Short.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putShort(3 * Short.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putShort(i * Short.BYTES, srcArray[i]); } //put*(add, value) //get short[] dstArray = new short[len]; dstArray[0] = wmem.getShort(0); //get*(add) - wmem.getShortArray(1 * SB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getShort(3 * SB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * SB); } //get*(add) + wmem.getShortArray(1 * Short.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getShort(3 * Short.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * Short.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -630,7 +624,7 @@ public void checkAsWritableBufferRO() { public void checkAsBufferNonNative() { WritableMemory wmem = WritableMemory.allocate(64); wmem.putShort(0, (short) 1); - Buffer buf = wmem.asBuffer(NNBO); + Buffer buf = wmem.asBuffer(NON_NATIVE_BYTE_ORDER); assertEquals(buf.getShort(0), 256); } diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java index a10a9fd2..43dc54c7 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableBufferImplTest.java @@ -19,14 +19,8 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.UtilForTest.CB; -import static org.apache.datasketches.memory.internal.UtilForTest.DB; -import static org.apache.datasketches.memory.internal.UtilForTest.FB; -import static org.apache.datasketches.memory.internal.UtilForTest.IB; -import static org.apache.datasketches.memory.internal.UtilForTest.LB; -import static org.apache.datasketches.memory.internal.UtilForTest.NBO; -import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; -import static org.apache.datasketches.memory.internal.UtilForTest.SB; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -48,16 +42,16 @@ public void checkPutGetNonNativeCharacters() { char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * CB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putChar(srcArray[0]); //put*(value) wbuf.putCharArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putChar(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putChar(i * CB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putChar(i * Character.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * CB, NBO).asWritableBuffer(); - for (int i = 0; i < len * CB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Character.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Character.reverseBytes(wbuf2.getChar())); @@ -68,7 +62,7 @@ public void checkPutGetNonNativeCharacters() { dstArray[0] = wbuf.getChar(); //get*() wbuf.getCharArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getChar(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * CB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getChar(i * Character.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -77,16 +71,16 @@ public void checkPutGetNonNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * DB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putDouble(srcArray[0]); //put*(value) wbuf.putDoubleArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putDouble(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putDouble(i * DB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putDouble(i * Double.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * DB, NBO).asWritableBuffer(); - for (int i = 0; i < len * DB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Double.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wbuf2.getDouble())); @@ -97,7 +91,7 @@ public void checkPutGetNonNativeDoubles() { dstArray[0] = wbuf.getDouble(); //get*() wbuf.getDoubleArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getDouble(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * DB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getDouble(i * Double.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -106,16 +100,16 @@ public void checkPutGetNonNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * FB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putFloat(srcArray[0]); //put*(value) wbuf.putFloatArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putFloat(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putFloat(i * FB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putFloat(i * Float.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * FB, NBO).asWritableBuffer(); - for (int i = 0; i < len * FB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Float.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wbuf2.getFloat())); @@ -126,7 +120,7 @@ public void checkPutGetNonNativeFloats() { dstArray[0] = wbuf.getFloat(); //get*() wbuf.getFloatArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getFloat(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * FB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getFloat(i * Float.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -135,16 +129,16 @@ public void checkPutGetNonNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * IB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putInt(srcArray[0]); //put*(value) wbuf.putIntArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putInt(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putInt(i * IB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putInt(i * Integer.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * IB, NBO).asWritableBuffer(); - for (int i = 0; i < len * IB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Integer.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Integer.reverseBytes(wbuf2.getInt())); @@ -155,7 +149,7 @@ public void checkPutGetNonNativeInts() { dstArray[0] = wbuf.getInt(); //get*() wbuf.getIntArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getInt(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * IB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getInt(i * Integer.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -164,16 +158,16 @@ public void checkPutGetNonNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * LB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putLong(srcArray[0]); //put*(value) wbuf.putLongArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putLong(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putLong(i * LB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putLong(i * Long.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * LB, NBO).asWritableBuffer(); - for (int i = 0; i < len * LB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Long.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Long.reverseBytes(wbuf2.getLong())); @@ -184,7 +178,7 @@ public void checkPutGetNonNativeLongs() { dstArray[0] = wbuf.getLong(); //get*() wbuf.getLongArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getLong(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * LB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getLong(i * Long.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -193,16 +187,16 @@ public void checkPutGetNonNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableBuffer wbuf = WritableMemory.allocate(len * SB, NNBO).asWritableBuffer(); + WritableBuffer wbuf = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER).asWritableBuffer(); //put wbuf.putShort(srcArray[0]); //put*(value) wbuf.putShortArray(srcArray, 1, 2); //put*Array(src[], srcOff, len) wbuf.putShort(srcArray[3]); //put*(value) - for (int i = half; i < len; i++) { wbuf.putShort(i * SB, srcArray[i]); } //put*(add, value) + for (int i = half; i < len; i++) { wbuf.putShort(i * Short.BYTES, srcArray[i]); } //put*(add, value) wbuf.resetPosition(); //confirm - WritableBuffer wbuf2 = WritableMemory.allocate(len * SB, NBO).asWritableBuffer(); - for (int i = 0; i < len * SB; i++) { wbuf2.putByte(wbuf.getByte()); } + WritableBuffer wbuf2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER).asWritableBuffer(); + for (int i = 0; i < len * Short.BYTES; i++) { wbuf2.putByte(wbuf.getByte()); } wbuf2.resetPosition(); for (int i = 0; i < len; i++) { assertTrue(srcArray[i] == Short.reverseBytes(wbuf2.getShort())); @@ -213,7 +207,7 @@ public void checkPutGetNonNativeShorts() { dstArray[0] = wbuf.getShort(); //get*() wbuf.getShortArray(dstArray, 1, 2); //get*Array(dst[], dstOff, len) dstArray[3] = wbuf.getShort(); //get*() - for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * SB); } //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wbuf.getShort(i * Short.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -221,48 +215,48 @@ public void checkPutGetNonNativeShorts() { @Test public void checkDuplicate() { byte[] bArr = new byte[8]; - WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); WritableBuffer wbuf = wmem.asWritableBuffer(); WritableBuffer wdup = wbuf.writableDuplicate(); - assertEquals(wdup.getTypeByteOrder(), NNBO); + assertEquals(wdup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); WritableBuffer wreg = wbuf.writableRegion(); - assertEquals(wreg.getTypeByteOrder(), NNBO); + assertEquals(wreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); } @Test public void checkConversionByteOrder() { byte[] bArr = new byte[8]; bArr[1] = 1; - WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); - assertEquals(wmem.getTypeByteOrder(), NNBO); + WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); + assertEquals(wmem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(wmem.getChar(0), 1); Buffer buf = wmem.asBuffer(); - assertEquals(buf.getTypeByteOrder(), NNBO); // + assertEquals(buf.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); // assertEquals(buf.getChar(0), 1); Buffer dup = buf.duplicate(); - assertEquals(dup.getTypeByteOrder(), NNBO); + assertEquals(dup.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(dup.getChar(0), 1); Buffer reg = buf.region(); - assertEquals(reg.getTypeByteOrder(), NNBO); + assertEquals(reg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(reg.getChar(0), 1); Memory mem = reg.asMemory(); - assertEquals(mem.getTypeByteOrder(), NNBO); + assertEquals(mem.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(mem.getChar(0), 1); Memory mreg = mem.region(0, 8); - assertEquals(mreg.getTypeByteOrder(), NNBO); + assertEquals(mreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); assertEquals(mreg.getChar(0), 1); } @Test public void checkPutIntArray() { - WritableMemory wmem = WritableMemory.allocate(12, NNBO); - WritableBuffer wbuf = wmem.asWritableBuffer(NNBO); + WritableMemory wmem = WritableMemory.allocate(12, NON_NATIVE_BYTE_ORDER); + WritableBuffer wbuf = wmem.asWritableBuffer(NON_NATIVE_BYTE_ORDER); wbuf.putInt(1); int[] array = new int[] { 2 }; diff --git a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java index f296d430..7c0685a1 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/NonNativeWritableMemoryImplTest.java @@ -19,20 +19,11 @@ package org.apache.datasketches.memory.internal; -import static org.apache.datasketches.memory.internal.UtilForTest.CB; -import static org.apache.datasketches.memory.internal.UtilForTest.DB; -import static org.apache.datasketches.memory.internal.UtilForTest.FB; -import static org.apache.datasketches.memory.internal.UtilForTest.IB; -import static org.apache.datasketches.memory.internal.UtilForTest.LB; -import static org.apache.datasketches.memory.internal.UtilForTest.NBO; -import static org.apache.datasketches.memory.internal.UtilForTest.NNBO; -import static org.apache.datasketches.memory.internal.UtilForTest.SB; +import static org.apache.datasketches.memory.internal.ResourceImpl.NATIVE_BYTE_ORDER; +import static org.apache.datasketches.memory.internal.ResourceImpl.NON_NATIVE_BYTE_ORDER; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; - import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -41,7 +32,7 @@ */ public class NonNativeWritableMemoryImplTest { private byte[] bArr = new byte[8]; - private final WritableMemory wmem = WritableMemory.writableWrap(bArr, NNBO); + private final WritableMemory wmem = WritableMemory.writableWrap(bArr, NON_NATIVE_BYTE_ORDER); //Check primitives @@ -50,24 +41,24 @@ public void checkPutGetNonNativeCharacters() { char[] srcArray = { 'a','b','c','d','e','f','g','h' }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * CB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Character.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putChar(0, srcArray[0]); //put*(add, value) - wmem.putCharArray(1 * CB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putChar(3 * CB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putChar(i * CB, srcArray[i]); } //put*(add, value) + wmem.putCharArray(1 * Character.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putChar(3 * Character.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putChar(i * Character.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * CB, NBO); - wmem.copyTo(0, wmem2, 0, len * CB); + WritableMemory wmem2 = WritableMemory.allocate(len * Character.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Character.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Character.reverseBytes(wmem2.getChar(i * CB))); + assertTrue(srcArray[i] == Character.reverseBytes(wmem2.getChar(i * Character.BYTES))); } //get char[] dstArray = new char[len]; dstArray[0] = wmem.getChar(0); //get*(add) - wmem.getCharArray(1 * CB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getChar(3 * CB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * CB); } //get*(add) + wmem.getCharArray(1 * Character.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getChar(3 * Character.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getChar(i * Character.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -76,24 +67,24 @@ public void checkPutGetNonNativeDoubles() { double[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * DB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Double.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putDouble(0, srcArray[0]); //put*(add, value) - wmem.putDoubleArray(1 * DB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putDouble(3 * DB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putDouble(i * DB, srcArray[i]); } //put*(add, value) + wmem.putDoubleArray(1 * Double.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putDouble(3 * Double.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putDouble(i * Double.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * DB, NBO); - wmem.copyTo(0, wmem2, 0, len * DB); + WritableMemory wmem2 = WritableMemory.allocate(len * Double.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Double.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wmem2.getDouble(i * DB))); + assertTrue(srcArray[i] == UtilForTest.doubleReverseBytes(wmem2.getDouble(i * Double.BYTES))); } //get double[] dstArray = new double[len]; dstArray[0] = wmem.getDouble(0); //get*(add) - wmem.getDoubleArray(1 * DB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getDouble(3 * DB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * DB); } //get*(add) + wmem.getDoubleArray(1 * Double.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getDouble(3 * Double.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getDouble(i * Double.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -102,24 +93,24 @@ public void checkPutGetNonNativeFloats() { float[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * FB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Float.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putFloat(0, srcArray[0]); //put*(add, value) - wmem.putFloatArray(1 * FB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putFloat(3 * FB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putFloat(i * FB, srcArray[i]); } //put*(add, value) + wmem.putFloatArray(1 * Float.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putFloat(3 * Float.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putFloat(i * Float.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * FB, NBO); - wmem.copyTo(0, wmem2, 0, len * FB); + WritableMemory wmem2 = WritableMemory.allocate(len * Float.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Float.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wmem2.getFloat(i * FB))); + assertTrue(srcArray[i] == UtilForTest.floatReverseBytes(wmem2.getFloat(i * Float.BYTES))); } //get float[] dstArray = new float[len]; dstArray[0] = wmem.getFloat(0); //get*(add) - wmem.getFloatArray(1 * FB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getFloat(3 * FB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * FB); } //get*(add) + wmem.getFloatArray(1 * Float.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getFloat(3 * Float.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getFloat(i * Float.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -128,24 +119,24 @@ public void checkPutGetNonNativeInts() { int[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * IB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Integer.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putInt(0, srcArray[0]); //put*(add, value) - wmem.putIntArray(1 * IB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putInt(3 * IB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putInt(i * IB, srcArray[i]); } //put*(add, value) + wmem.putIntArray(1 * Integer.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putInt(3 * Integer.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putInt(i * Integer.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * IB, NBO); - wmem.copyTo(0, wmem2, 0, len * IB); + WritableMemory wmem2 = WritableMemory.allocate(len * Integer.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Integer.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Integer.reverseBytes(wmem2.getInt(i * IB))); + assertTrue(srcArray[i] == Integer.reverseBytes(wmem2.getInt(i * Integer.BYTES))); } //get int[] dstArray = new int[len]; dstArray[0] = wmem.getInt(0); //get*(add) - wmem.getIntArray(1 * IB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getInt(3 * IB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * IB); } //get*(add) + wmem.getIntArray(1 * Integer.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getInt(3 * Integer.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getInt(i * Integer.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -154,24 +145,24 @@ public void checkPutGetNonNativeLongs() { long[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * LB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Long.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putLong(0, srcArray[0]); //put*(add, value) - wmem.putLongArray(1 * LB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putLong(3 * LB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putLong(i * LB, srcArray[i]); } //put*(add, value) + wmem.putLongArray(1 * Long.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putLong(3 * Long.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putLong(i * Long.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * LB, NBO); - wmem.copyTo(0, wmem2, 0, len * LB); + WritableMemory wmem2 = WritableMemory.allocate(len * Long.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Long.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Long.reverseBytes(wmem2.getLong(i * LB))); + assertTrue(srcArray[i] == Long.reverseBytes(wmem2.getLong(i * Long.BYTES))); } //get long[] dstArray = new long[len]; dstArray[0] = wmem.getLong(0); //get*(add) - wmem.getLongArray(1 * LB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getLong(3 * LB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * LB); } //get*(add) + wmem.getLongArray(1 * Long.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getLong(3 * Long.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getLong(i * Long.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -180,24 +171,24 @@ public void checkPutGetNonNativeShorts() { short[] srcArray = { 1, 2, 3, 4, 5, 6, 7, 8 }; final int len = srcArray.length; final int half = len / 2; - WritableMemory wmem = WritableMemory.allocate(len * SB, NNBO); + WritableMemory wmem = WritableMemory.allocate(len * Short.BYTES, NON_NATIVE_BYTE_ORDER); //put wmem.putShort(0, srcArray[0]); //put*(add, value) - wmem.putShortArray(1 * SB, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) - wmem.putShort(3 * SB, srcArray[3]); //put*(add, value) - for (int i = half; i < len; i++) { wmem.putShort(i * SB, srcArray[i]); } //put*(add, value) + wmem.putShortArray(1 * Short.BYTES, srcArray, 1, 2); //put*Array(add, src[], srcOff, len) + wmem.putShort(3 * Short.BYTES, srcArray[3]); //put*(add, value) + for (int i = half; i < len; i++) { wmem.putShort(i * Short.BYTES, srcArray[i]); } //put*(add, value) //confirm - WritableMemory wmem2 = WritableMemory.allocate(len * SB, NBO); - wmem.copyTo(0, wmem2, 0, len * SB); + WritableMemory wmem2 = WritableMemory.allocate(len * Short.BYTES, NATIVE_BYTE_ORDER); + wmem.copyTo(0, wmem2, 0, len * Short.BYTES); for (int i = 0; i < len; i++) { - assertTrue(srcArray[i] == Short.reverseBytes(wmem2.getShort(i * SB))); + assertTrue(srcArray[i] == Short.reverseBytes(wmem2.getShort(i * Short.BYTES))); } //get short[] dstArray = new short[len]; dstArray[0] = wmem.getShort(0); //get*(add) - wmem.getShortArray(1 * SB, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) - dstArray[3] = wmem.getShort(3 * SB); //get*(add) - for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * SB); } //get*(add) + wmem.getShortArray(1 * Short.BYTES, dstArray, 1, 2); //get*Array(add, dst[], dstOff, len) + dstArray[3] = wmem.getShort(3 * Short.BYTES); //get*(add) + for (int i = half; i < len; i++) { dstArray[i] = wmem.getShort(i * Short.BYTES); } //get*(add) assertEquals(srcArray, dstArray); } @@ -207,7 +198,7 @@ public void checkPutGetNonNativeShorts() { @Test public void checkRegion() { WritableMemory wreg = wmem.writableRegion(0, wmem.getCapacity()); - assertEquals(wreg.getTypeByteOrder(), NNBO); + assertEquals(wreg.getTypeByteOrder(), NON_NATIVE_BYTE_ORDER); } } diff --git a/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java index 81be1af1..27ed4fa8 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/UtilForTest.java @@ -26,16 +26,6 @@ public class UtilForTest { - static final int BB = Byte.BYTES; - static final int CB = Character.BYTES; - static final int SB = Short.BYTES; - static final int IB = Integer.BYTES; - static final int LB = Long.BYTES; - static final int DB = Double.BYTES; - static final int FB = Float.BYTES; - static ByteOrder NNBO = NON_NATIVE_BYTE_ORDER; - static ByteOrder NBO = NATIVE_BYTE_ORDER; - //Java does not provide reverse bytes on doubles or floats static double doubleReverseBytes(double value) { From 05b76b0e6ddb7af08d3c0784677b5a2736cf06de Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sat, 8 Feb 2025 20:41:28 -0800 Subject: [PATCH 09/19] Investigate Hash bug --- .../org/apache/datasketches/memory/internal/MurmurHash3v4.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java index 3c64961b..4948464a 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java +++ b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java @@ -247,7 +247,7 @@ public static long[] hash(final MemorySegment seg, final long offsetBytes, final } //$FALL-THROUGH$ case 12: { - k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; + k2 ^= (seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL); k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); break; } From 38df8a7b734f5713b755e1999975c6dde15318b2 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 11 Feb 2025 16:50:07 -0800 Subject: [PATCH 10/19] Only change required to fix the Hash Bug. Does not include adding tests for it. --- .../apache/datasketches/memory/internal/MurmurHash3v4.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java index 4948464a..2628bc07 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java +++ b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java @@ -243,11 +243,11 @@ public static long[] hash(final MemorySegment seg, final long offsetBytes, final } case 13: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFFFL) << 32; + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFL) << 32; } //$FALL-THROUGH$ case 12: { - k2 ^= (seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL); + k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); break; } From 1902d177fef033c699acaeea5ecedd5f8a77452a Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 11 Feb 2025 17:46:45 -0800 Subject: [PATCH 11/19] Added additional MurmurHash3 tests, especially to catch the Hash bug. --- ...ash3v3Test.java => MurmurHash3v4Test.java} | 6 +- .../memory/internal/MurmurHash3v4bTest.java | 428 ++++++++++++++++++ 2 files changed, 431 insertions(+), 3 deletions(-) rename src/test/java/org/apache/datasketches/memory/internal/{MurmurHash3v3Test.java => MurmurHash3v4Test.java} (99%) create mode 100644 src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java diff --git a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java similarity index 99% rename from src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java rename to src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java index 68069003..affdebb2 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java +++ b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java @@ -20,7 +20,7 @@ package org.apache.datasketches.memory.internal; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.memory.MurmurHash3.*; +import static org.apache.datasketches.memory.MurmurHash3.hash; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -28,9 +28,9 @@ import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; -import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.WritableMemory; import org.testng.Assert; import org.testng.annotations.Test; @@ -41,7 +41,7 @@ * * @author Lee Rhodes */ -public class MurmurHash3v3Test { +public class MurmurHash3v4Test { private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr; @Test diff --git a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java new file mode 100644 index 00000000..d2c868f0 --- /dev/null +++ b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.internal; + +import java.util.Random; + +import static org.apache.datasketches.memory.MurmurHash3.hash; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import org.testng.annotations.Test; + +import org.apache.datasketches.memory.MurmurHash3; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.internal.MurmurHash3v4; +import org.apache.datasketches.memory.WritableMemory; + +/** + * @author Lee Rhodes + */ +public class MurmurHash3v4bTest { + private Random rand = new Random(); + private static final int trials = 1 << 20; + + @Test + public void compareLongArrLong() { //long[] + int arrLen = 3; + int iPer = 8 / Long.BYTES; + long[] key = new long[arrLen]; + for (int i = 0; i < trials; i++) { //trials + for (int j = 0; j < arrLen / iPer; j++) { //longs + long r = rand.nextLong(); + key[j] = r; + } + long[] res1 = hashV1(key, 0); + long[] res2 = hashV2(key, 0); + assertEquals(res2, res1); + } + } + + @Test + public void compareIntArr() { //int[] + int bytes = Integer.BYTES; + int arrLen = 6; + int[] key = new int[arrLen]; + int iPer = 8 / bytes; + int nLongs = arrLen / iPer; + int shift = 64 / iPer; + + for (int i = 0; i < trials; i++) { //trials + for (int j = 0; j < nLongs; j++) { //longs + long r = rand.nextLong(); + for (int k = 0; k < iPer; k++) { //ints + int shft = k * shift; + key[k] = (int) (r >>> shft); + } + } + long[] res1 = hashV1(key, 0); + long[] res2 = hashV2(key, 0); + assertEquals(res2, res1); + } + } + + @Test + public void compareCharArr() { //char[] + int bytes = Character.BYTES; + int arrLen = 12; + char[] key = new char[arrLen]; + int iPer = 8 / bytes; + int nLongs = arrLen / iPer; + int shift = 64 / iPer; + + for (int i = 0; i < trials; i++) { //trials + for (int j = 0; j < nLongs; j++) { //longs + long r = rand.nextLong(); + for (int k = 0; k < iPer; k++) { //char + int shft = k * shift; + key[k] = (char) (r >>> shft); + } + } + long[] res1 = hashV1(key, 0); + long[] res2 = hashV2(key, 0); + assertEquals(res2, res1); + } + } + + @Test + public void compareByteArr() { //byte[] + int bytes = Byte.BYTES; + int arrLen = 12; + byte[] key = new byte[arrLen]; + int iPer = 8 / bytes; + int nLongs = arrLen / iPer; + int shift = 64 / iPer; + + for (int i = 0; i < trials; i++) { //trials + for (int j = 0; j < nLongs; j++) { //longs + long r = rand.nextLong(); + for (int k = 0; k < iPer; k++) { //bytes + int shft = k * shift; + key[k] = (byte) (r >>> shft); + } + } + long[] res1 = hashV1(key, 0); + long[] res2 = hashV2(key, 0); + assertEquals(res2, res1); + } + } + + @Test + public void compareLongVsLongArr() { + int arrLen = 1; + long[] key = new long[arrLen]; + long[] out = new long[2]; + for (int i = 0; i < trials; i++) { //trials + long r = rand.nextLong(); + key[0] = r; + long[] res1 = hashV1(key, 0); + long[] res2 = hashV2(r, 0, out); + assertEquals(res2, res1); + } + } + + private static final long[] hashV1(long[] key, long seed) { + return MurmurHash3.hash(key, seed); + } + + private static final long[] hashV1(int[] key, long seed) { + return MurmurHash3.hash(key, seed); + } + + private static final long[] hashV1(char[] key, long seed) { + return MurmurHash3.hash(key, seed); + } + + private static final long[] hashV1(byte[] key, long seed) { + return MurmurHash3.hash(key, seed); + } + + private static final long[] hashV2(long[] key, long seed) { + return MurmurHash3v4.hash(key, seed); + } + + private static final long[] hashV2(int[] key2, long seed) { + return MurmurHash3v4.hash(key2, seed); + } + + private static final long[] hashV2(char[] key, long seed) { + return MurmurHash3v4.hash(key, seed); + } + + private static final long[] hashV2(byte[] key, long seed) { + return MurmurHash3v4.hash(key, seed); + } + + //V2 single primitives + + private static final long[] hashV2(long key, long seed, long[] out) { + return MurmurHash3v4.hash(key, seed, out); + } + +// private static final long[] hashV2(double key, long seed, long[] out) { +// return MurmurHash3v4.hash(key, seed, out); +// } + +// private static final long[] hashV2(String key, long seed, long[] out) { +// return MurmurHash3v4.hash(key, seed, out); +// } + + + + @Test + public void offsetChecks() { + long seed = 12345; + int blocks = 6; + int cap = blocks * 16; + + long[] hash1 = new long[2]; + long[] hash2; + + WritableMemory wmem = WritableMemory.allocate(cap); + for (int i = 0; i < cap; i++) { wmem.putByte(i, (byte)(-128 + i)); } + + for (int offset = 0; offset < 16; offset++) { + int arrLen = cap - offset; + hash1 = MurmurHash3v4.hash(wmem, offset, arrLen, seed, hash1); + byte[] byteArr2 = new byte[arrLen]; + wmem.getByteArray(offset, byteArr2, 0, arrLen); + hash2 = MurmurHash3.hash(byteArr2, seed); + assertEquals(hash1, hash2); + } + } + + @Test + public void byteArrChecks() { + long seed = 0; + int offset = 0; + int bytes = 1024; + + long[] hash2 = new long[2]; + + for (int j = 1; j < bytes; j++) { + byte[] in = new byte[bytes]; + + WritableMemory wmem = WritableMemory.writableWrap(in); + for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } + + long[] hash1 = MurmurHash3.hash(in, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(in, seed); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + } + } + + @Test + public void charArrChecks() { + long seed = 0; + int offset = 0; + int chars = 16; + int bytes = chars << 1; + + long[] hash2 = new long[2]; + + for (int j = 1; j < chars; j++) { + char[] in = new char[chars]; + + WritableMemory wmem = WritableMemory.writableWrap(in); + for (int i = 0; i < j; i++) { wmem.putInt(i, i); } + + long[] hash1 = MurmurHash3.hash(in, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(in, seed); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + } + } + + @Test + public void intArrChecks() { + long seed = 0; + int offset = 0; + int ints = 16; + int bytes = ints << 2; + + long[] hash2 = new long[2]; + + for (int j = 1; j < ints; j++) { + int[] in = new int[ints]; + + WritableMemory wmem = WritableMemory.writableWrap(in); + for (int i = 0; i < j; i++) { wmem.putInt(i, i); } + + long[] hash1 = MurmurHash3.hash(in, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(in, seed); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + } + } + + @Test + public void longArrChecks() { + long seed = 0; + int offset = 0; + int longs = 16; + int bytes = longs << 3; + + long[] hash2 = new long[2]; + + for (int j = 1; j < longs; j++) { + long[] in = new long[longs]; + + WritableMemory wmem = WritableMemory.writableWrap(in); + for (int i = 0; i < j; i++) { wmem.putLong(i, i); } + + long[] hash1 = MurmurHash3.hash(in, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(in, seed); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + } + } + + @Test + public void longCheck() { + long seed = 0; + int offset = 0; + int bytes = 8; + + long[] hash2 = new long[2]; + long[] in = { 1 }; + WritableMemory wmem = WritableMemory.writableWrap(in); + + long[] hash1 = MurmurHash3.hash(in, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(in, seed); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + } + + @Test + public void checkEmptiesNulls() { + long seed = 123; + long[] hashOut = new long[2]; + try { + MurmurHash3v4.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + String s = ""; + MurmurHash3v4.hash(s, seed, hashOut); //string empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + String s = null; + MurmurHash3v4.hash(s, seed, hashOut); //string null + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + byte[] barr = new byte[0]; + MurmurHash3v4.hash(barr, seed); //byte[] empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + byte[] barr = null; + MurmurHash3v4.hash(barr, seed); //byte[] null + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + char[] carr = new char[0]; + MurmurHash3v4.hash(carr, seed); //char[] empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + char[] carr = null; + MurmurHash3v4.hash(carr, seed); //char[] null + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + int[] iarr = new int[0]; + MurmurHash3v4.hash(iarr, seed); //int[] empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + int[] iarr = null; + MurmurHash3v4.hash(iarr, seed); //int[] null + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + long[] larr = new long[0]; + MurmurHash3v4.hash(larr, seed); //long[] empty + fail(); + } catch (final IllegalArgumentException e) { } //OK + try { + long[] larr = null; + MurmurHash3v4.hash(larr, seed); //long[] null + fail(); + } catch (final IllegalArgumentException e) { } //OK + } + + @Test + public void checkStringLong() { + long seed = 123; + long[] hashOut = new long[2]; + String s = "123"; + assertTrue(MurmurHash3v4.hash(s, seed, hashOut)[0] != 0); + long v = 123; + assertTrue(MurmurHash3v4.hash(v, seed, hashOut)[0] != 0); + } + + @Test + public void doubleCheck() { + long[] hash1 = checkDouble(-0.0); + long[] hash2 = checkDouble(0.0); + assertEquals(hash1, hash2); + hash1 = checkDouble(Double.NaN); + long nan = (0x7FFL << 52) + 1L; + hash2 = checkDouble(Double.longBitsToDouble(nan)); + assertEquals(hash1, hash2); + checkDouble(1.0); + } + + private static long[] checkDouble(double dbl) { + long seed = 0; + int offset = 0; + int bytes = 8; + + long[] hash2 = new long[2]; + + final double d = (dbl == 0.0) ? 0.0 : dbl; // canonicalize -0.0, 0.0 + final long data = Double.doubleToLongBits(d);// canonicalize all NaN forms + final long[] dataArr = { data }; + + WritableMemory wmem = WritableMemory.writableWrap(dataArr); + long[] hash1 = MurmurHash3.hash(dataArr, 0); + hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v4.hash(dbl, seed, hash2); + + assertEquals(hash1, hash2); + assertEquals(hash1, hash3); + return hash1; + } + +} From 9d29ac90092a9db048773045f8059523a50efc0d Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sat, 15 Feb 2025 17:52:13 -0800 Subject: [PATCH 12/19] Update how Memory Request Server works. --- .../memory/DefaultMemoryRequestServer.java | 110 +++++++++++++++--- .../apache/datasketches/memory/Memory.java | 14 +-- .../memory/MemoryRequestServer.java | 28 ++--- .../apache/datasketches/memory/Resource.java | 9 +- .../datasketches/memory/WritableMemory.java | 40 +++++-- .../memory/internal/ResourceImpl.java | 9 +- .../memory/internal/WritableMemoryImpl.java | 25 +++- .../internal/AllocateDirectMemoryTest.java | 65 +++++++++-- .../memory/internal/DruidIssue11544Test.java | 12 +- .../ExampleMemoryRequestServerTest.java | 16 ++- 10 files changed, 246 insertions(+), 82 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/DefaultMemoryRequestServer.java b/src/main/java/org/apache/datasketches/memory/DefaultMemoryRequestServer.java index d288eee5..1077cfe4 100644 --- a/src/main/java/org/apache/datasketches/memory/DefaultMemoryRequestServer.java +++ b/src/main/java/org/apache/datasketches/memory/DefaultMemoryRequestServer.java @@ -27,39 +27,121 @@ * manage continuous requests for larger or smaller memory. * This capability is only available for writable, non-file-memory-mapping resources. * + *

The operation of this implementation is controlled by three conditions:

+ *
    + *
  • origOffHeap: If true, the originally allocated WritableMemory is off-heap.
    • + * + *
  • oneArena: If true, all subsequent off-heap allocations will use the same Arena + * obtained from the original off-heap WritableMemory. Otherwise, subsequent off-heap allocations will + * use a new confined Arena created by this implementation.
    • + * + *
  • offHeap: If true, all subsequent allocations will be off-heap. + * If the originally allocated WritableMemory is on-heap, this variable is ignored.
    • + *
+ * + *

These three variables work together as follows:

+ * + *
    + *
  • If the original WritableMemory is on-heap, all subsequent allocations will also be on-heap.
    • + * + *
  • If origOffHeap = true, oneArena = true, and offHeap = true, + * all subsequent allocations will also be off-heap and associated with the original Arena. + * It is the responsibility of the user to close the original Arena using a Try-With-Resource block, or directly.
    • + * + *
  • If the original WritableMemory is off-heap, oneArena is true, and offHeap is false, + * all subsequent allocations will be on-heap. + * It is the responsibility of the user to close the original Arena using a Try-With-Resource block, or directly.
    • + * + *
  • If the original WritableMemory is off-heap, oneArena is false, and offHeap is true, + * all subsequent allocations will also be off-heap and associated with a new confined Arena assigned by this implementation. + * It is the responsibility of the user to close the original Arena using a Try-With-Resource block, or directly, + * and close the last returned new WritableMemory directly.
    • + *
+ * + *

In summary:

+ * + * + * + * + * + * + * + *
Configuration Options
Original Off-Heap OneArena OffHeap Subsequent Allocations
false N/A N/A All on-heap
true N/A false All on-heap
true true true All off-heap in original Arena
true false true All off-heap in separate Arenas
+ * * @author Lee Rhodes */ +@SuppressWarnings("resource") //can't use TWRs here public final class DefaultMemoryRequestServer implements MemoryRequestServer { + private final long alignmentBytes; + private final ByteOrder byteOrder; + private final boolean oneArena; + private final boolean offHeap; /** * Default constructor. */ - public DefaultMemoryRequestServer() { } + public DefaultMemoryRequestServer() { + alignmentBytes = 8; + byteOrder = ByteOrder.nativeOrder(); + oneArena = false; + offHeap = false; + } - @Override - public WritableMemory request( - final long newCapacityBytes, + /** + * Optional constructor 1. + * @param oneArena if true, the original arena will be used for all requested allocations. + * @param offHeap if true, new allocations will be off-heap. + */ + public DefaultMemoryRequestServer( + final boolean oneArena, + final boolean offHeap) { + this.alignmentBytes = 8; + this.byteOrder = ByteOrder.nativeOrder(); + this.oneArena = oneArena; + this.offHeap = offHeap; + } + + /** + * Optional constructor 2. + * @param alignmentBytes requested segment alignment for all allocations. Typically 1, 2, 4 or 8. + * @param byteOrder the given ByteOrder. It must be non-null. + * @param oneArena if true, the same arena will be used for all requested allocations. + * @param offHeap if true, new allocations will be off-heap. + */ + public DefaultMemoryRequestServer( final long alignmentBytes, final ByteOrder byteOrder, - final Arena arena) { - final WritableMemory newWmem; + final boolean oneArena, + final boolean offHeap) { + this.alignmentBytes = alignmentBytes; + this.byteOrder = byteOrder; + this.oneArena = oneArena; + this.offHeap = offHeap; + } - if (arena != null) { - newWmem = WritableMemory.allocateDirect(newCapacityBytes, alignmentBytes, byteOrder, this, arena); - } - else { //On-heap + @Override + public WritableMemory request( + final WritableMemory oldWmem, + final long newCapacityBytes) { + + //On-heap + if (oldWmem.getArena() == null || !offHeap) { if (newCapacityBytes > Integer.MAX_VALUE) { throw new IllegalArgumentException("Requested capacity exceeds Integer.MAX_VALUE."); } - newWmem = WritableMemory.allocate((int)newCapacityBytes, byteOrder, this); + return WritableMemory.allocate((int)newCapacityBytes, byteOrder, this); } - return newWmem; + //Acquire Arena + final Arena arena = (oneArena) ? oldWmem.getArena() : Arena.ofConfined(); + return WritableMemory.allocateDirect(newCapacityBytes, alignmentBytes, byteOrder, this, arena); } @Override - public void requestClose(final Arena arena) { - if (arena.scope().isAlive()) { arena.close(); } + public void requestClose(final WritableMemory wmemToClose) { + final Arena arena = wmemToClose.getArena(); + if (oneArena || arena == null || !arena.scope().isAlive()) { return; } //can't close + arena.close(); } } diff --git a/src/main/java/org/apache/datasketches/memory/Memory.java b/src/main/java/org/apache/datasketches/memory/Memory.java index 071e8167..3ef3d341 100644 --- a/src/main/java/org/apache/datasketches/memory/Memory.java +++ b/src/main/java/org/apache/datasketches/memory/Memory.java @@ -222,7 +222,7 @@ static Memory wrap( int lengthBytes, ByteOrder byteOrder) { final MemorySegment slice = MemorySegment.ofArray(array).asSlice(offsetBytes, lengthBytes).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(slice, byteOrder, null); + return WritableMemoryImpl.wrapSegment(slice, byteOrder); } //intentionally removed wrap(boolean[]) @@ -234,7 +234,7 @@ static Memory wrap( */ static Memory wrap(char[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -244,7 +244,7 @@ static Memory wrap(char[] array) { */ static Memory wrap(short[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -254,7 +254,7 @@ static Memory wrap(short[] array) { */ static Memory wrap(int[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -264,7 +264,7 @@ static Memory wrap(int[] array) { */ static Memory wrap(long[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -274,7 +274,7 @@ static Memory wrap(long[] array) { */ static Memory wrap(float[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -284,7 +284,7 @@ static Memory wrap(float[] array) { */ static Memory wrap(double[] array) { final MemorySegment seg = MemorySegment.ofArray(array).asReadOnly(); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } //END OF CONSTRUCTOR-TYPE METHODS diff --git a/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java b/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java index e6de60ac..cbeaea9a 100644 --- a/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java +++ b/src/main/java/org/apache/datasketches/memory/MemoryRequestServer.java @@ -19,9 +19,6 @@ package org.apache.datasketches.memory; -import java.lang.foreign.Arena; -import java.nio.ByteOrder; - /** * The MemoryRequestServer is a callback interface to provide a means to request more or less memory * for heap and off-heap WritableMemory resources that are not file-memory-mapped backed resources. @@ -33,26 +30,21 @@ public interface MemoryRequestServer { /** - * Request new WritableMemory with the given newCapacityBytes. + * Request a new WritableMemory with the given newCapacityBytes. + * @param oldWmem the previous WritableMemory to be possibly closed and which provides an associated Arena + * that may be used for allocating the new WritableMemory. + * If the arena is null, the requested WritableMemory will be on-heap. * @param newCapacityBytes The capacity being requested. - * @param alignmentBytes requested segment alignment. Typically 1, 2, 4 or 8. - * @param byteOrder the given ByteOrder. It must be non-null. - * @param arena the given arena to manage the new off-heap WritableMemory. - * If arena is null, the requested WritableMemory will be on-heap. - * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. + * * @return new WritableMemory with the requested capacity. */ - WritableMemory request( - long newCapacityBytes, - long alignmentBytes, - ByteOrder byteOrder, - Arena arena); + WritableMemory request(WritableMemory oldWmem, long newCapacityBytes); /** - * Request to close the area managing all the related resources, if applicable. - * Be careful when you request to close the given Arena, you may be closing other resources as well. - * @param arena the given arena to use to close all its managed resources. + * Request to close the given WritableMemory. If applicable, it will be closed by its associated Arena. + * Be careful. Closing the associated Arena may be closing other resources as well. + * @param wmemToClose the given WritableMemory to close. */ - void requestClose( Arena arena); + void requestClose(WritableMemory wmemToClose); } diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 701b31ec..9740b4b0 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -36,8 +36,9 @@ public interface Resource { /** * The default MemoryRequestServer used primarily by test. + * Do not allocate requested memory off-heap. */ - static final MemoryRequestServer defaultMemReqSvr = new DefaultMemoryRequestServer(); + static final MemoryRequestServer defaultMemReqSvr = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), false, false); /** * Gets the {@link MemoryRequestServer} to request additional memory @@ -253,9 +254,10 @@ boolean equalTo( boolean isRegion(); /** - * Returns true if the underlying resource is the same underlying resource as that. + * Returns true if the backing resource of this is the same as the backing resource of that. + * This returns false if this and that are both on-heap and one of them is read-only. * @param that the other Resource object - * @return a long value representing the ordering and size of overlap between this and that + * @return true if the backing resource of this is the same as the backing resource of that. */ boolean isSameResource(Resource that); @@ -322,6 +324,7 @@ boolean equalTo( * @param arena the given arena. * If the desired result is to be off-heap, the arena must not be null. * Otherwise, the result will be on-heap. + * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. * @param alignment requested segment alignment. Typically 1, 2, 4 or 8. * @return a copy of the underlying MemorySegment in the given arena. */ diff --git a/src/main/java/org/apache/datasketches/memory/WritableMemory.java b/src/main/java/org/apache/datasketches/memory/WritableMemory.java index 23d0ba00..69656ac1 100644 --- a/src/main/java/org/apache/datasketches/memory/WritableMemory.java +++ b/src/main/java/org/apache/datasketches/memory/WritableMemory.java @@ -117,11 +117,10 @@ static WritableMemory writableMap( * Allocates and provides access to capacityBytes directly in native (off-heap) memory. * The allocated memory will be 8-byte aligned. * Native byte order is assumed. - * A new DefaultMemoryRequestServer() is created. + * A new DefaultMemoryRequestServer() is created, which allocates on-heap. * - *

NOTE: Native/Direct memory acquired may have garbage in it. - * It is the responsibility of the using application to clear this memory, if required, - * and to call close() when done.

+ *

NOTE:It is the responsibility of the using application to call + * WritableMemory::getArena().close() when done.

* @param capacityBytes the size of the desired memory in bytes. * @param arena the given arena to manage the new off-heap WritableMemory. It must be non-null. * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. @@ -132,6 +131,25 @@ static WritableMemory allocateDirect(long capacityBytes, Arena arena) { return allocateDirect(capacityBytes, 8, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer(), arena); } + /** + * Allocates and provides access to capacityBytes directly in native (off-heap) memory. + * The allocated memory will be 8-byte aligned. + * Native byte order is assumed. + * + *

NOTE:It is the responsibility of the using application to call + * WritableMemory::getArena().close() when done.

+ * @param capacityBytes the size of the desired memory in bytes. + * @param memReqSvr A user-specified MemoryRequestServer, which may be null. + * This is a callback mechanism for a user client of direct memory to request more memory. + * @param arena the given arena to manage the new off-heap WritableMemory. It must be non-null. + * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. + * + * @return a WritableMemory for this off-heap resource. + */ + static WritableMemory allocateDirect(long capacityBytes, MemoryRequestServer memReqSvr, Arena arena) { + return allocateDirect(capacityBytes, 8, ByteOrder.nativeOrder(), memReqSvr, arena); + } + /** * Allocates and provides access to capacityBytes directly in native (off-heap) memory. * The allocated memory will be aligned to the given alignmentBytes. @@ -340,7 +358,7 @@ static WritableMemory writableWrap( ByteOrder byteOrder, MemoryRequestServer memReqSvr) { final MemorySegment slice = MemorySegment.ofArray(array).asSlice(offsetBytes, lengthBytes); - return WritableMemoryImpl.wrapSegmentAsArray(slice, byteOrder, memReqSvr); + return WritableMemoryImpl.wrapSegment(slice, byteOrder, memReqSvr); } //intentionally removed writableWrap(boolean[]) @@ -352,7 +370,7 @@ static WritableMemory writableWrap( */ static WritableMemory writableWrap(char[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -362,7 +380,7 @@ static WritableMemory writableWrap(char[] array) { */ static WritableMemory writableWrap(short[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -372,7 +390,7 @@ static WritableMemory writableWrap(short[] array) { */ static WritableMemory writableWrap(int[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -382,7 +400,7 @@ static WritableMemory writableWrap(int[] array) { */ static WritableMemory writableWrap(long[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -392,7 +410,7 @@ static WritableMemory writableWrap(long[] array) { */ static WritableMemory writableWrap(float[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -402,7 +420,7 @@ static WritableMemory writableWrap(float[] array) { */ static WritableMemory writableWrap(double[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } //END OF CONSTRUCTOR-TYPE METHODS diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index 459af659..d4030b47 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -453,10 +453,13 @@ public final boolean isRegion() { @Override public final boolean isSameResource(final Resource that) { - Objects.requireNonNull(that); + if (that == null) { return false; } final ResourceImpl that2 = (ResourceImpl) that; - return this.seg.address() == that2.seg.address() - && this.seg.byteSize() == that2.seg.byteSize(); + if (this.arena == null && that2.arena == null) { //both on heap + if (this.seg.isReadOnly() || that2.seg.isReadOnly()) { return false; } + return this.seg.heapBase().get() == that2.seg.heapBase().get(); + } + return this.seg.address() == that2.seg.address(); } @Override diff --git a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java index 4604e092..339c434b 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java @@ -57,16 +57,35 @@ public abstract class WritableMemoryImpl extends ResourceImpl implements Writabl super(seg, typeId, memReqSvr, arena); } - //WRAP HEAP ARRAY RESOURCE + //WRAP SEGMENT RESOURCE /** - * Wrap a MemorySegment as an array + * Wrap a MemorySegment. + * @param seg the given MemorySegment. It must be non-null. + * @param byteOrder the given ByteOrder. It must be non-null. + * @return a WritableMemory. + */ + public static WritableMemory wrapSegment( + final MemorySegment seg, + final ByteOrder byteOrder) { + Objects.requireNonNull(byteOrder, "byteOrder must be non-null"); + int type = MEMORY + | (seg.isReadOnly() ? READONLY : 0); + if (byteOrder == NON_NATIVE_BYTE_ORDER) { + type |= NONNATIVE_BO; + return new NonNativeWritableMemoryImpl(seg, type, null, null); + } + return new NativeWritableMemoryImpl(seg, type, null, null); + } + + /** + * Wrap a MemorySegment. * @param seg the given MemorySegment. It must be non-null. * @param byteOrder the given ByteOrder. It must be non-null. * @param memReqSvr the given MemoryRequestServer. It may be null. * @return a WritableMemory. */ - public static WritableMemory wrapSegmentAsArray( + public static WritableMemory wrapSegment( final MemorySegment seg, final ByteOrder byteOrder, final MemoryRequestServer memReqSvr) { diff --git a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java index dd0049e2..135d236f 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java @@ -27,6 +27,7 @@ import java.lang.foreign.Arena; import java.nio.ByteOrder; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.WritableMemory; @@ -56,28 +57,76 @@ public void simpleAllocateDirect() { @Test public void checkDefaultMemoryRequestServer() { + boolean oneArena = false; + boolean offHeap = false; + checkDefaultMemoryRequestServerVariations(false, false, false); + checkDefaultMemoryRequestServerVariations(false, false, true); + checkDefaultMemoryRequestServerVariations(false, true, false); + checkDefaultMemoryRequestServerVariations(false, true, true); + checkDefaultMemoryRequestServerVariations(true, false, false); + checkDefaultMemoryRequestServerVariations(true, false, true); + checkDefaultMemoryRequestServerVariations(true, true, false); + checkDefaultMemoryRequestServerVariations(true, true, true); + } + + private void checkDefaultMemoryRequestServerVariations(boolean origArena, boolean oneArena, boolean offHeap) { int longs1 = 32; int bytes1 = longs1 << 3; - try (Arena arena = Arena.ofConfined()) { - WritableMemory origWmem = WritableMemory.allocateDirect(bytes1, 8, ByteOrder.LITTLE_ENDIAN, Resource.defaultMemReqSvr, arena); + WritableMemory origWmem, newWmem; + + if (origArena) { + MemoryRequestServer dmrs = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), oneArena, offHeap); + try (Arena arena = Arena.ofConfined()) { + origWmem = WritableMemory.allocateDirect(bytes1, 8, ByteOrder.LITTLE_ENDIAN, dmrs, arena); + assertTrue(origWmem.isDirect()); + for (int i = 0; i < longs1; i++) { //puts data in origWmem + origWmem.putLong(i << 3, i); + assertEquals(origWmem.getLong(i << 3), i); + } + println(origWmem.toString("Test", 0, longs1 << 3, true)); + + int longs2 = 2 * longs1; + int bytes2 = longs2 << 3; + MemoryRequestServer myMemReqSvr = origWmem.getMemoryRequestServer(); + + newWmem = myMemReqSvr.request(origWmem, bytes2); + assertTrue( (offHeap && origArena) ? newWmem.isDirect() : newWmem.isHeap() ); + for (int i = 0; i < longs2; i++) { + newWmem.putLong(i << 3, i); + assertEquals(newWmem.getLong(i << 3), i); + } + println(newWmem.toString("Test", 0, longs2 << 3, true)); + if (oneArena && offHeap) { assertTrue((newWmem.getArena() == origWmem.getArena()) && origWmem != null); } + if (oneArena && !offHeap) { assertTrue((newWmem.getArena() == null) && origWmem != null); } + if (!oneArena && offHeap) { assertTrue((newWmem.getArena() != origWmem.getArena()) && origWmem != null); } + } //allow the TWR to close the origWmem resource + assertFalse(origWmem.getArena().scope().isAlive()); + if (!oneArena && offHeap) { + newWmem.getArena().close(); + assertFalse(newWmem.getArena().scope().isAlive()); + } + + } else { + MemoryRequestServer dmrs = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), oneArena, offHeap); + origWmem = WritableMemory.allocate(bytes1,ByteOrder.LITTLE_ENDIAN, dmrs); for (int i = 0; i < longs1; i++) { //puts data in origWmem origWmem.putLong(i << 3, i); assertEquals(origWmem.getLong(i << 3), i); } - println(origWmem.toString("Test", 0, 32 * 8, true)); + println(origWmem.toString("Test", 0, longs1 << 3, true)); - int longs2 = 64; + int longs2 = 2 * longs1; int bytes2 = longs2 << 3; - origWmem.setMemoryRequestServer(Resource.defaultMemReqSvr); MemoryRequestServer myMemReqSvr = origWmem.getMemoryRequestServer(); - WritableMemory newWmem = myMemReqSvr.request(bytes2, 8, ByteOrder.LITTLE_ENDIAN, null); //null -> on-heap - assertTrue(newWmem.isHeap()); + newWmem = myMemReqSvr.request(origWmem, bytes2); + assertTrue( (offHeap && origArena) ? newWmem.isDirect() : newWmem.isHeap() ); for (int i = 0; i < longs2; i++) { newWmem.putLong(i << 3, i); assertEquals(newWmem.getLong(i << 3), i); } - } //allow the TWR to close all resources + println(newWmem.toString("Test", 0, longs2 << 3, true)); + } } @Test diff --git a/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java b/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java index a9e4d631..ea116a82 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java +++ b/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java @@ -59,26 +59,26 @@ public void withByteBuffer() { //Wrap bb into WritableMemory WritableMemory mem1 = WritableMemory.writableWrap(bb); - //ByteBuffers are automatically assigned an implicit shared scope (non-closeable) + //ByteBuffers are not directly closeable. They are closed by the GC. assertTrue(mem1.isDirect()); //confirm mem1 is off-heap - + assertTrue(mem1.getArena() == null); //and Arena is null //Request Bigger Memory on heap int size2 = size1 * 2; - WritableMemory mem2 = myMemReqSvr.request(size2, 8, ByteOrder.LITTLE_ENDIAN, null); + WritableMemory mem2 = myMemReqSvr.request(mem1, size2); - //Confirm that mem2 is on the heap (the default) and 2X size1 + //Confirm that mem2 is on the heap and 2X size1 assertFalse(mem2.isDirect()); assertEquals(mem2.getCapacity(), size2); //Move data to new memory mem1.copyTo(0, mem2, 0, size1); - assertTrue(mem1.isAlive()); + assertTrue(mem1.isAlive()); //because mem1 seg is holding a reference to it. assertTrue(mem2.isAlive()); //Now we are on the heap and need to grow again: int size3 = size2 * 2; - WritableMemory mem3 = myMemReqSvr.request(size3, 8, ByteOrder.LITTLE_ENDIAN, null); + WritableMemory mem3 = myMemReqSvr.request(mem2, size3); //Confirm that mem3 is still on the heap and 2X of size2 assertFalse(mem3.isDirect()); diff --git a/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java b/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java index 55dd7884..d24005f0 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java @@ -24,6 +24,7 @@ import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -35,7 +36,6 @@ * @author Lee Rhodes */ public class ExampleMemoryRequestServerTest { - private static long alignmentBytes = 8; /** * This version is without a TWR block. All of the memory allocations are done through the MemoryRequestServer @@ -48,15 +48,13 @@ public void checkExampleMemoryRequestServer1() { long workingMemBytes = 8; Arena arena = Arena.ofConfined(); - //Configure the default memReqSvr to create new memory off-heap and copy data from old to new - MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + //Configure the memReqSvr to create new subsequent allocations off-heap, each with a new Arena. + MemoryRequestServer myMemReqSvr = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), false, true); //Create the initial working memory for the client WritableMemory workingMem = WritableMemory.allocateDirect( workingMemBytes, - alignmentBytes, - ByteOrder.nativeOrder(), - memReqSvr, + myMemReqSvr, arena); MemoryHungryClient client = new MemoryHungryClient(workingMem); @@ -95,14 +93,14 @@ void process() { oldWorkingCap = newWorkingCap; newWorkingCap = 2 * oldWorkingCap; Arena arena = Arena.ofConfined(); // new confined scope for each iteration - newMem = memReqSvr.request(newWorkingCap, alignmentBytes, ByteOrder.LITTLE_ENDIAN, arena); + newMem = memReqSvr.request(workingMem, newWorkingCap); //done with old memory, close it - memReqSvr.requestClose(workingMem.getArena()); + memReqSvr.requestClose(workingMem); workingMem = newMem; itr++; } - + //close the last allocation workingMem.getArena().close(); } } From b529f867273e14413fd9190d5189dc9cbb2419b5 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 16 Feb 2025 21:53:53 -0800 Subject: [PATCH 13/19] Revised isSameResource() --- .../apache/datasketches/memory/Resource.java | 11 +++++++--- .../memory/internal/ResourceImpl.java | 22 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 9740b4b0..631e90ce 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -254,10 +254,15 @@ boolean equalTo( boolean isRegion(); /** - * Returns true if the backing resource of this is the same as the backing resource of that. - * This returns false if this and that are both on-heap and one of them is read-only. + * Returns true if the underlying resource is the same underlying resource as that. + * + *

Note: for on-heap resources neither this nor that can be read-only.

+ * + *

if two sub-regions (or slices) are derived from the same resource they both must have the same + * starting offset with respect to the resource and the same size.

+ * * @param that the other Resource object - * @return true if the backing resource of this is the same as the backing resource of that. + * @return true if the underlying resource is the same underlying resource as that. */ boolean isSameResource(Resource that); diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index d4030b47..e0b17ffc 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -453,13 +453,23 @@ public final boolean isRegion() { @Override public final boolean isSameResource(final Resource that) { - if (that == null) { return false; } - final ResourceImpl that2 = (ResourceImpl) that; - if (this.arena == null && that2.arena == null) { //both on heap - if (this.seg.isReadOnly() || that2.seg.isReadOnly()) { return false; } - return this.seg.heapBase().get() == that2.seg.heapBase().get(); + Objects.requireNonNull(that); + final MemorySegment thisSeg = this.seg; + final MemorySegment thatSeg = ((ResourceImpl)that).seg; + final boolean thisNative = thisSeg.isNative(); + final boolean thatNative = thatSeg.isNative(); + if (thisNative != thatNative) { return false; } + if (thisNative && thatNative) { //off-heap + return thisSeg.address() == thatSeg.address() + && thisSeg.byteSize() == thatSeg.byteSize(); + } else { //on heap + if (thisSeg.isReadOnly() || thatSeg.isReadOnly()) { + throw new IllegalArgumentException("Cannot determine 'is same resource' on heap if one resource is Read-only."); + } + return (thisSeg.heapBase().get() == thatSeg.heapBase().get()) + && (thisSeg.address() == thatSeg.address()) + && thisSeg.byteSize() == thatSeg.byteSize(); } - return this.seg.address() == that2.seg.address(); } @Override From 9a3b29be6fad0d1139252dc9b835a15682c710de Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 17 Feb 2025 10:19:14 -0800 Subject: [PATCH 14/19] Improve javadoc for 'isSameResource(...)' --- .../org/apache/datasketches/memory/Resource.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 631e90ce..1144064f 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -254,14 +254,16 @@ boolean equalTo( boolean isRegion(); /** - * Returns true if the underlying resource is the same underlying resource as that. + * Returns true if the underlying resource is the same resource as that. * - *

Note: for on-heap resources neither this nor that can be read-only.

+ *

Two resources are considered the same if one were to write a value at offset A in one resource + * and that same value appears in the other resource at the same offset A. In other words, + * if two regions (or slices) are derived from the same underlying resource they both must have the same + * starting offset with respect to the resource and the same size in order to be considered to be the same resource.

* - *

if two sub-regions (or slices) are derived from the same resource they both must have the same - * starting offset with respect to the resource and the same size.

+ *

Note: for on-heap resources neither this nor that can be read-only.

* - * @param that the other Resource object + * @param that the other Resource. * @return true if the underlying resource is the same underlying resource as that. */ boolean isSameResource(Resource that); From 6f8adeac502441def4848afb7e59ae9327b08854 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 17 Feb 2025 10:23:04 -0800 Subject: [PATCH 15/19] improve error comment. --- .../org/apache/datasketches/memory/internal/ResourceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index e0b17ffc..00190eb2 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -464,7 +464,7 @@ public final boolean isSameResource(final Resource that) { && thisSeg.byteSize() == thatSeg.byteSize(); } else { //on heap if (thisSeg.isReadOnly() || thatSeg.isReadOnly()) { - throw new IllegalArgumentException("Cannot determine 'is same resource' on heap if one resource is Read-only."); + throw new IllegalArgumentException("Cannot determine 'isSameResource(..)' on heap if either resource is Read-only."); } return (thisSeg.heapBase().get() == thatSeg.heapBase().get()) && (thisSeg.address() == thatSeg.address()) From 602eac2347c57ce7bcbd584b081dbaacc1a8e288 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 17 Feb 2025 12:05:43 -0800 Subject: [PATCH 16/19] Allow getMemorySegment as read-only. --- .../apache/datasketches/memory/Resource.java | 35 +++++++++++-------- .../memory/internal/ResourceImpl.java | 5 +++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 1144064f..20cc9a6d 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -157,11 +157,18 @@ boolean equalTo( long getCapacity(); /** - * Gets the relative base offset of this with respect to that, defined as: this - that. - * This method is only valid for native (off-heap) allocated resources. + * Gets the MemorySegment that backs this resource as a read-only MemorySegment. + * @return the MemorySegment that back this resource as a read-only MemorySegment. + */ + MemorySegment getMemorySegment(); + + /** + * Gets the relative base offset of this resource with respect to that resource, + * defined as: this - that. * @param that the given resource. * @return this - that offset - * @throws IllegalArgumentException if one of the resources is on-heap. + * @throws UnsupportedOperationException if the two resources cannot be compared, e.g. because they are of + * different kinds, or because they are backed by different Java arrays. */ long getRelativeOffset(Resource that); @@ -274,17 +281,6 @@ boolean equalTo( */ void load(); - /** - * Returns a positive number if this overlaps that and this base address is ≤ that - * base address. - * Returns a negative number if this overlaps that and this base address is > that - * base address. - * Returns a zero if there is no overlap or if one or both objects are null, not active or on heap. - * @param that the other Resource object - * @return a long value representing the ordering and size of overlap between this and that. - */ - long nativeOverlap(Resource that); - /** * Finds the first byte mismatch with that. * @param that the other Resource @@ -311,6 +307,17 @@ boolean equalTo( */ long mismatch(Resource src, long srcFromOffset, long srcToOffset, Resource dst, long dstFromOffset, long dstToOffset); + /** + * Returns a positive number if this overlaps that and this base address is ≤ that + * base address. + * Returns a negative number if this overlaps that and this base address is > that + * base address. + * Returns a zero if there is no overlap or if one or both objects are null, not active or on heap. + * @param that the other Resource object + * @return a long value representing the ordering and size of overlap between this and that. + */ + long nativeOverlap(Resource that); + /** * Returns the resource scope associated with this memory segment. * @return the resource scope associated with this memory segment. diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index 00190eb2..5321c8fd 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -379,6 +379,11 @@ public final long getCapacity() { return seg.byteSize(); } + @Override + public MemorySegment getMemorySegment() { + return seg.asReadOnly(); + } + @Override public final long getRelativeOffset(final Resource that) { final ResourceImpl that2 = (ResourceImpl) that; From c0b19797de4f32f1101134bc0bc0790abce13177 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 17 Feb 2025 21:22:43 -0800 Subject: [PATCH 17/19] Improve javadocs --- src/main/java/org/apache/datasketches/memory/Resource.java | 2 +- .../org/apache/datasketches/memory/internal/ResourceImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 20cc9a6d..986c471f 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -146,7 +146,7 @@ boolean equalTo( /** * Returns the arena used to create this resource and possibly other resources. * Be careful when you close the returned Arena, you may be closing other resources as well. - * @return the arena used to create this resource and possibly other resources. + * @return the arena used to create this resource and possibly other resources. It may be null. */ Arena getArena(); diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index 5321c8fd..f2e271f2 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -106,7 +106,7 @@ abstract class ResourceImpl implements Resource { /** * Root constructor. - * @param seg the given, one and only one MemorySegment + * @param seg the given, one and only MemorySegment * @param typeId the given typeId * @param memReqSvr the given MemoryRequestServer, or null. * @param arena the given Arena, or null if an on-heap MemorySegment. From 52ac28c91203cd27d11f55ee328a3a52a29fffeb Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 17 Feb 2025 22:21:00 -0800 Subject: [PATCH 18/19] Move MurmurHash3 out of Memory into ds-java. Move the tests into ds-java. --- .../datasketches/memory/MurmurHash3.java | 196 -------- .../memory/internal/MurmurHash3v4.java | 386 ---------------- .../memory/internal/MurmurHash3v4Test.java | 419 ----------------- .../memory/internal/MurmurHash3v4bTest.java | 428 ------------------ 4 files changed, 1429 deletions(-) delete mode 100644 src/main/java/org/apache/datasketches/memory/MurmurHash3.java delete mode 100644 src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java delete mode 100644 src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java delete mode 100644 src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java diff --git a/src/main/java/org/apache/datasketches/memory/MurmurHash3.java b/src/main/java/org/apache/datasketches/memory/MurmurHash3.java deleted file mode 100644 index bd821c72..00000000 --- a/src/main/java/org/apache/datasketches/memory/MurmurHash3.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory; - -import java.lang.foreign.MemorySegment; - -import org.apache.datasketches.memory.internal.MurmurHash3v4; - -/** - * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has - * excellent avalanche and 2-way bit independence properties. - * - *

Austin Appleby's C++ - * - * MurmurHash3_x64_128(...), final revision 150, - * which is in the Public Domain, was the inspiration for this implementation in Java.

- * - *

This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset - * and length. The calling API also allows the user to supply the small output array of two longs, - * so that the entire hash function is static and free of object allocations.

- * - *

This implementation produces exactly the same hash result as the - * MurmurHash3 function in datasketches-java given compatible inputs.

- * - *

This version 4 of the implementation leverages the java.lang.foreign package of JDK-21 in place of - * the Unsafe class. - * - * @author Lee Rhodes - */ -public final class MurmurHash3 { - - private MurmurHash3() { } - - //Provided for backward compatibility - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in long array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final long[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in int array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final int[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in char array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final char[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in byte array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final byte[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - //Single primitive inputs - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a long - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final long in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a double - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final double in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * An empty or null input throws IllegalArgumentException. - * @param in a String - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final String in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - //The main API calls - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param mem The input on-heap Memory. Must be non-null and non-empty. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash( - final Memory mem, - final long offsetBytes, - final long lengthBytes, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(mem, offsetBytes, lengthBytes, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param seg The input MemorySegment. Must be non-null and non-empty. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash( - final MemorySegment seg, - final long offsetBytes, - final long lengthBytes, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(seg, offsetBytes, lengthBytes, seed, hashOut); - } - -} diff --git a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java deleted file mode 100644 index 2628bc07..00000000 --- a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Objects; - -import org.apache.datasketches.memory.Memory; - -/** - * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has - * excellent avalanche and 2-way bit independence properties. - * - *

Austin Appleby's C++ - * - * MurmurHash3_x64_128(...), final revision 150, - * which is in the Public Domain, was the inspiration for this implementation in Java.

- * - *

This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset - * and length. The calling API also allows the user to supply the small output array of two longs, - * so that the entire hash function is static and free of object allocations.

- * - *

This implementation produces exactly the same hash result as the - * MurmurHash3 function in datasketches-java given compatible inputs.

- * - *

This version 4 of the implementation leverages the java.lang.foreign package of JDK-21 in place of - * the Unsafe class. - * - * @author Lee Rhodes - */ -public final class MurmurHash3v4 { - private static final long C1 = 0x87c37b91114253d5L; - private static final long C2 = 0x4cf5ad432745937fL; - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in long array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final long[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 3, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in int array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final int[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 2, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in char array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final char[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 1, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in byte array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final byte[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length, seed, new long[2]); - } - - //Single primitive inputs - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a long - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash(final long in, final long seed, final long[] hashOut) { - final long h1 = seed ^ mixK1(in); - final long h2 = seed; - return finalMix128(h1, h2, 8, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a double - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash(final double in, final long seed, final long[] hashOut) { - final double d = (in == 0.0) ? 0.0 : in; // canonicalize -0.0, 0.0 - final long k1 = Double.doubleToLongBits(d); // canonicalize all NaN forms - final long h1 = seed ^ mixK1(k1); - final long h2 = seed; - return finalMix128(h1, h2, 8, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * An empty or null input throws IllegalArgumentException. - * @param in a String - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final String in, final long seed, final long[] hashOut) { - if ((in == null) || (in.length() == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - final byte[] byteArr = in.getBytes(UTF_8); - return hash(MemorySegment.ofArray(byteArr), 0L, byteArr.length, seed, hashOut); - } - - //The main API calls - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param mem The input Memory. Must be non-null and non-empty, - * otherwise throws IllegalArgumentException. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash(final Memory mem, final long offsetBytes, final long lengthBytes, - final long seed, final long[] hashOut) { - Objects.requireNonNull(mem, "Input Memory is null"); - final MemorySegment seg = ((ResourceImpl)mem).seg; - return hash(seg, offsetBytes, lengthBytes, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param seg The input MemorySegment. Must be non-null and non-empty, - * otherwise throws IllegalArgumentException. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - * @throws IllegalArgumentException if input MemorySegment is empty - */ - public static long[] hash(final MemorySegment seg, final long offsetBytes, final long lengthBytes, - final long seed, final long[] hashOut) { - Objects.requireNonNull(seg, "Input MemorySegment is null"); - if (seg.byteSize() == 0L) { throw new IllegalArgumentException("Input MemorySegment is empty."); } - - long cumOff = offsetBytes; - - long h1 = seed; - long h2 = seed; - long rem = lengthBytes; - - // Process the 128-bit blocks (the body) into the hash - while (rem >= 16L) { - final long k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); //0, 16, 32, ... - final long k2 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff + 8); //8, 24, 40, ... - cumOff += 16L; - rem -= 16L; - - h1 ^= mixK1(k1); - h1 = Long.rotateLeft(h1, 27); - h1 += h2; - h1 = (h1 * 5) + 0x52dce729L; - - h2 ^= mixK2(k2); - h2 = Long.rotateLeft(h2, 31); - h2 += h1; - h2 = (h2 * 5) + 0x38495ab5L; - } - - // Get the tail (if any): 1 to 15 bytes - if (rem > 0L) { - long k1 = 0; - long k2 = 0; - switch ((int) rem) { - case 15: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 14) & 0xFFL) << 48; - } - //$FALL-THROUGH$ - case 14: { - k2 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 12) & 0xFFFFL) << 32; - k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 13: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFL) << 32; - } - //$FALL-THROUGH$ - case 12: { - k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 11: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 10) & 0xFFL) << 16; - } - //$FALL-THROUGH$ - case 10: { - k2 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 8) & 0xFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 9: { - k2 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff + 8) & 0xFFL; - } - //$FALL-THROUGH$ - case 8: { - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 7: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 6) & 0xFFL) << 48; - } - //$FALL-THROUGH$ - case 6: { - k1 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 4) & 0xFFFFL) << 32; - k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; - break; - } - - case 5: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 4) & 0xFFL) << 32; - } - //$FALL-THROUGH$ - case 4: { - k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; - break; - } - - case 3: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 2) & 0xFFL) << 16; - } - //$FALL-THROUGH$ - case 2: { - k1 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff) & 0xFFFFL; - break; - } - - case 1: { - k1 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff) & 0xFFL; - break; - } - default: break; //can't happen - } - - h1 ^= mixK1(k1); - h2 ^= mixK2(k2); - } - return finalMix128(h1, h2, lengthBytes, hashOut); - } - - //--Helper methods---------------------------------------------------- - - /** - * Self mix of k1 - * - * @param k1 input argument - * @return mix - */ - private static long mixK1(long k1) { - k1 *= C1; - k1 = Long.rotateLeft(k1, 31); - k1 *= C2; - return k1; - } - - /** - * Self mix of k2 - * - * @param k2 input argument - * @return mix - */ - private static long mixK2(long k2) { - k2 *= C2; - k2 = Long.rotateLeft(k2, 33); - k2 *= C1; - return k2; - } - - /** - * Final self mix of h*. - * - * @param h input to final mix - * @return mix - */ - private static long finalMix64(long h) { - h ^= h >>> 33; - h *= 0xff51afd7ed558ccdL; - h ^= h >>> 33; - h *= 0xc4ceb9fe1a85ec53L; - h ^= h >>> 33; - return h; - } - - /** - * Finalization: Add the length into the hash and mix - * @param h1 intermediate hash - * @param h2 intermediate hash - * @param lengthBytes the length in bytes - * @param hashOut the output array of 2 longs - * @return hashOut - */ - private static long[] finalMix128(long h1, long h2, final long lengthBytes, final long[] hashOut) { - h1 ^= lengthBytes; - h2 ^= lengthBytes; - - h1 += h2; - h2 += h1; - - h1 = finalMix64(h1); - h2 = finalMix64(h2); - - h1 += h2; - h2 += h1; - - hashOut[0] = h1; - hashOut[1] = h2; - return hashOut; - } - - private MurmurHash3v4() { } - -} diff --git a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java deleted file mode 100644 index affdebb2..00000000 --- a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4Test.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.memory.MurmurHash3.hash; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; - -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.Resource; -import org.apache.datasketches.memory.WritableMemory; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * Tests the MurmurHash3 against specific, known hash results given known - * inputs obtained from the public domain C++ version 150. - * - * @author Lee Rhodes - */ -public class MurmurHash3v4Test { - private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr; - - @Test - public void checkByteArrRemainderGT8() { //byte[], remainder > 8 - String keyStr = "The quick brown fox jumps over the lazy dog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe34bbc7bbc071b6cL; - long h2 = 0x7a433ca9c49a9347L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrRemainderGT8withSegment() { //byte[], remainder > 8 - String keyStr = "The quick brown fox jumps over the lazy dog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] out = new long[2]; - MemorySegment seg = MemorySegment.ofArray(key); - long[] result = hash(seg, 0, seg.byteSize(), 0, out); - //Should be: - long h1 = 0xe34bbc7bbc071b6cL; - long h2 = 0x7a433ca9c49a9347L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrChange1bit() { //byte[], change one bit - String keyStr = "The quick brown fox jumps over the lazy eog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x362108102c62d1c9L; - long h2 = 0x3285cd100292b305L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrRemainderLt8() { //byte[], test a remainder < 8 - String keyStr = "The quick brown fox jumps over the lazy dogdogdog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be; - long h1 = 0x9c8205300e612fc4L; - long h2 = 0xcbc0af6136aa3df9L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrReaminderEQ8() { //byte[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test should have the exact same output as Test4 - */ - @Test - public void checkLongArrRemainderEQ8() { //long[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; - long[] key = stringToLongs(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test should have the exact same output as Test4 - */ - @Test - public void checkIntArrRemainderEQ8() { //int[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; //40B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkIntArrRemainderEQ0() { //int[], test a remainder = 0 - String keyStr = "The quick brown fox jumps over t"; //32B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xdf6af91bb29bdacfL; - long h2 = 0x91a341c58df1f3a6L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkIntArrOddRemainder() { //int[], odd remainder - String keyStr = "The quick brown fox jumps over the lazy dog"; //43B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x1eb232b0087543f5L; - long h2 = 0xfc4c1383c3ace40fL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkCharArrOddRemainder() { //char[], odd remainder - String keyStr = "The quick brown fox jumps over the lazy dog.."; //45B - char[] key = keyStr.toCharArray(); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xca77b498ea9ed953L; - long h2 = 0x8b8f8ec3a8f4657eL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkCharArrRemainderEQ0() { //char[], remainder of 0 - String keyStr = "The quick brown fox jumps over the lazy "; //40B - char[] key = keyStr.toCharArray(); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x51b15e9d0887f9f1L; - long h2 = 0x8106d226786511ebL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrAllOnesZeros() { //byte[], test a ones byte and a zeros byte - byte[] key = { - 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, - 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, 0x6f, 0x76, 0x65, - 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67, - (byte) 0xff, 0x64, 0x6f, 0x67, 0x00 - }; - long[] result = MurmurHash3v4.hash(key, 0); - //Should be: - long h1 = 0xe88abda785929c9eL; - long h2 = 0x96b98587cacc83d6L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test demonstrates that the hash of byte[], char[], int[], or long[] will produce the - * same hash result if, and only if, all the arrays have the same exact length in bytes, and if - * the contents of the values in the arrays have the same byte endianness and overall order. - */ - @Test - public void checkCrossTypeHashConsistency() { - long[] out; - println("Bytes"); - byte[] bArr = {1,2,3,4,5,6,7,8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24}; - long[] out1 = hash(bArr, 0L); - println(longToHexBytes(out1[0])); - println(longToHexBytes(out1[1])); - - println("Chars"); - char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807, 0X0a09, 0X0c0b, 0X0e0d, 0X100f, - 0X1211, 0X1413, 0X1615, 0X1817}; - out = hash(cArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - - println("Ints"); - int[] iArr = {0X04030201, 0X08070605, 0X0c0b0a09, 0X100f0e0d, 0X14131211, 0X18171615}; - out = hash(iArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - - println("Longs"); - long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L}; - out = hash(lArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - } - - @Test - public void checkEmptyOrNullExceptions() { - try { - long[] arr = null; - hash(arr, 1L); - fail(); - } - catch (final IllegalArgumentException e) { } - try { - int[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - try { - char[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - try { - byte[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - - long[] out = new long[2]; - try { - String in = null; hash(in, 1L, out); fail(); - } catch (final IllegalArgumentException e) { } - try { - Memory mem = Memory.wrap(new byte[0]); - hash(mem, 0L, 4L, 1L, out); - } catch (final IllegalArgumentException e) { } - try (Arena arena = Arena.ofConfined()) { - Memory mem = WritableMemory.allocateDirect(8, 1, ByteOrder.nativeOrder(), memReqSvr, arena); - out = hash(mem, 0L, 4L, 1L, out); - } - assertTrue((out[0] != 0) && (out[1] != 0)); - } - - @Test - public void checkHashTails() { - long[] out = new long[2]; - WritableMemory mem = WritableMemory.allocate(32); - mem.fill((byte)85); - - for (int i = 16; i <= 32; i++) { - out = hash(mem, 0, i, 1L, out); - } - } - - @Test - public void checkSinglePrimitives() { - long[] out = new long[2]; - out = hash(1L, 1L, out); - out = hash(0.0, 1L, out); - out = hash("123", 1L, out); - } - - //Helper methods - - private static long[] stringToLongs(String in) { - byte[] bArr = in.getBytes(UTF_8); - int inLen = bArr.length; - int outLen = (inLen / 8) + (((inLen % 8) != 0) ? 1 : 0); - long[] out = new long[outLen]; - - for (int i = 0; i < (outLen - 1); i++ ) { - for (int j = 0; j < 8; j++ ) { - out[i] |= ((bArr[(i * 8) + j] & 0xFFL) << (j * 8)); - } - } - int inTail = 8 * (outLen - 1); - int rem = inLen - inTail; - for (int j = 0; j < rem; j++ ) { - out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); - } - return out; - } - - private static int[] stringToInts(String in) { - byte[] bArr = in.getBytes(UTF_8); - int inLen = bArr.length; - int outLen = (inLen / 4) + (((inLen % 4) != 0) ? 1 : 0); - int[] out = new int[outLen]; - - for (int i = 0; i < (outLen - 1); i++ ) { - for (int j = 0; j < 4; j++ ) { - out[i] |= ((bArr[(i * 4) + j] & 0xFFL) << (j * 8)); - } - } - int inTail = 4 * (outLen - 1); - int rem = inLen - inTail; - for (int j = 0; j < rem; j++ ) { - out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); - } - return out; - } - - /** - * Returns a string of spaced hex bytes in Big-Endian order. - * @param v the given long - * @return string of spaced hex bytes in Big-Endian order. - */ - private static String longToHexBytes(final long v) { - final long mask = 0XFFL; - final StringBuilder sb = new StringBuilder(); - for (int i = 8; i-- > 0; ) { - final String s = Long.toHexString((v >>> (i * 8)) & mask); - sb.append(zeroPad(s, 2)).append(" "); - } - return sb.toString(); - } - - /** - * Prepend the given string with zeros. If the given string is equal or greater than the given - * field length, it will be returned without modification. - * @param s the given string - * @param fieldLength desired total field length including the given string - * @return the given string prepended with zeros. - */ - private static final String zeroPad(final String s, final int fieldLength) { - return characterPad(s, fieldLength, '0', false); - } - - /** - * Prepend or postpend the given string with the given character to fill the given field length. - * If the given string is equal or greater than the given field length, it will be returned - * without modification. - * @param s the given string - * @param fieldLength the desired field length - * @param padChar the desired pad character - * @param postpend if true append the padCharacters to the end of the string. - * @return prepended or postpended given string with the given character to fill the given field - * length. - */ - private static final String characterPad(final String s, final int fieldLength, final char padChar, - final boolean postpend) { - final char[] chArr = s.toCharArray(); - final int sLen = chArr.length; - if (sLen < fieldLength) { - final char[] out = new char[fieldLength]; - final int blanks = fieldLength - sLen; - - if (postpend) { - for (int i = 0; i < sLen; i++) { - out[i] = chArr[i]; - } - for (int i = sLen; i < fieldLength; i++) { - out[i] = padChar; - } - } else { //prepend - for (int i = 0; i < blanks; i++) { - out[i] = padChar; - } - for (int i = blanks; i < fieldLength; i++) { - out[i] = chArr[i - blanks]; - } - } - - return String.valueOf(out); - } - return s; - } - - @Test - public void printlnTest() { - println("PRINTING: "+this.getClass().getName()); - } - - /** - * @param s value to print - */ - static void println(String s) { - //System.out.println(s); //disable here - } - -} diff --git a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java deleted file mode 100644 index d2c868f0..00000000 --- a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v4bTest.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -import java.util.Random; - -import static org.apache.datasketches.memory.MurmurHash3.hash; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import org.testng.annotations.Test; - -import org.apache.datasketches.memory.MurmurHash3; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.internal.MurmurHash3v4; -import org.apache.datasketches.memory.WritableMemory; - -/** - * @author Lee Rhodes - */ -public class MurmurHash3v4bTest { - private Random rand = new Random(); - private static final int trials = 1 << 20; - - @Test - public void compareLongArrLong() { //long[] - int arrLen = 3; - int iPer = 8 / Long.BYTES; - long[] key = new long[arrLen]; - for (int i = 0; i < trials; i++) { //trials - for (int j = 0; j < arrLen / iPer; j++) { //longs - long r = rand.nextLong(); - key[j] = r; - } - long[] res1 = hashV1(key, 0); - long[] res2 = hashV2(key, 0); - assertEquals(res2, res1); - } - } - - @Test - public void compareIntArr() { //int[] - int bytes = Integer.BYTES; - int arrLen = 6; - int[] key = new int[arrLen]; - int iPer = 8 / bytes; - int nLongs = arrLen / iPer; - int shift = 64 / iPer; - - for (int i = 0; i < trials; i++) { //trials - for (int j = 0; j < nLongs; j++) { //longs - long r = rand.nextLong(); - for (int k = 0; k < iPer; k++) { //ints - int shft = k * shift; - key[k] = (int) (r >>> shft); - } - } - long[] res1 = hashV1(key, 0); - long[] res2 = hashV2(key, 0); - assertEquals(res2, res1); - } - } - - @Test - public void compareCharArr() { //char[] - int bytes = Character.BYTES; - int arrLen = 12; - char[] key = new char[arrLen]; - int iPer = 8 / bytes; - int nLongs = arrLen / iPer; - int shift = 64 / iPer; - - for (int i = 0; i < trials; i++) { //trials - for (int j = 0; j < nLongs; j++) { //longs - long r = rand.nextLong(); - for (int k = 0; k < iPer; k++) { //char - int shft = k * shift; - key[k] = (char) (r >>> shft); - } - } - long[] res1 = hashV1(key, 0); - long[] res2 = hashV2(key, 0); - assertEquals(res2, res1); - } - } - - @Test - public void compareByteArr() { //byte[] - int bytes = Byte.BYTES; - int arrLen = 12; - byte[] key = new byte[arrLen]; - int iPer = 8 / bytes; - int nLongs = arrLen / iPer; - int shift = 64 / iPer; - - for (int i = 0; i < trials; i++) { //trials - for (int j = 0; j < nLongs; j++) { //longs - long r = rand.nextLong(); - for (int k = 0; k < iPer; k++) { //bytes - int shft = k * shift; - key[k] = (byte) (r >>> shft); - } - } - long[] res1 = hashV1(key, 0); - long[] res2 = hashV2(key, 0); - assertEquals(res2, res1); - } - } - - @Test - public void compareLongVsLongArr() { - int arrLen = 1; - long[] key = new long[arrLen]; - long[] out = new long[2]; - for (int i = 0; i < trials; i++) { //trials - long r = rand.nextLong(); - key[0] = r; - long[] res1 = hashV1(key, 0); - long[] res2 = hashV2(r, 0, out); - assertEquals(res2, res1); - } - } - - private static final long[] hashV1(long[] key, long seed) { - return MurmurHash3.hash(key, seed); - } - - private static final long[] hashV1(int[] key, long seed) { - return MurmurHash3.hash(key, seed); - } - - private static final long[] hashV1(char[] key, long seed) { - return MurmurHash3.hash(key, seed); - } - - private static final long[] hashV1(byte[] key, long seed) { - return MurmurHash3.hash(key, seed); - } - - private static final long[] hashV2(long[] key, long seed) { - return MurmurHash3v4.hash(key, seed); - } - - private static final long[] hashV2(int[] key2, long seed) { - return MurmurHash3v4.hash(key2, seed); - } - - private static final long[] hashV2(char[] key, long seed) { - return MurmurHash3v4.hash(key, seed); - } - - private static final long[] hashV2(byte[] key, long seed) { - return MurmurHash3v4.hash(key, seed); - } - - //V2 single primitives - - private static final long[] hashV2(long key, long seed, long[] out) { - return MurmurHash3v4.hash(key, seed, out); - } - -// private static final long[] hashV2(double key, long seed, long[] out) { -// return MurmurHash3v4.hash(key, seed, out); -// } - -// private static final long[] hashV2(String key, long seed, long[] out) { -// return MurmurHash3v4.hash(key, seed, out); -// } - - - - @Test - public void offsetChecks() { - long seed = 12345; - int blocks = 6; - int cap = blocks * 16; - - long[] hash1 = new long[2]; - long[] hash2; - - WritableMemory wmem = WritableMemory.allocate(cap); - for (int i = 0; i < cap; i++) { wmem.putByte(i, (byte)(-128 + i)); } - - for (int offset = 0; offset < 16; offset++) { - int arrLen = cap - offset; - hash1 = MurmurHash3v4.hash(wmem, offset, arrLen, seed, hash1); - byte[] byteArr2 = new byte[arrLen]; - wmem.getByteArray(offset, byteArr2, 0, arrLen); - hash2 = MurmurHash3.hash(byteArr2, seed); - assertEquals(hash1, hash2); - } - } - - @Test - public void byteArrChecks() { - long seed = 0; - int offset = 0; - int bytes = 1024; - - long[] hash2 = new long[2]; - - for (int j = 1; j < bytes; j++) { - byte[] in = new byte[bytes]; - - WritableMemory wmem = WritableMemory.writableWrap(in); - for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } - - long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(in, seed); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - } - } - - @Test - public void charArrChecks() { - long seed = 0; - int offset = 0; - int chars = 16; - int bytes = chars << 1; - - long[] hash2 = new long[2]; - - for (int j = 1; j < chars; j++) { - char[] in = new char[chars]; - - WritableMemory wmem = WritableMemory.writableWrap(in); - for (int i = 0; i < j; i++) { wmem.putInt(i, i); } - - long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(in, seed); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - } - } - - @Test - public void intArrChecks() { - long seed = 0; - int offset = 0; - int ints = 16; - int bytes = ints << 2; - - long[] hash2 = new long[2]; - - for (int j = 1; j < ints; j++) { - int[] in = new int[ints]; - - WritableMemory wmem = WritableMemory.writableWrap(in); - for (int i = 0; i < j; i++) { wmem.putInt(i, i); } - - long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(in, seed); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - } - } - - @Test - public void longArrChecks() { - long seed = 0; - int offset = 0; - int longs = 16; - int bytes = longs << 3; - - long[] hash2 = new long[2]; - - for (int j = 1; j < longs; j++) { - long[] in = new long[longs]; - - WritableMemory wmem = WritableMemory.writableWrap(in); - for (int i = 0; i < j; i++) { wmem.putLong(i, i); } - - long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(in, seed); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - } - } - - @Test - public void longCheck() { - long seed = 0; - int offset = 0; - int bytes = 8; - - long[] hash2 = new long[2]; - long[] in = { 1 }; - WritableMemory wmem = WritableMemory.writableWrap(in); - - long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(in, seed); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - } - - @Test - public void checkEmptiesNulls() { - long seed = 123; - long[] hashOut = new long[2]; - try { - MurmurHash3v4.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - String s = ""; - MurmurHash3v4.hash(s, seed, hashOut); //string empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - String s = null; - MurmurHash3v4.hash(s, seed, hashOut); //string null - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - byte[] barr = new byte[0]; - MurmurHash3v4.hash(barr, seed); //byte[] empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - byte[] barr = null; - MurmurHash3v4.hash(barr, seed); //byte[] null - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - char[] carr = new char[0]; - MurmurHash3v4.hash(carr, seed); //char[] empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - char[] carr = null; - MurmurHash3v4.hash(carr, seed); //char[] null - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - int[] iarr = new int[0]; - MurmurHash3v4.hash(iarr, seed); //int[] empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - int[] iarr = null; - MurmurHash3v4.hash(iarr, seed); //int[] null - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - long[] larr = new long[0]; - MurmurHash3v4.hash(larr, seed); //long[] empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - long[] larr = null; - MurmurHash3v4.hash(larr, seed); //long[] null - fail(); - } catch (final IllegalArgumentException e) { } //OK - } - - @Test - public void checkStringLong() { - long seed = 123; - long[] hashOut = new long[2]; - String s = "123"; - assertTrue(MurmurHash3v4.hash(s, seed, hashOut)[0] != 0); - long v = 123; - assertTrue(MurmurHash3v4.hash(v, seed, hashOut)[0] != 0); - } - - @Test - public void doubleCheck() { - long[] hash1 = checkDouble(-0.0); - long[] hash2 = checkDouble(0.0); - assertEquals(hash1, hash2); - hash1 = checkDouble(Double.NaN); - long nan = (0x7FFL << 52) + 1L; - hash2 = checkDouble(Double.longBitsToDouble(nan)); - assertEquals(hash1, hash2); - checkDouble(1.0); - } - - private static long[] checkDouble(double dbl) { - long seed = 0; - int offset = 0; - int bytes = 8; - - long[] hash2 = new long[2]; - - final double d = (dbl == 0.0) ? 0.0 : dbl; // canonicalize -0.0, 0.0 - final long data = Double.doubleToLongBits(d);// canonicalize all NaN forms - final long[] dataArr = { data }; - - WritableMemory wmem = WritableMemory.writableWrap(dataArr); - long[] hash1 = MurmurHash3.hash(dataArr, 0); - hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v4.hash(dbl, seed, hash2); - - assertEquals(hash1, hash2); - assertEquals(hash1, hash3); - return hash1; - } - -} From 36b980aa8180c3963eed748c52dd4dca9cce2610 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 18 Feb 2025 11:36:50 -0800 Subject: [PATCH 19/19] Small change to POM. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 089a245f..f31639a5 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ under the License. - 7.10.2 + 7.11.0 3.6.3 @@ -182,7 +182,7 @@ under the License. - ${java.version} + ${java.version}, [${maven.version},4.0.0)