update move_to_17_using_FFM with 6.1.X

Author: leerho

src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java

@@ -32,22 +32,16 @@
  * This class defines the preamble data structure and provides basic utilities for some of the key
  * fields.
  *
- * <p>
- * The intent of the design of this class was to isolate the detailed knowledge of the bit and byte
+ * <p>The intent of the design of this class was to isolate the detailed knowledge of the bit and byte
  * layout of the serialized form of the sketches derived from the Sketch class into one place. This
  * allows the possibility of the introduction of different serialization schemes with minimal impact
- * on the rest of the library.
- * </p>
+ * on the rest of the library.</p>
  *
- * <p>
- * MAP: Low significance bytes of this <i>long</i> data structure are on the right. However, the
+ * <p>MAP: Low significance bytes of this <i>long</i> data structure are on the right. However, the
  * multi-byte integers (<i>int</i> and <i>long</i>) are stored in native byte order. The <i>byte</i>
- * values are treated as unsigned.
- * </p>
+ * values are treated as unsigned.</p>
  *
- * <p>
- * An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble.
- * </p>
+ * <p>An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble.</p>
  *
  * <pre>
  *  * Long || Start Byte Adr:

src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java

@@ -58,7 +58,7 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin
         | (ordered ? ORDERED_FLAG_MASK : 0)
         | (compact ? (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK) : 0);
 
-    if (empty && !sketch.hasMemory()) { //empty & has Memory
+    if (empty && !sketch.hasMemory()) { //empty & !has Memory
       final byte[] outByteArr = new byte[Long.BYTES];
       final WritableMemory memOut = WritableMemory.writableWrap(outByteArr);
       final int preLongs = 1;
@@ -79,31 +79,31 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin
    */
   private static byte[] convertToByteArray(final DoublesSketch sketch, final int flags,
                                            final boolean ordered, final boolean compact) {
-    final int preLongs = 2;
-    final int extra = 2; // extra space for min and max quantiles
-    final int prePlusExtraBytes = (preLongs + extra) << 3;
-    final int k = sketch.getK();
-    final long n = sketch.getN();
-
-    // If not-compact, have accessor always report full levels. Then use level size to determine
-    // whether to copy data out.
-    final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact);
+    final int preLongs = sketch.isEmpty() ? 1 : 2;
 
     final int outBytes = (compact ? sketch.getCurrentCompactSerializedSizeBytes()
         : sketch.getCurrentUpdatableSerializedSizeBytes());
 
     final byte[] outByteArr = new byte[outBytes];
     final WritableMemory memOut = WritableMemory.writableWrap(outByteArr);
 
-    //insert preamble-0, N, min, max
+    //insert pre0
+    final int k = sketch.getK();
     insertPre0(memOut, preLongs, flags, k);
     if (sketch.isEmpty()) { return outByteArr; }
 
+    //insert N, min, max
+    final long n = sketch.getN();
     insertN(memOut, n);
     insertMinDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMinItem());
     insertMaxDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMaxItem());
 
-    long memOffsetBytes = prePlusExtraBytes;
+    // If not-compact, have accessor always report full levels. Then use level size to determine
+    // whether to copy data out.
+    final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact);
+
+    final int minAndMax = 2; // extra space for min and max quantiles
+    long memOffsetBytes = (preLongs + minAndMax) << 3;
 
     // might need to sort base buffer but don't want to change input sketch
     final int bbCnt = computeBaseBufferItems(k, n);

src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java

@@ -32,6 +32,7 @@
 import org.apache.datasketches.memory.WritableBuffer;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.QuantilesAPI;
+import org.apache.datasketches.quantilescommon.QuantilesUtil;
 
 /**
  * t-Digest for estimating quantiles and ranks.
@@ -125,7 +126,7 @@ public void merge(final TDigestDouble other) {
   /**
    * Process buffered values and merge centroids if needed
    */
-  public void compress() {
+  private void compress() {
     if (numBuffered_ == 0) { return; }
     final int num = numBuffered_ + numCentroids_;
     final double[] values =  new double[num];
@@ -277,6 +278,51 @@ public double getQuantile(final double rank) {
     return weightedAverage(centroidWeights_[numCentroids_ - 1], w1, maxValue_, w2);
   }
 
+  /**
+   * Returns an approximation to the Probability Mass Function (PMF) of the input stream
+   * given a set of split points.
+   *
+   * @param splitPoints an array of <i>m</i> unique, monotonically increasing values
+   * that divide the input domain into <i>m+1</i> consecutive disjoint intervals (bins).
+   *
+   * @return an array of m+1 doubles each of which is an approximation
+   * to the fraction of the input stream values (the mass) that fall into one of those intervals.
+   * @throws SketchesStateException if sketch is empty.
+   */
+  public double[] getPMF(final double[] splitPoints) {
+    final double[] buckets = getCDF(splitPoints);
+    for (int i = buckets.length; i-- > 1; ) {
+      buckets[i] -= buckets[i - 1];
+    }
+    return buckets;
+  }
+
+  /**
+   * Returns an approximation to the Cumulative Distribution Function (CDF), which is the
+   * cumulative analog of the PMF, of the input stream given a set of split points.
+   *
+   * @param splitPoints an array of <i>m</i> unique, monotonically increasing values
+   * that divide the input domain into <i>m+1</i> consecutive disjoint intervals.
+   *
+   * @return an array of m+1 doubles, which are a consecutive approximation to the CDF
+   * of the input stream given the splitPoints. The value at array position j of the returned
+   * CDF array is the sum of the returned values in positions 0 through j of the returned PMF
+   * array. This can be viewed as array of ranks of the given split points plus one more value
+   * that is always 1.
+   * @throws SketchesStateException if sketch is empty.
+   */
+  public double[] getCDF(final double[] splitPoints) {
+    if (isEmpty()) { throw new SketchesStateException(QuantilesAPI.EMPTY_MSG); }
+    QuantilesUtil.checkDoublesSplitPointsOrder(splitPoints);
+    final int len = splitPoints.length + 1;
+    final double[] ranks = new double[len];
+    for (int i = 0; i < len - 1; i++) {
+      ranks[i] = getRank(splitPoints[i]);
+    }
+    ranks[len - 1] = 1.0;
+    return ranks;
+  }
+
   /**
    * Computes size needed to serialize the current state.
    * @return size in bytes needed to serialize this tdigest

src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java

@@ -139,7 +139,7 @@ public void basicWritableWrapTest() {
   @Test
   public void countWritableWrappedBitsWhenDirty() {
     // like basicOperationTest but with setBit which does
-    // not neecssarily track numBitsSet_
+    // not necessarily track numBitsSet_
     final HeapBitArray hba = new HeapBitArray(128);
     assertFalse(hba.getAndSetBit(1));
     assertFalse(hba.getAndSetBit(2));

src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java

@@ -31,7 +31,6 @@
 
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 
 /**
@@ -74,8 +73,6 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool
     byte[] barr1;
     WritableMemory wmem;
     try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) {
-      //byte[] byteArr = new byte[bytes];
-      //WritableMemory wmem = WritableMemory.wrap(byteArr);
       hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
       assertTrue(hllSketch.isEmpty());
 

src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java

@@ -109,7 +109,6 @@ public void checkCorruptMemoryInput() {
     HllSketch sk = new HllSketch(12);
     byte[] memObj = sk.toCompactByteArray();
     WritableMemory wmem = WritableMemory.writableWrap(memObj);
-    //long memAdd = wmem.getCumulativeOffset(0);
     HllSketch bad;
 
     //checkFamily
@@ -148,7 +147,6 @@ public void checkCorruptMemoryInput() {
     for (int i = 1; i <= 15; i++) { sk.update(i); }
     memObj = sk.toCompactByteArray();
     wmem = WritableMemory.writableWrap(memObj);
-    //memAdd = wmem.getCumulativeOffset(0);
 
     //check wrong PreInts and SET
     try {
@@ -162,7 +160,6 @@ public void checkCorruptMemoryInput() {
     for (int i = 15; i <= 1000; i++) { sk.update(i); }
     memObj = sk.toCompactByteArray();
     wmem = WritableMemory.writableWrap(memObj);
-    //memAdd = wmem.getCumulativeOffset(0);
 
     //check wrong PreInts and HLL
     try {
@@ -179,7 +176,6 @@ public void checkExtractFlags() {
     int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
     WritableMemory wmem = WritableMemory.allocate(bytes);
     Object memObj = wmem.getArray();
-    //long memAdd = wmem.getCumulativeOffset(0L);
     HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
     int flags = extractFlags(wmem);
     assertEquals(flags, EMPTY_FLAG_MASK);

src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java

@@ -67,6 +67,16 @@ public void generateKllFloatsSketchBinaries() throws IOException {
     }
   }
 
+  @Test(groups = {GENERATE_JAVA_FILES})
+  public void generateKllLongsSketchBinaries() throws IOException {
+    final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+    for (int n: nArr) {
+      final KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+      for (int i = 1; i <= n; i++) { sk.update(i); }
+      Files.newOutputStream(javaPath.resolve("kll_long_n" + n + "_java.sk")).write(sk.toByteArray());
+    }
+  }
+
   @Test(groups = {GENERATE_JAVA_FILES})
   public void generateKllItemsSketchBinaries() throws IOException {
     final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};

src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java

@@ -110,13 +110,13 @@ public void checkDirectCompactGetDoubleItemsArray() {
 
     KllDoublesSketch sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
     double[] itemsArr = sk2.getDoubleItemsArray();
-    for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0F); }
+    for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0.0); }
 
     sk.update(1);
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
     itemsArr = sk2.getDoubleItemsArray();
-    for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0F); }
-    assertEquals(itemsArr[19], 1F);
+    for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0.0); }
+    assertEquals(itemsArr[19], 1.0);
 
     for (int i = 2; i <= 21; i++) { sk.update(i); }
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
@@ -169,12 +169,12 @@ public void checkMinAndMax() {
     try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
     sk.update(1);
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
-    assertEquals(sk2.getMaxItem(),1.0F);
-    assertEquals(sk2.getMinItem(),1.0F);
+    assertEquals(sk2.getMaxItem(),1.0);
+    assertEquals(sk2.getMinItem(),1.0);
     for (int i = 2; i <= 21; i++) { sk.update(i); }
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
-    assertEquals(sk2.getMaxItem(),21.0F);
-    assertEquals(sk2.getMinItem(),1.0F);
+    assertEquals(sk2.getMaxItem(),21.0);
+    assertEquals(sk2.getMinItem(),1.0);
   }
 
   @Test

src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java

@@ -41,7 +41,7 @@ public void oneItemSketch() {
     sketch.update(0);
     QuantilesDoublesSketchIterator it = sketch.iterator();
     Assert.assertTrue(it.next());
-    Assert.assertEquals(it.getQuantile(), 0f);
+    Assert.assertEquals(it.getQuantile(), 0);
     Assert.assertEquals(it.getWeight(), 1);
     Assert.assertFalse(it.next());
   }

src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java

@@ -189,11 +189,11 @@ public void mergeLowerK() {
       sketch2.update(2 * n - i - 1);
     }
 
-    assertEquals(sketch1.getMinItem(), 0.0f);
-    assertEquals(sketch1.getMaxItem(), n - 1f);
+    assertEquals(sketch1.getMinItem(), 0.0);
+    assertEquals(sketch1.getMaxItem(), n - 1.0);
 
     assertEquals(sketch2.getMinItem(), n);
-    assertEquals(sketch2.getMaxItem(), 2f * n - 1f);
+    assertEquals(sketch2.getMaxItem(), 2.0 * n - 1.0);
 
     assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
     assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
@@ -613,7 +613,7 @@ public void checkWritableWrapOfCompactForm() {
   public void checkReadOnlyExceptions() {
     int k = 20;
     double[] dblArr = new double[0];
-    double dblV = 1.0f;
+    double dblV = 1.0;
     int idx = 1;
     boolean bool = true;
     KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k);