diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 42a6e8073576..41d4ab0e49f7 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -291,7 +291,8 @@ Improvements Optimizations --------------------- -(No changes) +* GITHUB#16283: Use Panama Vector API to SIMD-evaluate fixed-cardinality sorted numeric range + queries in rangeIntoBitSet. (Costin Leau) Bug Fixes --------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index 61f4f2942428..6f982af6ee6a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -495,29 +495,6 @@ private static int fixedCardinality( return (int) cardinality; } - private static void sortedNumericScalarRangeIntoBitSet( - LongValues values, - int fromDoc, - int toDoc, - int cardinality, - long minValue, - long maxValue, - FixedBitSet bitSet, - int offset) { - for (int doc = fromDoc; doc < toDoc; doc++) { - long valueOffset = (long) doc * cardinality; - for (int i = 0; i < cardinality; i++) { - long value = values.get(valueOffset + i); - if (value >= minValue) { - if (value <= maxValue) { - bitSet.set(doc - offset); - } - break; - } - } - } - } - private static boolean sortedNumericMatchesRange( LongValues values, long start, long end, long minValue, long maxValue) { for (long valueOffset = start; valueOffset < end; valueOffset++) { @@ -1906,7 +1883,7 @@ public void rangeIntoBitSet( } int cardinality = denseFixedCardinality; if (cardinality > 1) { - sortedNumericScalarRangeIntoBitSet( + DOC_VALUES_RANGE_SUPPORT.sortedNumericRangeIntoBitSet( values, fromDoc, endDoc, cardinality, minValue, maxValue, bitSet, offset); return; } diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java index 7b8b4e2c949c..db8a79e16052 100644 --- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java +++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java @@ -50,4 +50,36 @@ void rangeIntoBitSet( long maxValue, FixedBitSet bitSet, int offset); + + /** + * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose sorted numeric values contain + * at least one value in {@code [minValue, maxValue]}. + * + *

This method only supports fixed-cardinality fields where every document has exactly + * {@code cardinality} values stored contiguously starting at index {@code doc * cardinality}. + * + * @param cardinality number of values per document (must be > 0) + */ + default void sortedNumericRangeIntoBitSet( + LongValues values, + int fromDoc, + int toDoc, + int cardinality, + long minValue, + long maxValue, + FixedBitSet bitSet, + int offset) { + for (int doc = fromDoc; doc < toDoc; doc++) { + long valueOffset = (long) doc * cardinality; + for (int i = 0; i < cardinality; i++) { + long value = values.get(valueOffset + i); + if (value >= minValue) { + if (value <= maxValue) { + bitSet.set(doc - offset); + } + break; + } + } + } + } } diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java index 72fef3c31b69..a1a6ca1591e1 100644 --- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java +++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java @@ -69,4 +69,61 @@ public void rangeIntoBitSet( } } } + + @Override + public void sortedNumericRangeIntoBitSet( + LongValues values, + int fromDoc, + int toDoc, + int cardinality, + long minValue, + long maxValue, + FixedBitSet bitSet, + int offset) { + assert cardinality > 0 : "cardinality must be positive: " + cardinality; + final int vectorLen = LONG_SPECIES.length(); + final int docsPerVector = vectorLen / cardinality; + if (docsPerVector == 0 || vectorLen % cardinality != 0) { + DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet( + values, fromDoc, toDoc, cardinality, minValue, maxValue, bitSet, offset); + return; + } + + final long[] scratch = new long[vectorLen]; + final int vectorDocEnd = fromDoc + (toDoc - fromDoc) / docsPerVector * docsPerVector; + int doc = fromDoc; + for (; doc < vectorDocEnd; doc += docsPerVector) { + long valueOffset = (long) doc * cardinality; + for (int lane = 0; lane < vectorLen; lane++) { + scratch[lane] = values.get(valueOffset + lane); + } + LongVector vector = LongVector.fromArray(LONG_SPECIES, scratch, 0); + // Flat range check on all lanes. Equivalent to the scalar early-break on sorted values: + // a value outside [min,max] that is >= min must be > max, so its lane stays unset, and + // collapseToDocMask OR-reduces lanes per doc to match "any value in range" semantics. + VectorMask inRange = + vector + .compare(VectorOperators.GE, minValue) + .and(vector.compare(VectorOperators.LE, maxValue)); + long docMask = collapseToDocMask(inRange.toLong(), cardinality, docsPerVector); + if (docMask != 0) { + bitSet.orMask(doc - offset, docMask, docsPerVector); + } + } + + DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet( + values, doc, toDoc, cardinality, minValue, maxValue, bitSet, offset); + } + + private static long collapseToDocMask(long valueMask, int cardinality, int docsPerVector) { + final long valueMaskPerDoc = (1L << cardinality) - 1L; + long docMask = 0L; + for (int d = 0; d < docsPerVector; d++) { + long perDoc = (valueMask >>> (d * cardinality)) & valueMaskPerDoc; + if (perDoc != 0) { + docMask |= 1L << d; + } + } + return docMask; + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java index 76e01cae13c3..56e61d15ae55 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java @@ -652,6 +652,17 @@ public void testSortedNumericRangeIntoBitSetSparseVariableCardinality() throws E private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCardinality) throws Exception { + doTestSortedNumericRangeIntoBitSet(dense, fixedCardinality, 4); + } + + public void testSortedNumericRangeIntoBitSetVaryingCardinality() throws Exception { + for (int cardinality : new int[] {2, 3, 4, 5, 7, 8}) { + doTestSortedNumericRangeIntoBitSet(true, true, cardinality); + } + } + + private void doTestSortedNumericRangeIntoBitSet( + boolean dense, boolean fixedCardinality, int fixedCardinalityValue) throws Exception { int numDocs = 4096 * 2; try (Directory dir = newDirectory()) { IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec()); @@ -659,7 +670,7 @@ private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCard for (int docID = 0; docID < numDocs; docID++) { Document doc = new Document(); if (dense || docID % 3 != 0) { - int valueCount = fixedCardinality ? 4 : 1 + (docID & 3); + int valueCount = fixedCardinality ? fixedCardinalityValue : 1 + (docID & 3); long firstValue = (docID * 13L) % 100; for (int i = 0; i < valueCount; i++) { doc.add(SortedNumericDocValuesField.indexedField("sn", firstValue + i * 3L));