From 015f370b2f7789e16a50bd8cb83d5fd141d86835 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Sun, 21 Jun 2026 18:00:00 +0300 Subject: [PATCH 1/2] Add fixed-cardinality SIMD for sorted numeric ranges Dense fixed-cardinality sorted numeric values can evaluate range blocks with the vectorization provider when the flattened value layout is raw and contiguous. Keep the optimization gated to layouts that benchmark well and retain scalar fallback behavior for other encodings. --- lucene/CHANGES.txt | 3 +- .../lucene90/Lucene90DocValuesProducer.java | 25 +-------- .../vectorization/DocValuesRangeSupport.java | 27 +++++++++ .../PanamaDocValuesRangeSupport.java | 56 +++++++++++++++++++ 4 files changed, 86 insertions(+), 25 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 42a6e8073576..41d4ab0e49f7 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -291,7 +291,8 @@ Improvements Optimizations --------------------- -(No changes) +* GITHUB#16283: Use Panama Vector API to SIMD-evaluate fixed-cardinality sorted numeric range + queries in rangeIntoBitSet. (Costin Leau) Bug Fixes --------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index 61f4f2942428..6f982af6ee6a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -495,29 +495,6 @@ private static int fixedCardinality( return (int) cardinality; } - private static void sortedNumericScalarRangeIntoBitSet( - LongValues values, - int fromDoc, - int toDoc, - int cardinality, - long minValue, - long maxValue, - FixedBitSet bitSet, - int offset) { - for (int doc = fromDoc; doc < toDoc; doc++) { - long valueOffset = (long) doc * cardinality; - for (int i = 0; i < cardinality; i++) { - long value = values.get(valueOffset + i); - if (value >= minValue) { - if (value <= maxValue) { - bitSet.set(doc - offset); - } - break; - } - } - } - } - private static boolean sortedNumericMatchesRange( LongValues values, long start, long end, long minValue, long maxValue) { for (long valueOffset = start; valueOffset < end; valueOffset++) { @@ -1906,7 +1883,7 @@ public void rangeIntoBitSet( } int cardinality = denseFixedCardinality; if (cardinality > 1) { - sortedNumericScalarRangeIntoBitSet( + DOC_VALUES_RANGE_SUPPORT.sortedNumericRangeIntoBitSet( values, fromDoc, endDoc, cardinality, minValue, maxValue, bitSet, offset); return; } diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java index 7b8b4e2c949c..24570a6ae191 100644 --- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java +++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java @@ -50,4 +50,31 @@ void rangeIntoBitSet( long maxValue, FixedBitSet bitSet, int offset); + + /** + * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose {@code cardinality} sorted + * values contain at least one value in {@code [minValue, maxValue]}. + */ + default void sortedNumericRangeIntoBitSet( + LongValues values, + int fromDoc, + int toDoc, + int cardinality, + long minValue, + long maxValue, + FixedBitSet bitSet, + int offset) { + for (int doc = fromDoc; doc < toDoc; doc++) { + long valueOffset = (long) doc * cardinality; + for (int i = 0; i < cardinality; i++) { + long value = values.get(valueOffset + i); + if (value >= minValue) { + if (value <= maxValue) { + bitSet.set(doc - offset); + } + break; + } + } + } + } } diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java index 72fef3c31b69..32668741b985 100644 --- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java +++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java @@ -69,4 +69,60 @@ public void rangeIntoBitSet( } } } + + @Override + public void sortedNumericRangeIntoBitSet( + LongValues values, + int fromDoc, + int toDoc, + int cardinality, + long minValue, + long maxValue, + FixedBitSet bitSet, + int offset) { + final int vectorLen = LONG_SPECIES.length(); + final int docsPerVector = vectorLen / cardinality; + if (docsPerVector == 0 || vectorLen % cardinality != 0) { + DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet( + values, fromDoc, toDoc, cardinality, minValue, maxValue, bitSet, offset); + return; + } + + final long[] scratch = new long[vectorLen]; + final int vectorDocEnd = fromDoc + (toDoc - fromDoc) / docsPerVector * docsPerVector; + int doc = fromDoc; + for (; doc < vectorDocEnd; doc += docsPerVector) { + long valueOffset = (long) doc * cardinality; + for (int lane = 0; lane < vectorLen; lane++) { + scratch[lane] = values.get(valueOffset + lane); + } + LongVector vector = LongVector.fromArray(LONG_SPECIES, scratch, 0); + // Flat range check on all lanes. Equivalent to the scalar early-break on sorted values: + // a value outside [min,max] that is >= min must be > max, so its lane stays unset, and + // collapseToDocMask OR-reduces lanes per doc to match "any value in range" semantics. + VectorMask inRange = + vector + .compare(VectorOperators.GE, minValue) + .and(vector.compare(VectorOperators.LE, maxValue)); + long docMask = collapseToDocMask(inRange.toLong(), cardinality, docsPerVector); + if (docMask != 0) { + bitSet.orMask(doc - offset, docMask, docsPerVector); + } + } + + DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet( + values, doc, toDoc, cardinality, minValue, maxValue, bitSet, offset); + } + + private static long collapseToDocMask(long valueMask, int cardinality, int docsPerVector) { + final long valueMaskPerDoc = (1L << cardinality) - 1L; + long docMask = 0L; + for (int d = 0; d < docsPerVector; d++) { + long perDoc = (valueMask >>> (d * cardinality)) & valueMaskPerDoc; + if (perDoc != 0) { + docMask |= 1L << d; + } + } + return docMask; + } } From b8f15a9fc13cedff61f047dc20959b3c0720cebf Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Thu, 25 Jun 2026 14:05:32 +0300 Subject: [PATCH 2/2] Address feedback --- .../vectorization/DocValuesRangeSupport.java | 9 +++++++-- .../vectorization/PanamaDocValuesRangeSupport.java | 1 + .../TestSkipBlockRangeIteratorIntoBitSet.java | 13 ++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java index 24570a6ae191..db8a79e16052 100644 --- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java +++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java @@ -52,8 +52,13 @@ void rangeIntoBitSet( int offset); /** - * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose {@code cardinality} sorted - * values contain at least one value in {@code [minValue, maxValue]}. + * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose sorted numeric values contain + * at least one value in {@code [minValue, maxValue]}. + * + *

This method only supports fixed-cardinality fields where every document has exactly + * {@code cardinality} values stored contiguously starting at index {@code doc * cardinality}. + * + * @param cardinality number of values per document (must be > 0) */ default void sortedNumericRangeIntoBitSet( LongValues values, diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java index 32668741b985..a1a6ca1591e1 100644 --- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java +++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java @@ -80,6 +80,7 @@ public void sortedNumericRangeIntoBitSet( long maxValue, FixedBitSet bitSet, int offset) { + assert cardinality > 0 : "cardinality must be positive: " + cardinality; final int vectorLen = LONG_SPECIES.length(); final int docsPerVector = vectorLen / cardinality; if (docsPerVector == 0 || vectorLen % cardinality != 0) { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java index 76e01cae13c3..56e61d15ae55 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java @@ -652,6 +652,17 @@ public void testSortedNumericRangeIntoBitSetSparseVariableCardinality() throws E private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCardinality) throws Exception { + doTestSortedNumericRangeIntoBitSet(dense, fixedCardinality, 4); + } + + public void testSortedNumericRangeIntoBitSetVaryingCardinality() throws Exception { + for (int cardinality : new int[] {2, 3, 4, 5, 7, 8}) { + doTestSortedNumericRangeIntoBitSet(true, true, cardinality); + } + } + + private void doTestSortedNumericRangeIntoBitSet( + boolean dense, boolean fixedCardinality, int fixedCardinalityValue) throws Exception { int numDocs = 4096 * 2; try (Directory dir = newDirectory()) { IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec()); @@ -659,7 +670,7 @@ private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCard for (int docID = 0; docID < numDocs; docID++) { Document doc = new Document(); if (dense || docID % 3 != 0) { - int valueCount = fixedCardinality ? 4 : 1 + (docID & 3); + int valueCount = fixedCardinality ? fixedCardinalityValue : 1 + (docID & 3); long firstValue = (docID * 13L) % 100; for (int i = 0; i < valueCount; i++) { doc.add(SortedNumericDocValuesField.indexedField("sn", firstValue + i * 3L));