From cf2e64e98b5da66700cf15bd78eaa94c1100c138 Mon Sep 17 00:00:00 2001 From: sanghun Date: Sat, 27 Jun 2026 10:37:32 +0900 Subject: [PATCH 1/5] LUCENE-14611: Reduce memory pressure in TestTermInSetQuery.testDuel --- .../lucene/search/TestTermInSetQuery.java | 107 +++++++++++++++++- 1 file changed, 101 insertions(+), 6 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java index 481449478bc3..1359a616a300 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -53,6 +54,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.ByteRunnable; @@ -139,6 +141,7 @@ public void testDuel() throws IOException { iw.commit(); final IndexReader reader = iw.getReader(); final IndexSearcher searcher = newSearcher(reader); + searcher.setQueryCache(null); iw.close(); if (reader.numDocs() == 0) { @@ -284,14 +287,106 @@ public void testSkipperOptimizationGapAssumption() throws IOException { private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boolean scores) throws IOException { final int maxDoc = searcher.getIndexReader().maxDoc(); - final TopDocs td1 = searcher.search(q1, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); - final TopDocs td2 = searcher.search(q2, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); - assertEquals(td1.totalHits.value(), td2.totalHits.value()); - for (int i = 0; i < td1.scoreDocs.length; ++i) { - assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc); - if (scores) { + if (scores) { + final TopDocs td1 = searcher.search(q1, maxDoc); + final TopDocs td2 = searcher.search(q2, maxDoc); + assertEquals(td1.totalHits.value(), td2.totalHits.value()); + for (int i = 0; i < td1.scoreDocs.length; ++i) { + assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc); assertEquals(td1.scoreDocs[i].score, td2.scoreDocs[i].score, 10e-7); } + } else { + final MatchSet matches1 = collectMatches(searcher, q1, maxDoc); + final MatchSet matches2 = collectMatches(searcher, q2, maxDoc); + assertEquals(matches1.totalHits, matches2.totalHits); + for (int doc = matches1.docs.nextSetBit(0); + doc != DocIdSetIterator.NO_MORE_DOCS; + doc = matches1.docs.nextSetBit(doc + 1)) { + assertTrue(matches2.docs.get(doc)); + } + for (int doc = matches2.docs.nextSetBit(0); + doc != DocIdSetIterator.NO_MORE_DOCS; + doc = matches2.docs.nextSetBit(doc + 1)) { + assertTrue(matches1.docs.get(doc)); + } + } + } + + private static MatchSet collectMatches(IndexSearcher searcher, Query query, int maxDoc) + throws IOException { + return searcher.search( + query, + new CollectorManager() { + @Override + public MatchSetCollector newCollector() { + return new MatchSetCollector(maxDoc); + } + + @Override + public MatchSet reduce(Collection collectors) { + MatchSet reduced = new MatchSet(maxDoc); + for (MatchSetCollector collector : collectors) { + reduced.or(collector.matches); + } + return reduced; + } + }); + } + + private static class MatchSet { + final FixedBitSet docs; + int totalHits; + + MatchSet(int maxDoc) { + docs = new FixedBitSet(maxDoc); + } + + void or(MatchSet other) { + docs.or(other.docs); + totalHits += other.totalHits; + } + } + + private static class MatchSetCollector implements Collector { + private final MatchSet matches; + + MatchSetCollector(int maxDoc) { + matches = new MatchSet(maxDoc); + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) { + final int docBase = context.docBase; + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) { + matches.docs.set(docBase + doc); + matches.totalHits++; + } + + @Override + public void collectRange(int min, int max) { + matches.docs.set(docBase + min, docBase + max); + matches.totalHits += max - min; + } + + @Override + public void collect(DocIdStream stream) throws IOException { + stream.forEach( + doc -> { + matches.docs.set(docBase + doc); + matches.totalHits++; + }); + } + }; + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; } } From 600b0b288988f9861248ad9c4507720459aa2892 Mon Sep 17 00:00:00 2001 From: sanghun Date: Sat, 27 Jun 2026 10:38:40 +0900 Subject: [PATCH 2/5] LUCENE-14611: Clarify TestTermInSetQuery match comparison --- .../test/org/apache/lucene/search/TestTermInSetQuery.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java index 1359a616a300..64daaa7960a0 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java @@ -141,6 +141,8 @@ public void testDuel() throws IOException { iw.commit(); final IndexReader reader = iw.getReader(); final IndexSearcher searcher = newSearcher(reader); + // This test checks query equivalence, not query-cache behavior. Keep the randomized + // test-framework searcher, but avoid retaining cached doc-id sets across iterations. searcher.setQueryCache(null); iw.close(); @@ -290,14 +292,18 @@ private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boole if (scores) { final TopDocs td1 = searcher.search(q1, maxDoc); final TopDocs td2 = searcher.search(q2, maxDoc); + assertEquals(td1.totalHits.value(), td2.totalHits.value()); for (int i = 0; i < td1.scoreDocs.length; ++i) { assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc); assertEquals(td1.scoreDocs[i].score, td2.scoreDocs[i].score, 10e-7); } } else { + // For no-score comparisons, only doc-id set equality matters. Avoid materializing + // all hits as sorted TopDocs for every query pair. final MatchSet matches1 = collectMatches(searcher, q1, maxDoc); final MatchSet matches2 = collectMatches(searcher, q2, maxDoc); + assertEquals(matches1.totalHits, matches2.totalHits); for (int doc = matches1.docs.nextSetBit(0); doc != DocIdSetIterator.NO_MORE_DOCS; From 88e1d09d3e901a6436bf82072272935eeec17a99 Mon Sep 17 00:00:00 2001 From: sanghun Date: Sat, 27 Jun 2026 10:53:09 +0900 Subject: [PATCH 3/5] LUCENE-14611: Add CHANGES entry --- lucene/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 42a6e8073576..232a922b4094 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -162,6 +162,9 @@ Optimizations Bug Fixes --------------------- +* GITHUB#16300: Reduce memory pressure in TestTermInSetQuery.testDuel stress runs. + (Sanghun Lee) + * GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero when all values are identical. (Mike Sokolov) From d1615a9f1fef6ff89283c22a085f1cdc27ae23d7 Mon Sep 17 00:00:00 2001 From: sanghun Date: Sat, 27 Jun 2026 11:22:43 +0900 Subject: [PATCH 4/5] LUCENE-14611: Use FixedBitSet equality for match comparison --- .../org/apache/lucene/search/TestTermInSetQuery.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java index 64daaa7960a0..53399a7fa92e 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java @@ -305,16 +305,7 @@ private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boole final MatchSet matches2 = collectMatches(searcher, q2, maxDoc); assertEquals(matches1.totalHits, matches2.totalHits); - for (int doc = matches1.docs.nextSetBit(0); - doc != DocIdSetIterator.NO_MORE_DOCS; - doc = matches1.docs.nextSetBit(doc + 1)) { - assertTrue(matches2.docs.get(doc)); - } - for (int doc = matches2.docs.nextSetBit(0); - doc != DocIdSetIterator.NO_MORE_DOCS; - doc = matches2.docs.nextSetBit(doc + 1)) { - assertTrue(matches1.docs.get(doc)); - } + assertEquals(matches1.docs, matches2.docs); } } From d1f5aa45ff2ea3bacba2dac8f29cfe27e0799309 Mon Sep 17 00:00:00 2001 From: sanghun Date: Sat, 27 Jun 2026 11:46:11 +0900 Subject: [PATCH 5/5] LUCENE-14611: Reuse FixedBitSetCollector for match comparison --- .../lucene/search/TestTermInSetQuery.java | 86 ++----------------- 1 file changed, 6 insertions(+), 80 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java index 53399a7fa92e..22efbdb156d6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -48,6 +47,7 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.search.FixedBitSetCollector; import org.apache.lucene.tests.search.QueryUtils; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.RamUsageTester; @@ -301,90 +301,16 @@ private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boole } else { // For no-score comparisons, only doc-id set equality matters. Avoid materializing // all hits as sorted TopDocs for every query pair. - final MatchSet matches1 = collectMatches(searcher, q1, maxDoc); - final MatchSet matches2 = collectMatches(searcher, q2, maxDoc); + final FixedBitSet matches1 = collectMatches(searcher, q1, maxDoc); + final FixedBitSet matches2 = collectMatches(searcher, q2, maxDoc); - assertEquals(matches1.totalHits, matches2.totalHits); - assertEquals(matches1.docs, matches2.docs); + assertEquals(matches1, matches2); } } - private static MatchSet collectMatches(IndexSearcher searcher, Query query, int maxDoc) + private static FixedBitSet collectMatches(IndexSearcher searcher, Query query, int maxDoc) throws IOException { - return searcher.search( - query, - new CollectorManager() { - @Override - public MatchSetCollector newCollector() { - return new MatchSetCollector(maxDoc); - } - - @Override - public MatchSet reduce(Collection collectors) { - MatchSet reduced = new MatchSet(maxDoc); - for (MatchSetCollector collector : collectors) { - reduced.or(collector.matches); - } - return reduced; - } - }); - } - - private static class MatchSet { - final FixedBitSet docs; - int totalHits; - - MatchSet(int maxDoc) { - docs = new FixedBitSet(maxDoc); - } - - void or(MatchSet other) { - docs.or(other.docs); - totalHits += other.totalHits; - } - } - - private static class MatchSetCollector implements Collector { - private final MatchSet matches; - - MatchSetCollector(int maxDoc) { - matches = new MatchSet(maxDoc); - } - - @Override - public LeafCollector getLeafCollector(LeafReaderContext context) { - final int docBase = context.docBase; - return new LeafCollector() { - @Override - public void setScorer(Scorable scorer) {} - - @Override - public void collect(int doc) { - matches.docs.set(docBase + doc); - matches.totalHits++; - } - - @Override - public void collectRange(int min, int max) { - matches.docs.set(docBase + min, docBase + max); - matches.totalHits += max - min; - } - - @Override - public void collect(DocIdStream stream) throws IOException { - stream.forEach( - doc -> { - matches.docs.set(docBase + doc); - matches.totalHits++; - }); - } - }; - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } + return searcher.search(query, FixedBitSetCollector.createManager(maxDoc)); } public void testHashCodeAndEquals() {