Enabled by default. By default, sorting on a numeric field activates point sort optimization - * that can efficiently skip over non-competitive hits. Sort optimization has a number of - * requirements, one of which is that SortField.Type matches the Point type with which the field - * was indexed (e.g. sort on IntPoint field should use SortField.Type.INT). Another requirement is - * that the same data is indexed with points and doc values for the field. - * - *
By default, sorting on a SORTED(_SET) field activates sort optimization that can efficiently - * skip over non-competitive hits. Sort optimization requires that the same data is indexed with - * term index and doc values for the field. - * - * @param optimizeSortWithIndexedData providing {@code false} disables the optimization, in cases - * where these requirements can't be met. - * @deprecated should only be used for compatibility with 8.x indices that got created with - * inconsistent data across fields, or the wrong sort configuration in the index sort - */ - @Deprecated // Remove in Lucene 10 - public void setOptimizeSortWithIndexedData(boolean optimizeSortWithIndexedData) { - this.optimizeSortWithIndexedData = optimizeSortWithIndexedData; - } - - /** - * Returns whether sort optimization should be optimized with indexed data - * - * @return whether sort optimization should be optimized with indexed data - */ - @Deprecated // Remove in Lucene 10 - public boolean getOptimizeSortWithIndexedData() { - return optimizeSortWithIndexedData; - } - - /** - * Enables/disables numeric sort optimization to use the Points index. - * - *
Enabled by default. By default, sorting on a numeric field activates point sort optimization - * that can efficiently skip over non-competitive hits. Sort optimization has a number of - * requirements, one of which is that SortField.Type matches the Point type with which the field - * was indexed (e.g. sort on IntPoint field should use SortField.Type.INT). Another requirement is - * that the same data is indexed with points and doc values for the field. - * - * @param optimizeSortWithPoints providing {@code false} disables the optimization, in cases where - * these requirements can't be met. - * @deprecated should only be used for compatibility with 8.x indices that got created with - * inconsistent data across fields, or the wrong sort configuration in the index sort. This is - * a duplicate method for {@code SortField#setOptimizeSortWithIndexedData}. - */ - @Deprecated // Remove in Lucene 10 - public void setOptimizeSortWithPoints(boolean optimizeSortWithPoints) { - setOptimizeSortWithIndexedData(optimizeSortWithPoints); - } - - /** - * Returns whether sort optimization should be optimized with points index - * - * @return whether sort optimization should be optimized with points index - * @deprecated This is a duplicate method for {@code SortField#getOptimizeSortWithIndexedData}. - */ - @Deprecated // Remove in Lucene 10 - public boolean getOptimizeSortWithPoints() { - return getOptimizeSortWithIndexedData(); - } } diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java index 2ff8fd56ae80..3cc85c5ebf59 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java @@ -359,9 +359,7 @@ protected NumericDocValues getNumericDocValues( default: throw new AssertionError(); } - if (getOptimizeSortWithIndexedData() == false) { - fieldComparator.disableSkipping(); - } + return fieldComparator; } diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java index c94d4f7f5c11..081a40387f71 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java @@ -171,9 +171,8 @@ public String toString() { @Override public FieldComparator getComparator(int numHits, Pruning pruning) { - Pruning finalPruning = getOptimizeSortWithIndexedData() ? pruning : Pruning.NONE; return new TermOrdValComparator( - numHits, getField(), missingValue == STRING_LAST, reverse, finalPruning) { + numHits, getField(), missingValue == STRING_LAST, reverse, pruning) { @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java index 2442d7dbf218..87f9e0458601 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java @@ -331,33 +331,11 @@ private void testNumericDocValuesOptimizationWithMissingValues( final DirectoryReader reader = DirectoryReader.open(writer); writer.close(); final int numHits = 3; - TopDocs topDocs1; - TopDocs topDocs2; - { // Test that optimization is run with NumericDocValues when missing value is NOT competitive final SortField sortField = new SortField("my_field", SortField.Type.LONG, true, 0L); final Sort sort = new Sort(sortField); - topDocs1 = assertSearchHits(reader, sort, numHits, null); - assertNonCompetitiveHitsAreSkipped(topDocs1.totalHits.value(), numDocs); - } - { // Test that sort on sorted numeric field without sort optimization and with sort optimization - // produce the same results - final SortField sortField = new SortField("my_field", SortField.Type.LONG, true, 0L); - final Sort sort = new Sort(sortField); - sortField.setOptimizeSortWithPoints(false); - topDocs2 = assertSearchHits(reader, sort, numHits, null); - // assert that the resulting hits are the same - assertEquals(topDocs1.scoreDocs.length, topDocs2.scoreDocs.length); - assertEquals(topDocs1.scoreDocs.length, numHits); - ScoreDoc[] scoreDocs1 = topDocs1.scoreDocs; - ScoreDoc[] scoreDocs2 = topDocs2.scoreDocs; - for (int i = 0; i < numHits; i++) { - FieldDoc fieldDoc = (FieldDoc) scoreDocs1[i]; - FieldDoc fieldDoc2 = (FieldDoc) scoreDocs2[i]; - assertEquals(fieldDoc.fields[0], fieldDoc2.fields[0]); - assertEquals(fieldDoc.doc, fieldDoc2.doc); - } - assertTrue(topDocs1.totalHits.value() < topDocs2.totalHits.value()); + TopDocs topDocs = assertSearchHits(reader, sort, numHits, null); + assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs); } { // Test that we can't do optimization via NumericDocValues when there are multiple comparators @@ -795,25 +773,15 @@ public void testPointValidation() throws IOException { assertThrows( IllegalArgumentException.class, () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(longSortOnIntField))); - // assert that when sort optimization is disabled we can use LONG sort on int field - longSortOnIntField.setOptimizeSortWithIndexedData(false); - searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(longSortOnIntField)); - SortField intSortOnLongField = new SortField("longField", SortField.Type.INT); assertThrows( IllegalArgumentException.class, () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnLongField))); - // assert that when sort optimization is disabled we can use INT sort on long field - intSortOnLongField.setOptimizeSortWithIndexedData(false); - searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnLongField)); SortField intSortOnIntRangeField = new SortField("intRange", SortField.Type.INT); assertThrows( IllegalArgumentException.class, () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnIntRangeField))); - // assert that when sort optimization is disabled we can use INT sort on intRange field - intSortOnIntRangeField.setOptimizeSortWithIndexedData(false); - searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnIntRangeField)); reader.close(); dir.close(); @@ -948,14 +916,8 @@ public void testSortOptimizationOnSortedNumericField() throws IOException { RandomPicks.randomFrom(random(), SortedNumericSelector.Type.values()); boolean reverse = random().nextBoolean(); final SortField sortField = LongField.newSortField("my_field", reverse, type); - sortField.setOptimizeSortWithIndexedData(false); - final Sort sort = new Sort(sortField); // sort without sort optimization - final SortField sortField2 = LongField.newSortField("my_field", reverse, type); - final Sort sort2 = new Sort(sortField2); // sort with sort optimization - - long expectedCollectedHits = 0; - long collectedHits = 0; - long collectedHits2 = 0; + final Sort sort = new Sort(sortField); // sort with sort optimization + int visitedHits = 0; FieldDoc after = null; while (visitedHits < numDocs) { @@ -965,30 +927,14 @@ public void testSortOptimizationOnSortedNumericField() throws IOException { TopDocs topDocs = assertSearchHits(reader, sort, batch, after); ScoreDoc[] scoreDocs = topDocs.scoreDocs; - TopDocs topDocs2 = assertSearchHits(reader, sort2, batch, after); - ScoreDoc[] scoreDocs2 = topDocs2.scoreDocs; - - // assert that the resulting hits are the same assertEquals(expectedHits, topDocs.scoreDocs.length); - assertEquals(topDocs.scoreDocs.length, topDocs2.scoreDocs.length); for (int i = 0; i < scoreDocs.length; i++) { - FieldDoc fieldDoc = (FieldDoc) scoreDocs[i]; - FieldDoc fieldDoc2 = (FieldDoc) scoreDocs2[i]; - assertEquals(fieldDoc.fields[0], fieldDoc2.fields[0]); - assertEquals(fieldDoc.doc, fieldDoc2.doc); visitedHits++; } - expectedCollectedHits += numDocs; - collectedHits += topDocs.totalHits.value(); - collectedHits2 += topDocs2.totalHits.value(); after = (FieldDoc) scoreDocs[expectedHits - 1]; } assertEquals(visitedHits, numDocs); - assertEquals(expectedCollectedHits, collectedHits); - // assert that the second sort with optimization collected less or equal hits - assertTrue(collectedHits >= collectedHits2); - // System.out.println(expectedCollectedHits + "\t" + collectedHits + "\t" + collectedHits2); reader.close(); dir.close(); @@ -1099,7 +1045,7 @@ private void testStringSortOptimization( final DirectoryReader reader = DirectoryReader.open(writer); writer.close(); doTestStringSortOptimization(reader); - doTestStringSortOptimizationDisabled(reader); + reader.close(); dir.close(); } @@ -1202,13 +1148,7 @@ private void testStringSortOptimizationFieldMissingInSegment( KeywordField.newSortField( "my_field", true, SortedSetSelector.Type.MIN, SortField.STRING_LAST); Sort sort = new Sort(sortField); - TopDocs topDocs = assertSearchHits(reader, sort, numHits, null); - - sortField.setOptimizeSortWithIndexedData(false); - sort = new Sort(sortField); - TopDocs unpruned = assertSearchHits(reader, sort, numHits, null); - - CheckHits.checkEqual(MatchAllDocsQuery.INSTANCE, topDocs.scoreDocs, unpruned.scoreDocs); + assertSearchHits(reader, sort, numHits, null); } { // descending sort with missing-first: once the queue fills from the first segment, @@ -1226,13 +1166,7 @@ private void testStringSortOptimizationFieldMissingInSegment( KeywordField.newSortField( "my_field", false, SortedSetSelector.Type.MIN, SortField.STRING_FIRST); Sort sort = new Sort(sortField); - TopDocs topDocs = assertSearchHits(reader, sort, numHits, null); - - sortField.setOptimizeSortWithIndexedData(false); - sort = new Sort(sortField); - TopDocs unpruned = assertSearchHits(reader, sort, numHits, null); - - CheckHits.checkEqual(MatchAllDocsQuery.INSTANCE, topDocs.scoreDocs, unpruned.scoreDocs); + assertSearchHits(reader, sort, numHits, null); } reader.close(); @@ -1342,20 +1276,6 @@ private void doTestStringSortOptimization(DirectoryReader reader) throws IOExcep } } - public void doTestStringSortOptimizationDisabled(DirectoryReader reader) throws IOException { - SortField sortField = - KeywordField.newSortField( - "my_field", false, SortedSetSelector.Type.MIN, SortField.STRING_LAST); - sortField.setOptimizeSortWithIndexedData(false); - - Sort sort = new Sort(sortField); - final int numDocs = reader.numDocs(); - final int numHits = 5; - - TopDocs topDocs = assertSearchHits(reader, sort, numHits, null); - assertEquals(numDocs, topDocs.totalHits.value()); - } - private TopDocs assertSort(DirectoryReader reader, Sort sort, int n, FieldDoc after) throws IOException { TopDocs topDocs = assertSearchHits(reader, sort, n, after);