Fix OR filters returning duplicate documents when using multiple indexes (#1184)

* Initial plan

* Add test case for OR filter duplicate results issue

Co-authored-by: anidotnet <696662+anidotnet@users.noreply.github.com>

* Fix OR filter duplicate results with multiple indexes

Co-authored-by: anidotnet <696662+anidotnet@users.noreply.github.com>

* Update tests to reflect correct OR filter behavior

Co-authored-by: anidotnet <696662+anidotnet@users.noreply.github.com>

* Improve test comments with document field values

Co-authored-by: anidotnet <696662+anidotnet@users.noreply.github.com>

* Update expected cursor size in compound index tests

Changed the expected cursor size from 5 to 3 in CollectionFindByCompoundIndexTest for both mvstore and rocksdb adapters to reflect the correct number of results returned by the query.

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: anidotnet <696662+anidotnet@users.noreply.github.com>
Co-authored-by: Anindya Chatterjee <anidotnet@gmail.com>

Author: Copilot

nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java

@@ -200,7 +200,7 @@ public void testFindByOrFilter() throws ParseException {
 
         FindPlan findPlan = cursor.getFindPlan();
         assertEquals(3, findPlan.getSubPlans().size());
-        assertEquals(5, cursor.size());
+        assertEquals(3, cursor.size());
 
         // distinct
         cursor = collection.find(

nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java

@@ -200,7 +200,7 @@ public void testFindByOrFilter() throws ParseException {
 
         FindPlan findPlan = cursor.getFindPlan();
         assertEquals(3, findPlan.getSubPlans().size());
-        assertEquals(5, cursor.size());
+        assertEquals(3, cursor.size());
 
         // distinct
         cursor = collection.find(

nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java

@@ -127,10 +127,9 @@ private RecordStream<Pair<NitriteId, Document>> findSuitableStream(FindPlan find
             // concat all suitable stream of all sub plans
             rawStream = new ConcatStream(subStreams);
 
-            if (findPlan.isDistinct()) {
-                // return only distinct items
-                rawStream = new DistinctStream(rawStream);
-            }
+            // Always apply distinct stream for OR filters to avoid duplicates
+            // when the same document matches multiple sub-plans (different indexes)
+            rawStream = new DistinctStream(rawStream);
         } else {
             // and or single filter
             if (findPlan.getByIdFilter() != null) {

nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java

@@ -86,7 +86,13 @@ public void testFindByOrFilterAndFilter() {
             )
         );
 
-        assertEquals(3, cursor.size());
+        // With the fix, OR filters no longer return duplicates
+        // doc2 = {firstName: "fn2", lastName: "ln2"}
+        // doc3 = {firstName: "fn3", lastName: "ln2"}
+        // First AND: lastName=ln2 AND firstName!=fn1 → matches doc2 and doc3
+        // Second AND: firstName=fn3 AND lastName=ln2 → matches doc3
+        // Union without duplicates: doc2 and doc3 (2 total)
+        assertEquals(2, cursor.size());
 
         FindPlan findPlan = cursor.getFindPlan();
         assertNull(findPlan.getIndexScanFilter());
@@ -101,11 +107,11 @@ public void testFindByOrFilterAndFilter() {
             d.get("firstName", String.class).equals("fn2")
                 && d.get("lastName", String.class).equals("ln2")).count());
 
-        assertEquals(2, cursor.toList().stream().filter(d ->
+        assertEquals(1, cursor.toList().stream().filter(d ->
             d.get("firstName", String.class).equals("fn3")
                 && d.get("lastName", String.class).equals("ln2")).count());
 
-        // distinct test
+        // distinct test - should still return the same results since we're already deduplicating
         cursor = collection.find(
             or(
                 and(
@@ -231,11 +237,21 @@ public void testFindByOrFilter() throws ParseException {
             )
         );
 
+        // With the fix, OR filters no longer return duplicates
+        // doc1 = {firstName: "fn1", lastName: "ln1", birthDay: "2012-07-01"}
+        // doc2 = {firstName: "fn2", lastName: "ln2", birthDay: "2010-06-12"}
+        // doc3 = {firstName: "fn3", lastName: "ln2", birthDay: "2014-04-17"}
+        // Flattened OR conditions:
+        // 1. lastName=ln2 → doc2, doc3
+        // 2. firstName!=fn1 → doc2, doc3
+        // 3. birthDay=2012-07-01 → doc1
+        // 4. firstName!=fn1 → doc2, doc3 (duplicate)
+        // Union without duplicates: doc1, doc2, doc3 (3 total)
         FindPlan findPlan = cursor.getFindPlan();
         assertEquals(3, findPlan.getSubPlans().size());
-        assertEquals(5, cursor.size());
+        assertEquals(3, cursor.size());
 
-        // distinct
+        // distinct test - should still return the same results since we're already deduplicating
         cursor = collection.find(
             or(
                 or(

nitrite/src/test/java/org/dizitart/no2/integration/collection/IssueTest.java

@@ -2,14 +2,20 @@
 
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.collection.DocumentCursor;
 import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.filters.FluentFilter;
 import org.dizitart.no2.index.IndexOptions;
 import org.dizitart.no2.index.IndexType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
 
 public class IssueTest {
@@ -47,4 +53,32 @@ public void testOriginalIssue() {
         assertEquals(1, collection.find(FluentFilter.where("value").lte(42L)).size());
         assertEquals(1, collection.find(FluentFilter.where("value").gte(42L)).size());
     }
+
+    @Test
+    public void testMultipleIndexesOrFilterDuplicates() {
+        NitriteCollection items = db.getCollection("items");
+        items.createIndex(IndexOptions.indexOptions(IndexType.NON_UNIQUE), "field_a");
+        items.createIndex(IndexOptions.indexOptions(IndexType.NON_UNIQUE), "field_b");
+
+        Document doc = Document.createDocument();
+        doc.put("field_a", "A");
+        doc.put("field_b", "B");
+        items.insert(doc);
+
+        Filter aFilter = FluentFilter.where("field_a").eq("A");
+        Filter bFilter = FluentFilter.where("field_b").eq("B");
+
+        Filter orFilter = Filter.or(aFilter, bFilter);
+
+        DocumentCursor cursor = items.find(orFilter);
+        Iterator<Document> docIter = cursor.iterator();
+
+        List<Long> matches = new ArrayList<>();
+        while (docIter.hasNext()) {
+            Document match = docIter.next();
+            long id = match.getId().getIdValue();
+            matches.add(id);
+        }
+        assertEquals("Single document must yield single match", 1, matches.size());
+    }
 }