diff --git a/KNOWN-ISSUES.md b/KNOWN-ISSUES.md index fcc945e2d..a7cfae75b 100644 --- a/KNOWN-ISSUES.md +++ b/KNOWN-ISSUES.md @@ -2,52 +2,7 @@ ## Search Functionality -### 1. NOT Operator Issues - -**Status:** Known bug, not yet fixed - -**Issue:** The NOT operator has two problems: - -1. **Standalone NOT crashes:** `km search "NOT foo"` throws FTS5 syntax error -2. **NOT doesn't exclude:** `km search "foo NOT bar"` returns documents containing both instead of excluding "bar" - -**Examples:** -```bash -# Problem 1: Standalone NOT crashes -km search "NOT important" -# Error: SQLite Error 1: 'fts5: syntax error near "NOT"' - -# Problem 2: NOT doesn't exclude -km put "foo and bar together" -km put "only foo here" -km search "foo NOT bar" -# Expected: 1 result (only foo here) -# Actual: 2 results (both documents) -``` - -**Root Cause:** -- FTS5 requires NOT to have a left operand (e.g., `foo NOT bar`), standalone `NOT term` is invalid -- Even when valid, FTS query extraction passes `"NOT (bar)"` to SQLite FTS5 which doesn't work as expected -- No LINQ post-filtering is applied to exclude NOT terms -- The architecture assumes FTS handles all logic, but NOT needs LINQ filtering - -**Workaround:** -- For literal text containing "NOT", use quotes: `km search '"NOT important"'` -- Avoid using NOT as a boolean operator - -**Fix Required:** -1. Handle standalone NOT gracefully (either treat as literal or provide clear error) -2. Split query: extract positive terms for FTS, negative terms for filtering -3. Apply LINQ filter to FTS results using QueryLinqBuilder -4. Filter out documents matching NOT terms - -**Files Affected:** -- `src/Core/Search/NodeSearchService.cs:190` - ExtractLogical NOT handling -- Need to add LINQ filtering after line 89 - ---- - -### 2. Field Queries with Quoted Values Fail +### 1. Field Queries with Quoted Values Fail **Status:** Known bug, not yet fixed @@ -70,41 +25,10 @@ km search 'content:"user:password"' --- -## Resolved Issues +## Notes -### Quoted Phrases Don't Escape Operators (Resolved) - -**Status:** Fixed - -**Issue:** Cannot search for literal phrases containing reserved words like "AND", "OR", "NOT". - -**Example:** -```bash -km put "Meeting with Alice AND Bob" -km search '"Alice AND Bob"' -# Now works correctly and finds the document -``` - -**Resolution:** -- The tokenizer correctly handles quoted strings and preserves them as literal text -- The FTS query extractor properly quotes phrases containing reserved words -- E2E tests added in `SearchEndToEndTests.cs` to prevent regression (tests: `KnownIssue2_*`) - ---- - -## Testing Gaps - -These bugs were discovered through comprehensive E2E testing. Previous tests only verified: -- AST structure correctness -- LINQ expression building -- Direct FTS calls - -But did NOT test: -- Full pipeline: Parse -> Extract FTS -> Search -> Filter -> Rank -- Default settings (MinRelevance=0.3) -- Actual result verification - -**Lesson:** Exit code testing and structure testing are insufficient. Must test actual behavior with real data. - ---- +### Query Syntax +The infix query parser requires explicit AND between terms: +- `foo AND NOT bar` (correct) instead of `foo NOT bar` (incorrect - ignores NOT) +- `(foo OR baz) AND NOT bar` (correct) instead of `(foo OR baz) NOT bar` (incorrect) diff --git a/src/Core/Search/NodeSearchService.cs b/src/Core/Search/NodeSearchService.cs index b2883d5ab..cd7333abc 100644 --- a/src/Core/Search/NodeSearchService.cs +++ b/src/Core/Search/NodeSearchService.cs @@ -6,6 +6,25 @@ namespace KernelMemory.Core.Search; +/// +/// Result of FTS query extraction from the AST. +/// Contains the FTS query string for SQLite and a list of NOT terms for post-filtering. +/// SQLite FTS5 has limited NOT support (requires left operand), so NOT terms +/// are filtered via LINQ after FTS returns initial results. +/// +/// The FTS5 query string for positive terms. +/// Terms to exclude via LINQ post-filtering. Each term includes optional field info. +[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays")] +public sealed record FtsQueryResult(string FtsQuery, NotTerm[] NotTerms); + +/// +/// Represents a term that should be excluded from search results. +/// Used for LINQ post-filtering since SQLite FTS5 NOT has limitations. +/// +/// The term to exclude. +/// Optional field to check (title/description/content). If null, checks all fields. +public sealed record NotTerm(string Term, string? Field); + /// /// Per-node search service. /// Executes searches within a single node's indexes. @@ -61,12 +80,12 @@ public NodeSearchService( // Query the FTS index var maxResults = request.MaxResultsPerNode ?? SearchConstants.DefaultMaxResultsPerNode; - // Convert QueryNode to FTS query string - var ftsQuery = this.ExtractFtsQuery(queryNode); + // Convert QueryNode to FTS query string and extract NOT terms for post-filtering + var queryResult = this.ExtractFtsQuery(queryNode); // Search the FTS index var ftsMatches = await this._ftsIndex.SearchAsync( - ftsQuery, + queryResult.FtsQuery, maxResults, cts.Token).ConfigureAwait(false); @@ -95,6 +114,13 @@ public NodeSearchService( } } + // Apply NOT term filtering via LINQ (SQLite FTS5 NOT has limitations) + // Filter out any documents that contain the NOT terms + if (queryResult.NotTerms.Length > 0) + { + results = this.ApplyNotTermFiltering(results, queryResult.NotTerms); + } + stopwatch.Stop(); return ([.. results], stopwatch.Elapsed); } @@ -117,11 +143,79 @@ public NodeSearchService( } /// - /// Extract FTS query string from query AST. - /// Converts the AST to SQLite FTS5 query syntax. - /// Only includes text search terms; filtering is done via LINQ on results. + /// Apply NOT term filtering to results via LINQ. + /// Excludes documents that contain any of the NOT terms. + /// + /// The search results to filter. + /// The terms to exclude. + /// Filtered results excluding documents containing NOT terms. + private List ApplyNotTermFiltering(List results, NotTerm[] notTerms) + { + return results + .Where(result => !this.ContainsAnyNotTerm(result, notTerms)) + .ToList(); + } + + /// + /// Check if a result contains any of the NOT terms. + /// + /// The search result to check. + /// The NOT terms to check for. + /// True if the result contains any NOT term. + private bool ContainsAnyNotTerm(SearchIndexResult result, NotTerm[] notTerms) + { + foreach (var notTerm in notTerms) + { + if (this.ContainsNotTerm(result, notTerm)) + { + return true; + } + } + + return false; + } + + /// + /// Check if a result contains a specific NOT term. + /// + /// The search result to check. + /// The NOT term to check for. + /// True if the result contains the NOT term. + private bool ContainsNotTerm(SearchIndexResult result, NotTerm notTerm) + { + // Case-insensitive contains check + var term = notTerm.Term; + + // Check specific field if specified + if (notTerm.Field != null) + { + var fieldValue = notTerm.Field.ToLowerInvariant() switch + { + "title" => result.Title ?? string.Empty, + "description" => result.Description ?? string.Empty, + "content" => result.Content ?? string.Empty, + _ => string.Empty + }; + + return fieldValue.Contains(term, StringComparison.OrdinalIgnoreCase); + } + + // Check all FTS fields (title, description, content) + var title = result.Title ?? string.Empty; + var description = result.Description ?? string.Empty; + var content = result.Content ?? string.Empty; + + return title.Contains(term, StringComparison.OrdinalIgnoreCase) || + description.Contains(term, StringComparison.OrdinalIgnoreCase) || + content.Contains(term, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Extract FTS query string and NOT terms from query AST. + /// Converts the AST to SQLite FTS5 query syntax for positive terms. + /// NOT terms are collected separately for LINQ post-filtering. /// - private string ExtractFtsQuery(QueryNode queryNode) + private FtsQueryResult ExtractFtsQuery(QueryNode queryNode) { var visitor = new FtsQueryExtractor(); return visitor.Extract(queryNode); @@ -131,9 +225,12 @@ private string ExtractFtsQuery(QueryNode queryNode) /// Visitor that extracts FTS query terms from the AST. /// Focuses only on TextSearchNode and field-specific text searches. /// Logical operators are preserved for FTS query syntax. + /// NOT operators are handled specially - their terms are collected for LINQ post-filtering. /// private sealed class FtsQueryExtractor { + private readonly List _notTerms = []; + /// /// SQLite FTS5 reserved words that must be quoted when used as search terms. /// These keywords have special meaning in FTS5 query syntax. @@ -143,10 +240,15 @@ private sealed class FtsQueryExtractor "AND", "OR", "NOT", "NEAR" }; - public string Extract(QueryNode node) + public FtsQueryResult Extract(QueryNode node) { var terms = this.ExtractTerms(node); - return string.IsNullOrEmpty(terms) ? "*" : terms; + + // If only NOT terms exist (no positive terms), use wildcard to get all documents + // then filter with NOT terms + var ftsQuery = string.IsNullOrEmpty(terms) ? "*" : terms; + + return new FtsQueryResult(ftsQuery, [.. this._notTerms]); } private string ExtractTerms(QueryNode node) @@ -198,6 +300,14 @@ private string ExtractTextSearch(TextSearchNode node) private string ExtractLogical(LogicalNode node) { + // Handle NOT and NOR specially - collect terms for LINQ post-filtering + if (node.Operator == LogicalOperator.Not || node.Operator == LogicalOperator.Nor) + { + this.CollectNotTerms(node); + // Return empty string - NOT terms are not included in FTS query + return string.Empty; + } + var childTerms = node.Children .Select(this.ExtractTerms) .Where(t => !string.IsNullOrEmpty(t)) @@ -212,12 +322,60 @@ private string ExtractLogical(LogicalNode node) { LogicalOperator.And => string.Join(" AND ", childTerms.Select(t => $"({t})")), LogicalOperator.Or => string.Join(" OR ", childTerms.Select(t => $"({t})")), - LogicalOperator.Not => childTerms.Length > 0 ? $"NOT ({childTerms[0]})" : string.Empty, - LogicalOperator.Nor => string.Join(" AND ", childTerms.Select(t => $"NOT ({t})")), _ => string.Empty }; } + /// + /// Collect NOT terms from a NOT or NOR node. + /// These terms will be filtered via LINQ after FTS returns results. + /// + private void CollectNotTerms(LogicalNode node) + { + foreach (var child in node.Children) + { + this.CollectNotTermsFromNode(child); + } + } + + /// + /// Recursively collect NOT terms from a node. + /// + private void CollectNotTermsFromNode(QueryNode node) + { + switch (node) + { + case TextSearchNode textNode: + // Extract the term and optional field + this._notTerms.Add(new NotTerm(textNode.SearchText, textNode.Field?.FieldPath)); + break; + + case ComparisonNode comparisonNode: + // Handle field:value comparisons for NOT + if ((comparisonNode.Operator == ComparisonOperator.Contains || + comparisonNode.Operator == ComparisonOperator.Equal) && + comparisonNode.Field?.FieldPath != null && + comparisonNode.Value != null) + { + var term = comparisonNode.Value.AsString(); + this._notTerms.Add(new NotTerm(term, comparisonNode.Field.FieldPath)); + } + + break; + + case LogicalNode logicalNode: + // Recursively collect from nested logical nodes + // For nested NOT/NOR, we add all children as NOT terms + // For nested AND/OR within NOT, all their children become NOT terms + foreach (var child in logicalNode.Children) + { + this.CollectNotTermsFromNode(child); + } + + break; + } + } + private string ExtractComparison(ComparisonNode node) { // Extract text search from Contains OR Equal operator on FTS fields diff --git a/src/Core/Search/Query/Parsers/InfixQueryParser.cs b/src/Core/Search/Query/Parsers/InfixQueryParser.cs index fa9fa5aa6..49da0a3d4 100644 --- a/src/Core/Search/Query/Parsers/InfixQueryParser.cs +++ b/src/Core/Search/Query/Parsers/InfixQueryParser.cs @@ -161,12 +161,13 @@ private List Tokenize(string query) continue; } - // Quoted string - if (query[i] == '"') + // Quoted string (double or single quotes) + if (query[i] == '"' || query[i] == '\'') { + var quoteChar = query[i]; i++; var start = i; - while (i < query.Length && query[i] != '"') + while (i < query.Length && query[i] != quoteChar) { i++; } diff --git a/src/Core/Search/SqliteFtsIndex.cs b/src/Core/Search/SqliteFtsIndex.cs index d7109da16..779f811cd 100644 --- a/src/Core/Search/SqliteFtsIndex.cs +++ b/src/Core/Search/SqliteFtsIndex.cs @@ -149,6 +149,14 @@ public async Task> SearchAsync(string query, int limit = return []; } + // Handle special "*" query to return all documents (for standalone NOT queries) + // FTS5 doesn't support "*" alone as a match-all operator + // Used when query contains only NOT terms - we get all docs and filter externally + if (query == "*") + { + return await this.GetAllDocumentsAsync(limit, cancellationToken).ConfigureAwait(false); + } + // Search using FTS5 MATCH operator // Use bm25() for better scoring (returns negative values, more negative = better match) // We negate and normalize to 0-1 range @@ -208,6 +216,55 @@ LIMIT @limit } } + /// + /// Returns all documents from the FTS index without filtering. + /// Used for standalone NOT queries where we need to get all documents + /// and then filter externally using LINQ. + /// + /// Maximum number of documents to return. + /// Cancellation token. + /// All documents up to the limit. + private async Task> GetAllDocumentsAsync(int limit, CancellationToken cancellationToken) + { + // Select all documents without FTS MATCH filtering + // Since there's no FTS query, we can't use bm25() - assign a default score of 1.0 + var searchSql = $""" + SELECT + content_id, + substr(content, 1, 200) as snippet + FROM {TableName} + LIMIT @limit + """; + + var searchCommand = this._connection!.CreateCommand(); + await using (searchCommand.ConfigureAwait(false)) + { + searchCommand.CommandText = searchSql; + searchCommand.Parameters.AddWithValue("@limit", limit); + + var results = new List(); + var reader = await searchCommand.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await using (reader.ConfigureAwait(false)) + { + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var contentId = reader.GetString(0); + var snippet = reader.GetString(1); + + results.Add(new FtsMatch + { + ContentId = contentId, + Score = 1.0, // Default score for match-all + Snippet = snippet + }); + } + } + + this._logger.LogDebug("GetAllDocuments returned {Count} results", results.Count); + return results; + } + } + /// public async Task ClearAsync(CancellationToken cancellationToken = default) { diff --git a/src/Main/CLI/Commands/ExamplesCommand.cs b/src/Main/CLI/Commands/ExamplesCommand.cs index 3f705c4f4..1eb653837 100644 --- a/src/Main/CLI/Commands/ExamplesCommand.cs +++ b/src/Main/CLI/Commands/ExamplesCommand.cs @@ -139,6 +139,13 @@ private void ShowSearchExamples() AnsiConsole.MarkupLine("[dim]Find documents with either python or javascript[/]"); AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Exclude terms with NOT[/]"); + AnsiConsole.MarkupLine("[cyan]km search \"docker AND NOT kubernetes\"[/]"); + AnsiConsole.MarkupLine("[dim]Find docker docs that don't mention kubernetes[/]"); + AnsiConsole.MarkupLine("[cyan]km search \"NOT deprecated\"[/]"); + AnsiConsole.MarkupLine("[dim]Find all docs that don't contain \"deprecated\"[/]"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Search for literal reserved words[/]"); AnsiConsole.MarkupLine("[cyan]km search '\"NOT\"'[/]"); AnsiConsole.MarkupLine("[dim]Use quotes to search for literal AND, OR, NOT as words[/]"); @@ -162,6 +169,13 @@ private void ShowSearchExamples() AnsiConsole.MarkupLine("[dim]Full-text search across all fields[/]"); AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]JSON format - exclude with $not and $nor[/]"); + AnsiConsole.MarkupLine($"[cyan]{Markup.Escape("km search '{\"content\": {\"$not\": \"deprecated\"}}'")}[/]"); + AnsiConsole.MarkupLine("[dim]Find docs where content doesn't contain \"deprecated\"[/]"); + AnsiConsole.MarkupLine($"[cyan]{Markup.Escape("km search '{\"$nor\": [{\"content\": \"alpha\"}, {\"content\": \"beta\"}]}'")}[/]"); + AnsiConsole.MarkupLine("[dim]Exclude docs containing \"alpha\" or \"beta\"[/]"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]JSON format - escaping special characters[/]"); AnsiConsole.MarkupLine($"[cyan]{Markup.Escape("km search '{\"content\": \"quotes: \\\"hello\\\"\"}'")}[/]"); AnsiConsole.MarkupLine("[dim]Escape quotes in JSON with backslash[/]"); diff --git a/tests/Core.Tests/Search/SearchEndToEndTests.cs b/tests/Core.Tests/Search/SearchEndToEndTests.cs index 04e2f7c50..7792982cf 100644 --- a/tests/Core.Tests/Search/SearchEndToEndTests.cs +++ b/tests/Core.Tests/Search/SearchEndToEndTests.cs @@ -745,4 +745,210 @@ public async Task KnownIssue2_MixedQuotedPhraseAndOperator_WorksCorrectly() } #endregion + + #region Known Issue 1: NOT Operator Fixes + + [Fact] + public async Task KnownIssue1_StandaloneNOT_DoesNotCrash() + { + // Known Issue 1: Standalone NOT crashes with FTS5 syntax error + // The query "NOT foo" should not throw an exception + // Expected behavior: Return all documents that do NOT contain "foo" + + // Arrange + await this.InsertAsync("doc1", "contains foo term").ConfigureAwait(false); + await this.InsertAsync("doc2", "does not contain the term").ConfigureAwait(false); + await this.InsertAsync("doc3", "another document without it").ConfigureAwait(false); + + // Act: This should NOT throw an exception + var response = await this.SearchAsync("NOT foo").ConfigureAwait(false); + + // Assert: Should return documents that do NOT contain "foo" + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Contains "foo" + Assert.Contains(this._insertedIds["doc2"], resultIds); // No "foo" + Assert.Contains(this._insertedIds["doc3"], resultIds); // No "foo" + } + + [Fact] + public async Task KnownIssue1_NotWithPositiveTerm_ExcludesCorrectly() + { + // Known Issue 1: "foo AND NOT bar" should exclude documents with "bar" + // Expected behavior: Return documents with "foo" but NOT "bar" + // Note: Explicit AND is required - "foo NOT bar" is parsed as just "foo" + + // Arrange + await this.InsertAsync("doc1", "foo and bar together").ConfigureAwait(false); + await this.InsertAsync("doc2", "only foo here").ConfigureAwait(false); + await this.InsertAsync("doc3", "only bar here").ConfigureAwait(false); + await this.InsertAsync("doc4", "neither term here").ConfigureAwait(false); + + // Act: Search for "foo AND NOT bar" (explicit AND required) + var response = await this.SearchAsync("foo AND NOT bar").ConfigureAwait(false); + + // Assert: Should return only doc2 (has "foo" but not "bar") + Assert.Equal(1, response.TotalResults); + Assert.Single(response.Results); + Assert.Equal(this._insertedIds["doc2"], response.Results[0].Id); + } + + [Fact] + public async Task KnownIssue1_MultipleNOT_ExcludesAllTerms() + { + // Known Issue 1: Multiple NOT terms should all be excluded + // Expected behavior: "foo AND NOT bar AND NOT baz" returns docs with "foo" but without "bar" and without "baz" + // Note: Explicit AND is required between all terms + + // Arrange + await this.InsertAsync("doc1", "foo bar baz all").ConfigureAwait(false); + await this.InsertAsync("doc2", "foo bar only").ConfigureAwait(false); + await this.InsertAsync("doc3", "foo baz only").ConfigureAwait(false); + await this.InsertAsync("doc4", "foo alone here").ConfigureAwait(false); + + // Act: Search for foo but not bar and not baz (explicit AND required) + var response = await this.SearchAsync("foo AND NOT bar AND NOT baz").ConfigureAwait(false); + + // Assert: Should return only doc4 (has "foo" without "bar" or "baz") + Assert.Equal(1, response.TotalResults); + Assert.Single(response.Results); + Assert.Equal(this._insertedIds["doc4"], response.Results[0].Id); + } + + [Fact] + public async Task KnownIssue1_NOTWithOR_WorksCorrectly() + { + // Combined NOT with OR: "(foo OR baz) AND NOT bar" + // Should return documents with "foo" OR "baz" but NOT "bar" + // Note: Explicit AND is required between the OR group and NOT + + // Arrange + await this.InsertAsync("doc1", "foo and bar").ConfigureAwait(false); + await this.InsertAsync("doc2", "foo alone").ConfigureAwait(false); + await this.InsertAsync("doc3", "baz and bar").ConfigureAwait(false); + await this.InsertAsync("doc4", "baz alone").ConfigureAwait(false); + await this.InsertAsync("doc5", "neither term").ConfigureAwait(false); + + // Act: Search for (foo OR baz) AND NOT bar (explicit AND required) + var response = await this.SearchAsync("(foo OR baz) AND NOT bar").ConfigureAwait(false); + + // Assert: Should return doc2 and doc4 (have foo/baz but not bar) + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.Contains(this._insertedIds["doc2"], resultIds); + Assert.Contains(this._insertedIds["doc4"], resultIds); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Has bar + Assert.DoesNotContain(this._insertedIds["doc3"], resultIds); // Has bar + } + + [Fact] + public async Task KnownIssue1_NOTInFieldQuery_ExcludesFromField() + { + // NOT with field-specific search: "content:foo AND NOT content:bar" + // Should search in content field specifically + // Note: Explicit AND is required + + // Arrange + await this.InsertAsync("doc1", "foo bar in content", "title1").ConfigureAwait(false); + await this.InsertAsync("doc2", "foo only in content", "title2").ConfigureAwait(false); + await this.InsertAsync("doc3", "different content", "foo bar title").ConfigureAwait(false); + + // Act: Search for foo in content but not bar in content (explicit AND required) + var response = await this.SearchAsync("content:foo AND NOT content:bar").ConfigureAwait(false); + + // Assert: Should return only doc2 (has foo in content, no bar in content) + Assert.Equal(1, response.TotalResults); + Assert.Single(response.Results); + Assert.Equal(this._insertedIds["doc2"], response.Results[0].Id); + } + + [Fact] + public async Task KnownIssue1_MongoNot_ExcludesCorrectly() + { + // MongoDB $not operator should work correctly + // $not: excludes documents matching the condition + + // Arrange + await this.InsertAsync("doc1", "kubernetes deployment").ConfigureAwait(false); + await this.InsertAsync("doc2", "docker deployment").ConfigureAwait(false); + await this.InsertAsync("doc3", "other content").ConfigureAwait(false); + + // Act: MongoDB NOT - find documents NOT containing "kubernetes" + var response = await this.SearchAsync("{\"$not\": {\"content\": \"kubernetes\"}}").ConfigureAwait(false); + + // Assert: Should return doc2 and doc3 (not containing kubernetes) + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Contains kubernetes + Assert.Contains(this._insertedIds["doc2"], resultIds); + Assert.Contains(this._insertedIds["doc3"], resultIds); + } + + [Fact] + public async Task KnownIssue1_MongoNor_ExcludesAllConditions() + { + // MongoDB $nor operator should exclude all conditions + + // Arrange + await this.InsertAsync("doc1", "kubernetes deployment").ConfigureAwait(false); + await this.InsertAsync("doc2", "docker deployment").ConfigureAwait(false); + await this.InsertAsync("doc3", "helm charts").ConfigureAwait(false); + await this.InsertAsync("doc4", "ansible automation").ConfigureAwait(false); + + // Act: MongoDB NOR - find documents NOT containing kubernetes NOR docker + var response = await this.SearchAsync("{\"$nor\": [{\"content\": \"kubernetes\"}, {\"content\": \"docker\"}]}").ConfigureAwait(false); + + // Assert: Should return doc3 and doc4 (not containing kubernetes or docker) + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Contains kubernetes + Assert.DoesNotContain(this._insertedIds["doc2"], resultIds); // Contains docker + Assert.Contains(this._insertedIds["doc3"], resultIds); + Assert.Contains(this._insertedIds["doc4"], resultIds); + } + + [Fact] + public async Task KnownIssue1_NOTWithSingleQuotedReservedWord_ExcludesCorrectly() + { + // Single-quoted reserved words in NOT should be excluded correctly + // Bug: NOT 'AND' was searching for literal "'AND'" instead of "AND" + + // Arrange + await this.InsertAsync("doc1", "Meeting with Alice AND Bob tomorrow").ConfigureAwait(false); + await this.InsertAsync("doc2", "Regular meeting notes").ConfigureAwait(false); + await this.InsertAsync("doc3", "Project status update").ConfigureAwait(false); + + // Act: NOT with single-quoted AND (reserved word) - should exclude docs containing literal AND + var response = await this.SearchAsync("NOT 'AND'").ConfigureAwait(false); + + // Assert: Should return doc2 and doc3 (not containing "AND") + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Contains "AND" + Assert.Contains(this._insertedIds["doc2"], resultIds); + Assert.Contains(this._insertedIds["doc3"], resultIds); + } + + [Fact] + public async Task KnownIssue1_NOTWithDoubleQuotedReservedWord_ExcludesCorrectly() + { + // Double-quoted reserved words in NOT should be excluded correctly + + // Arrange + await this.InsertAsync("doc1", "This is NOT important").ConfigureAwait(false); + await this.InsertAsync("doc2", "Regular document content").ConfigureAwait(false); + await this.InsertAsync("doc3", "Something else entirely").ConfigureAwait(false); + + // Act: NOT with double-quoted NOT (reserved word) - should exclude docs containing literal NOT + var response = await this.SearchAsync("NOT \"NOT\"").ConfigureAwait(false); + + // Assert: Should return doc2 and doc3 (not containing "NOT") + Assert.Equal(2, response.TotalResults); + var resultIds = response.Results.Select(r => r.Id).ToHashSet(); + Assert.DoesNotContain(this._insertedIds["doc1"], resultIds); // Contains "NOT" + Assert.Contains(this._insertedIds["doc2"], resultIds); + Assert.Contains(this._insertedIds["doc3"], resultIds); + } + + #endregion }