diff --git a/internal/agent/encoding/agent.go b/internal/agent/encoding/agent.go index d585e6a..1a95aed 100644 --- a/internal/agent/encoding/agent.go +++ b/internal/agent/encoding/agent.go @@ -72,8 +72,9 @@ type EncodingConfig struct { BatchSizeEvent int // batch size for EncodeAllPending (default: 50) BatchSizePoll int // batch size for polling loop (default: 10) EmbedBatchSize int // max memories to batch-embed in one call (default 10) - DeduplicationThreshold float32 // cosine sim above which new memory is a duplicate (default: 0.9) - SalienceFloor float32 // min salience to encode; non-MCP sources below this are skipped (default: 0.0) + DeduplicationThreshold float32 // cosine sim above which new memory is a duplicate (default: 0.95) + MCPDeduplicationThreshold float32 // higher threshold for MCP-sourced memories (default: 0.98) + SalienceFloor float32 // min salience to encode; non-MCP sources below this are skipped (default: 0.0) DisablePolling bool // if true, skip the polling loop (MCP processes should not poll) } @@ -124,8 +125,10 @@ func DefaultConfig() EncodingConfig { BackoffThreshold: 3, BackoffBaseSec: 30, BackoffMaxSec: 300, - BatchSizeEvent: 50, - BatchSizePoll: 10, + BatchSizeEvent: 50, + BatchSizePoll: 10, + DeduplicationThreshold: 0.95, + MCPDeduplicationThreshold: 0.98, } } @@ -710,11 +713,8 @@ func (ea *EncodingAgent) finalizeEncodedMemory(ctx context.Context, raw store.Ra ea.log.Warn("failed to search for similar memories", "raw_id", raw.ID, "error", err) } else { // Check for near-duplicate before creating a new memory - dedupThreshold := ea.config.DeduplicationThreshold - if dedupThreshold <= 0 { - dedupThreshold = 0.9 - } - if dup := findDuplicate(similar, dedupThreshold); dup != nil { + dc := ea.buildDedupContext(raw) + if dup := findDuplicate(similar, dc); dup != nil { ea.log.Info("dedup: boosting existing memory instead of creating duplicate", "raw_id", raw.ID, "existing_id", dup.Memory.ID, @@ -990,11 +990,8 @@ func (ea *EncodingAgent) encodeMemory(ctx context.Context, rawID string) error { ea.log.Debug("similarity search completed", "raw_id", raw.ID, "results", len(similar)) // Dedup check: if a near-duplicate already exists, boost it instead of creating a new memory - dedupThreshold := ea.config.DeduplicationThreshold - if dedupThreshold <= 0 { - dedupThreshold = 0.9 - } - if dup := findDuplicate(similar, dedupThreshold); dup != nil { + dc := ea.buildDedupContext(raw) + if dup := findDuplicate(similar, dc); dup != nil { ea.log.Info("dedup: boosting existing memory instead of creating duplicate", "raw_id", raw.ID, "existing_id", dup.Memory.ID, "similarity", dup.Score) newSalience := dup.Memory.Salience + 0.05 @@ -1939,12 +1936,64 @@ func truncateString(s string, maxLen int) string { return string(runes[:maxLen]) + "..." } -// findDuplicate returns the first result above the dedup threshold, or nil. -func findDuplicate(results []store.RetrievalResult, threshold float32) *store.RetrievalResult { +// buildDedupContext creates a dedup context from the agent config and raw memory. +func (ea *EncodingAgent) buildDedupContext(raw store.RawMemory) dedupContext { + threshold := ea.config.DeduplicationThreshold + if threshold <= 0 { + threshold = 0.95 + } + mcpThreshold := ea.config.MCPDeduplicationThreshold + if mcpThreshold <= 0 { + mcpThreshold = 0.98 + } + return dedupContext{ + Threshold: threshold, + MCPThreshold: mcpThreshold, + RawSource: raw.Source, + RawType: raw.Type, + RawProject: raw.Project, + } +} + +// dedupContext holds the context needed for smart deduplication decisions. +type dedupContext struct { + Threshold float32 // base cosine similarity threshold + MCPThreshold float32 // higher threshold for MCP-sourced memories (explicit user input) + RawSource string // source of the incoming memory + RawType string // type of the incoming memory (decision, error, insight, etc.) + RawProject string // project of the incoming memory +} + +// findDuplicate returns the best dedup candidate, applying type-aware, +// project-aware, and source-aware filtering. Returns nil if no valid +// duplicate is found. +// +// Rules: +// - Never dedup across different memory types (decision != error) +// - Never dedup across different projects +// - MCP-sourced memories use a higher threshold (default 0.98) since +// they represent explicit user/agent input worth preserving +// - All other sources use the base threshold (default 0.95) +func findDuplicate(results []store.RetrievalResult, dc dedupContext) *store.RetrievalResult { + threshold := dc.Threshold + if dc.RawSource == "mcp" && dc.MCPThreshold > 0 { + threshold = dc.MCPThreshold + } + for i := range results { - if results[i].Score >= threshold { - return &results[i] + r := &results[i] + if r.Score < threshold { + continue + } + // Skip cross-type dedup: a decision and an error are never duplicates. + if dc.RawType != "" && r.Memory.Type != "" && dc.RawType != r.Memory.Type { + continue + } + // Skip cross-project dedup: same topic in different projects is distinct. + if dc.RawProject != "" && r.Memory.Project != "" && dc.RawProject != r.Memory.Project { + continue } + return r } return nil } diff --git a/internal/agent/encoding/agent_test.go b/internal/agent/encoding/agent_test.go index 30180e6..333c162 100644 --- a/internal/agent/encoding/agent_test.go +++ b/internal/agent/encoding/agent_test.go @@ -2100,12 +2100,14 @@ func TestCompressionResponseRoundTrip(t *testing.T) { // --------------------------------------------------------------------------- func TestFindDuplicate(t *testing.T) { + baseDC := dedupContext{Threshold: 0.9, MCPThreshold: 0.98} + t.Run("returns first result above threshold", func(t *testing.T) { results := []store.RetrievalResult{ {Memory: store.Memory{ID: "a"}, Score: 0.95}, {Memory: store.Memory{ID: "b"}, Score: 0.85}, } - dup := findDuplicate(results, 0.9) + dup := findDuplicate(results, baseDC) if dup == nil { t.Fatal("expected duplicate to be found") } @@ -2119,14 +2121,14 @@ func TestFindDuplicate(t *testing.T) { {Memory: store.Memory{ID: "a"}, Score: 0.85}, {Memory: store.Memory{ID: "b"}, Score: 0.70}, } - dup := findDuplicate(results, 0.9) + dup := findDuplicate(results, baseDC) if dup != nil { t.Errorf("expected nil, got %q", dup.Memory.ID) } }) t.Run("empty results returns nil", func(t *testing.T) { - dup := findDuplicate(nil, 0.9) + dup := findDuplicate(nil, baseDC) if dup != nil { t.Error("expected nil for empty results") } @@ -2136,9 +2138,75 @@ func TestFindDuplicate(t *testing.T) { results := []store.RetrievalResult{ {Memory: store.Memory{ID: "a"}, Score: 0.9}, } - dup := findDuplicate(results, 0.9) + dup := findDuplicate(results, baseDC) if dup == nil { t.Fatal("expected duplicate at exact threshold") } }) + + t.Run("skips cross-type dedup", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a", Type: "error"}, Score: 0.99}, + } + dc := dedupContext{Threshold: 0.9, RawType: "decision"} + dup := findDuplicate(results, dc) + if dup != nil { + t.Error("should not dedup across different types") + } + }) + + t.Run("allows same-type dedup", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a", Type: "decision"}, Score: 0.95}, + } + dc := dedupContext{Threshold: 0.9, RawType: "decision"} + dup := findDuplicate(results, dc) + if dup == nil { + t.Fatal("should dedup same type above threshold") + } + }) + + t.Run("skips cross-project dedup", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a", Project: "felix-lm"}, Score: 0.99}, + } + dc := dedupContext{Threshold: 0.9, RawProject: "mnemonic"} + dup := findDuplicate(results, dc) + if dup != nil { + t.Error("should not dedup across different projects") + } + }) + + t.Run("MCP source uses higher threshold", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a"}, Score: 0.96}, + } + dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "mcp"} + dup := findDuplicate(results, dc) + if dup != nil { + t.Error("MCP at 0.96 should NOT dedup when MCP threshold is 0.98") + } + }) + + t.Run("MCP source dedupes above MCP threshold", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a"}, Score: 0.99}, + } + dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "mcp"} + dup := findDuplicate(results, dc) + if dup == nil { + t.Fatal("MCP at 0.99 should dedup when MCP threshold is 0.98") + } + }) + + t.Run("non-MCP source uses base threshold", func(t *testing.T) { + results := []store.RetrievalResult{ + {Memory: store.Memory{ID: "a"}, Score: 0.96}, + } + dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "filesystem"} + dup := findDuplicate(results, dc) + if dup == nil { + t.Fatal("filesystem at 0.96 should dedup when base threshold is 0.95") + } + }) } diff --git a/internal/mcp/server.go b/internal/mcp/server.go index e0a069a..5a26d99 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -382,7 +382,7 @@ func (srv *MCPServer) handleRemember(ctx context.Context, args map[string]interf srv.log.Info("memory stored", "id", raw.ID, "source", source, "type", memType, "project", project) - return toolResult(fmt.Sprintf("Stored memory %s (type: %s, project: %s)\n Raw ID: %s\n Initial salience: %.2f\n Encoding: queued (async)\n\nTip: Use check_memory with raw_id \"%s\" to verify encoding status.", + return toolResult(fmt.Sprintf("Stored memory %s (type: %s, project: %s)\n Raw ID: %s\n Initial salience: %.2f\n Encoding: queued (async)\n\nTip: Use check_memory with raw_id %q to verify encoding status. Dedup protections: same-type, same-project, source-aware thresholds.", raw.ID, memType, project, raw.ID, raw.InitialSalience, raw.ID)), nil } @@ -1757,9 +1757,9 @@ func (srv *MCPServer) handleCheckMemory(ctx context.Context, args map[string]int if err != nil { return toolResult(fmt.Sprintf("No memory found for raw_id %q or memory_id %q.", rawID, memoryID)), nil } - status := "pending" + status := "pending encoding" if raw.Processed { - status = "processed (encoding may have been deduplicated)" + status = "deduplicated — a similar memory already existed, so this one boosted its salience instead of creating a duplicate" } return toolResult(fmt.Sprintf("Raw memory %s found but not yet encoded.\n Status: %s\n Source: %s\n Type: %s\n Salience: %.2f\n Created: %s", raw.ID, status, raw.Source, raw.Type, raw.InitialSalience, raw.CreatedAt.Format(time.RFC3339))), nil